From f665ba60aee74c19b31feace9364db6facbed1dd Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 31 Oct 2026 17:46:36 +0100 Subject: [PATCH 1/5] engines: pin the bound of ReadHeaderTimeout on a silent connection (Refs #1189) A connection that sends nothing, or one byte, must be closed within ReadHeaderTimeout plus a stated slack of its connect (TCP_DEFER_ACCEPT off), and within the kernel's deferral window plus ReadHeaderTimeout plus the slack with it on. Both native engines run the scenarios (monotest). On io_uring a second test asserts the header timer of a connection whose protocol is not decided yet is in flight when the sweep meets it. RED on main for io_uring: the silent and one-byte connections are closed at 1.7 to 1.9 s for a 1 s ReadHeaderTimeout (9 of 10 runs; the phase of the engine's sweep decides, and one run landed before the deadline), and no sweep finds the timer armed (every run). epoll passes (a 25 ms timerfd sweep): it is the pin. --- .../silent_conn_bound_1189_linux_test.go | 58 ++++++++++ .../silent_conn_bound_1189_linux_test.go | 104 ++++++++++++++++++ internal/mono/monotest/silent.go | 95 ++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 internal/engine/epoll/silent_conn_bound_1189_linux_test.go create mode 100644 internal/engine/iouring/silent_conn_bound_1189_linux_test.go create mode 100644 internal/mono/monotest/silent.go diff --git a/internal/engine/epoll/silent_conn_bound_1189_linux_test.go b/internal/engine/epoll/silent_conn_bound_1189_linux_test.go new file mode 100644 index 000000000..8c97e2d99 --- /dev/null +++ b/internal/engine/epoll/silent_conn_bound_1189_linux_test.go @@ -0,0 +1,58 @@ +//go:build linux + +package epoll + +import ( + "context" + "net" + "testing" + "time" + + "github.com/goceleris/celeris/internal/mono/monotest" + "github.com/goceleris/celeris/internal/protocol/h2/stream" + "github.com/goceleris/celeris/internal/resource" +) + +// celeris#1189. A connection that sends nothing is closed at ReadHeaderTimeout from +// the engine's accept: no later than the 25 ms sweep of the timerfd after it, and, +// with TCP_DEFER_ACCEPT on (the default), the kernel's deferral of a silent connection +// (about 1 s) is added before the accept. See monotest.SilentConnIsCutAtReadHeaderTimeout. +func TestSilentConnIsCutAtReadHeaderTimeout1189(t *testing.T) { + monotest.SilentConnIsCutAtReadHeaderTimeout(t, startSilent1189) +} + +func TestSilentConnIsCutAfterDeferAccept1189(t *testing.T) { + monotest.SilentConnIsCutAfterDeferAccept(t, startSilent1189) +} + +func startSilent1189(t *testing.T, cfg resource.Config, h stream.Handler) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("pick port: %v", err) + } + cfg.Addr = ln.Addr().String() + _ = ln.Close() + e, err := New(cfg, h) + if err != nil { + t.Fatalf("epoll engine: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { errCh <- e.Listen(ctx) }() + t.Cleanup(func() { + cancel() + select { + case <-errCh: + case <-time.After(10 * time.Second): + t.Error("epoll engine did not stop within 10 s") + } + }) + for dl := time.Now().Add(10 * time.Second); e.Addr() == nil && time.Now().Before(dl); { + time.Sleep(10 * time.Millisecond) + } + if e.Addr() == nil { + t.Fatal("engine did not bind") + } + return e.Addr().String() +} diff --git a/internal/engine/iouring/silent_conn_bound_1189_linux_test.go b/internal/engine/iouring/silent_conn_bound_1189_linux_test.go new file mode 100644 index 000000000..7c8fa957e --- /dev/null +++ b/internal/engine/iouring/silent_conn_bound_1189_linux_test.go @@ -0,0 +1,104 @@ +//go:build linux + +package iouring + +import ( + "net" + "sync" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" + "github.com/goceleris/celeris/internal/mono/monotest" + "github.com/goceleris/celeris/internal/protocol/h2/stream" + "github.com/goceleris/celeris/internal/resource" +) + +// celeris#1189. A connection that sends nothing is closed at ReadHeaderTimeout from +// the engine's accept, by a timer of its own armed at the accept (before, by the +// sweep, which came up to 0.86 s late), and, with TCP_DEFER_ACCEPT on (the default), +// the kernel's deferral of a silent connection (about 1 s) is added before the accept. +// See monotest.SilentConnIsCutAtReadHeaderTimeout. +func TestSilentConnIsCutAtReadHeaderTimeout1189(t *testing.T) { + monotest.SilentConnIsCutAtReadHeaderTimeout(t, startSilent1189) +} + +func TestSilentConnIsCutAfterDeferAccept1189(t *testing.T) { + monotest.SilentConnIsCutAfterDeferAccept(t, startSilent1189) +} + +func startSilent1189(t *testing.T, cfg resource.Config, h stream.Handler) string { + t.Helper() + r := startEngine806(t, h, func(c *resource.Config) { + addr := c.Addr + *c = cfg + c.Addr = addr + }) + return r.addr +} + +// A connection whose protocol is not decided yet has its header deadline armed in the +// kernel when it is accepted, not left to the sweep (celeris#1189). The sweep of an +// idle worker comes every 0.8 s, so a deadline enforced there is up to 0.8 s late. +// The check is the sweep's own: every sweep that meets a connection that is still +// undecided and has a deadline must find the timer in flight. A sweep that finds +// none means the deadline waits for a sweep. +func TestUndecidedConnHasItsHeaderTimerArmed1189(t *testing.T) { + var ( + mu sync.Mutex + undecided, armed int + addr string + ) + hook := func(w *Worker) { + mu.Lock() + defer mu.Unlock() + // The hook is the package's, not the engine's: workers of an earlier test may + // still be winding down. + if addr == "" || w.cfg.Addr != addr { + return + } + for _, fd := range w.liveConns { + cs := w.conns[fd] + if cs == nil || cs.detected || cs.detectDeadline == 0 { + continue + } + undecided++ + if cs.headerTimerArmed { + armed++ + } + } + } + testHookCheckTimeouts.Store(&hook) + t.Cleanup(func() { testHookCheckTimeouts.Store(nil) }) + r := startEngine806(t, nil, func(c *resource.Config) { + c.Protocol = engine.Auto + c.ReadHeaderTimeout = 10 * time.Second + }) + mu.Lock() + addr = r.addr + mu.Unlock() + c, err := net.DialTimeout("tcp", r.addr, 3*time.Second) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + // A sweep is at most about a second away; wait for three looks at the connection. + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + mu.Lock() + n := undecided + mu.Unlock() + if n >= 3 { + break + } + time.Sleep(20 * time.Millisecond) + } + mu.Lock() + defer mu.Unlock() + if undecided == 0 { + t.Fatal("no sweep met the undecided connection") + } + if armed != undecided { + t.Errorf("%d of %d sweeps that met the undecided connection found its header timer in flight, want all (the deadline is enforced by the sweep)", armed, undecided) + } +} diff --git a/internal/mono/monotest/silent.go b/internal/mono/monotest/silent.go new file mode 100644 index 000000000..aaf60c2e7 --- /dev/null +++ b/internal/mono/monotest/silent.go @@ -0,0 +1,95 @@ +package monotest + +import ( + "net" + "testing" + "time" +) + +// The bound of ReadHeaderTimeout on a connection that sends nothing (celeris#1189). +// The scenarios here are not about the clock: they pin how late the deadline is +// enforced, on the real clock, for both native engines. +// +// The engines enforce the deadline from the accept. With TCP_DEFER_ACCEPT (the +// default) the accept of a silent connection is the kernel's promotion of it, one +// SYN-ACK retransmission timeout (1 s) after the SYN; measured 1.00 to 1.07 s +// (docker arm64, both engines), and deferlinger.DefaultLinger budgets 1.5 s for it. +const ( + silentTimeout = time.Second + + // silentSlack is what a silent connection may exceed its deadline by once it + // has been accepted. epoll sweeps every 25 ms and io_uring arms a timer of its + // own, so a connection is closed within about 30 ms of its deadline (measured, + // docker arm64 with 2 CPUs, 20 runs per engine); 400 ms is more than ten times + // that, and is not tuned to the machine. The io_uring sweep that enforced the + // deadline before celeris#1189 came 0.54 to 0.86 s late (40 runs). + silentSlack = 400 * time.Millisecond + + // settle is how long the engine idles before the connection, for its workers' + // idle backoff (1 ms, then 5 ms, then 25 ms waits) to have run its course. + settle = 700 * time.Millisecond + + // deferWindow is the longest a silent connection waits in the kernel, with + // TCP_DEFER_ACCEPT on, before the engine accepts it. + deferWindow = 1500 * time.Millisecond +) + +// SilentConnIsCutAtReadHeaderTimeout starts an engine with ReadHeaderTimeout 1 s and +// TCP_DEFER_ACCEPT off, so the deadline counts from the connect, opens a connection +// that sends nothing and another that sends one byte (the protocol is not decided +// yet) and then nothing, and asserts each is closed no sooner than 900 ms and no later +// than 1.4 s after its connect. +func SilentConnIsCutAtReadHeaderTimeout(t *testing.T, start Start) { + cfg := Config() + cfg.ReadHeaderTimeout = silentTimeout + cfg.DisableDeferAccept = true + addr := start(t, cfg, handler{}) + // The engine is left idle first, as a server is when a silent connection comes in + // for the first time: its workers wait in their longest idle wait, which is the + // state in which a sweep is furthest away from the connection. + time.Sleep(settle) + for _, c := range []struct{ name, first string }{{"silent", ""}, {"one-byte", "G"}} { + t.Run(c.name, func(t *testing.T) { + t.Parallel() + cutWithin(t, addr, c.first, silentTimeout-100*time.Millisecond, silentTimeout+silentSlack) + }) + } +} + +// SilentConnIsCutAfterDeferAccept is the same with TCP_DEFER_ACCEPT on, the default: +// a connection that sends nothing is accepted by the engine about a second after its +// SYN and is closed ReadHeaderTimeout after that, so no later than the deferral's +// window (1.5 s), the timeout and the slack after its connect. This is the bound the +// ReadHeaderTimeout documentation states. +func SilentConnIsCutAfterDeferAccept(t *testing.T, start Start) { + cfg := Config() + cfg.ReadHeaderTimeout = silentTimeout + addr := start(t, cfg, handler{}) + cutWithin(t, addr, "", silentTimeout-100*time.Millisecond, silentTimeout+deferWindow+silentSlack) +} + +// cutWithin connects to addr, sends first (nothing when it is empty), and asserts the +// server closes the connection within [lo, hi] of the connect. +func cutWithin(t *testing.T, addr, first string, lo, hi time.Duration) { + t.Helper() + t0 := time.Now() + c, err := net.DialTimeout("tcp", addr, 3*time.Second) + if err != nil { + t.Fatalf("dial %s: %v", addr, err) + } + defer func() { _ = c.Close() }() + if first != "" { + if _, err := c.Write([]byte(first)); err != nil { + t.Fatalf("write: %v", err) + } + } + d, closed := closedAfter(c, t0, hi+time.Second) + switch { + case !closed: + t.Errorf("still open %v after the connect, want it closed by %v", d.Round(time.Millisecond), hi) + case d < lo: + t.Errorf("closed after %v, want no sooner than %v", d.Round(time.Millisecond), lo) + case d > hi: + t.Errorf("closed after %v, want by %v (ReadHeaderTimeout %v)", d.Round(time.Millisecond), hi, silentTimeout) + } +} From 421a1645d0417b4c554adbafc635a612b56857ab Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 31 Oct 2026 18:25:35 +0100 Subject: [PATCH 2/5] iouring: the header deadline of a connection whose protocol is not decided is a kernel timer armed at the accept, not the sweep's (Refs #1189) A connection that sends nothing, or too little to tell HTTP/1 from HTTP/2, was closed by the sweep, which runs about every 0.83 s on an idle worker (32 iterations of 25 ms waits): a silent connection under ReadHeaderTimeout 1 s was closed at 1.7 to 1.9 s with TCP_DEFER_ACCEPT off (2.0 to 2.9 s with it on, a second of which is the kernel's deferral, which epoll has as well and which this change does not touch). A connection whose protocol is fixed, or already detected, has had an IORING_OP_TIMEOUT of its own since v1.4.11. handleAccept now arms the same timer at detectDeadline for such a connection (armHeaderTimerAt, no SQE more than before for an HTTP/1 connection: initProtocol would arm it at the same instant, finds it in flight and does not). handleHeaderTimer closes an undecided connection at its deadline, or arms again if the timer came early; initProtocol closes a connection whose budget is spent by the time it is detected, which the arm did when it was the first one. An HTTP/2 connection keeps one timer that fires once and does nothing. With no SQE to be had the sweep stays the fallback. Nothing changes per request, and the sweep is untouched, so idle sweeps cost what they did. Docs: ReadHeaderTimeout states the bound of a silent connection (the deferral plus the timeout plus about 30 ms) and ReadTimeout the sweep period the other timeouts are enforced at. The comments that said the sweep was 50 ms apart say 0.8 s. monotest.silentConn sets DisableDeferAccept: the deferral's second made a deadline that fires at once (a forward step, on the wall clock) look like one on time, so the 600 ms lower bound did not bite (the 1.05 s of the #1188 review). BenchmarkAcceptChurnAuto1189 is the instrument for the cost of the arm at the accept. --- config.go | 22 ++++ .../iouring/accept_churn_1189_linux_test.go | 116 ++++++++++++++++++ internal/engine/iouring/worker.go | 56 ++++++++- internal/mono/monotest/monotest.go | 6 + internal/mono/monotest/silent.go | 5 +- 5 files changed, 197 insertions(+), 8 deletions(-) create mode 100644 internal/engine/iouring/accept_churn_1189_linux_test.go diff --git a/config.go b/config.go index 11ac70efd..dc43b7d3a 100644 --- a/config.go +++ b/config.go @@ -272,6 +272,16 @@ type Config struct { // IdleTimeout (alone, as net/http's idle timer) from the last stream // activity, which those frames do not move. A response the client does not // take, from a client that keeps sending frames, is not bounded by ReadTimeout. + // + // The native engines look at ReadTimeout, IdleTimeout and WriteTimeout in a + // sweep of their connections, so one is enforced up to a sweep period after + // its time: 25 ms on epoll when ReadHeaderTimeout is set (the default), and on + // io_uring about 0.8 s (0.83 s measured on an idle worker; about 1 s with + // ReadHeaderTimeout off). With ReadHeaderTimeout off epoll has no such timer and + // looks much more rarely (a 1 s IdleTimeout closed an idle connection at 3 to 6 s + // in measurement). A timeout of a few seconds or less is approximate by that + // much on io_uring. ReadHeaderTimeout, which has a timer of its own on io_uring, + // is not. ReadTimeout time.Duration // ReadHeaderTimeout caps the read of just the request line + headers // (separately from the request body, which ReadTimeout covers). This @@ -292,6 +302,18 @@ type Config struct { // connection's first byte or about a second after its SYN, whichever // comes first; std counts from the handshake. // + // So the bound on a connection that sends nothing is its connect, plus + // about one second of deferral on epoll, io_uring and adaptive (measured + // 1.00 to 1.07 s), plus ReadHeaderTimeout, plus + // the lateness of the engine's check: about 30 ms on epoll (a 25 ms timer) + // and on io_uring (a timer of the connection's own), measured on docker + // arm64 with 2 CPUs. With DisableDeferAccept the deferral is gone: a silent + // connection is closed at ReadHeaderTimeout from its connect, and a + // ReadHeaderTimeout of one second closed it at 1.00 to 1.03 s. The io_uring + // engine used to leave the check of a connection that had sent nothing, or + // not enough to tell HTTP/1 from HTTP/2, to its sweep (0.8 s apart when idle), + // and closed a silent connection 0.5 to 0.9 s late. + // // On HTTP/2 (epoll, io_uring, adaptive) it also bounds a header block: one // that begins with a HEADERS frame without END_HEADERS and is not complete this // long after that frame ends the connection with GOAWAY(ENHANCE_YOUR_CALM). diff --git a/internal/engine/iouring/accept_churn_1189_linux_test.go b/internal/engine/iouring/accept_churn_1189_linux_test.go new file mode 100644 index 000000000..4446c08ca --- /dev/null +++ b/internal/engine/iouring/accept_churn_1189_linux_test.go @@ -0,0 +1,116 @@ +//go:build linux + +package iouring + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "sync" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" + "github.com/goceleris/celeris/internal/resource" +) + +// BenchmarkAcceptChurnAuto1189 is the cost of arming the header timer of a +// connection at its accept, which is a connection whose protocol is not decided +// yet (celeris#1189): Protocol Auto, as the default is, with ReadHeaderTimeout +// set (the default is 10 s) or off (the control: the change is not reached). One +// op is one connection, on 8 client goroutines and one worker. "request" sends one +// request with Connection: close and reads the response to the end: the timer +// armed at the accept is the one the request would have armed at detection, so +// this arm must not move. "bare" connects and closes without a byte: it pays the +// arm and the cancel the request arm did not, so it is the worst case for the +// change. The kernel's connect and close dominate each op, so read the figures +// against an A/A twin of the same binary (RULE 74). +func BenchmarkAcceptChurnAuto1189(b *testing.B) { + for _, rht := range []time.Duration{10 * time.Second, -1} { + for _, arm := range []string{"request", "bare"} { + b.Run(fmt.Sprintf("%s/rht=%v", arm, rht), func(b *testing.B) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + b.Fatalf("pick port: %v", err) + } + addr := ln.Addr().String() + _ = ln.Close() + e, err := New(resource.Config{ + Addr: addr, + Protocol: engine.Auto, + Resources: resource.Resources{Workers: 2}, + ReadHeaderTimeout: rht, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }, newCarry543Handler()) + if err != nil { + b.Skipf("iouring engine unavailable: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { _ = e.Listen(ctx); close(done) }() + defer func() { cancel(); <-done }() + for dl := time.Now().Add(15 * time.Second); (e.Addr() == nil || e.NumWorkers() == 0) && time.Now().Before(dl); { + time.Sleep(2 * time.Millisecond) + } + if e.Addr() == nil { + b.Fatal("engine did not bind") + } + req := []byte("GET /bench HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n") + one := func() error { + c, err := net.Dial("tcp", addr) + if err != nil { + return err + } + defer func() { _ = c.Close() }() + if arm == "bare" { + return nil + } + if _, err := c.Write(req); err != nil { + return err + } + _, err = io.Copy(io.Discard, c) + return err + } + // Warm: the first connections after an idle ring wait for the worker's + // idle wake. + for range 2000 { + if err := one(); err != nil { + b.Fatalf("warm: %v", err) + } + } + base := e.Metrics().CloseCount + b.ReportAllocs() + b.ResetTimer() + var wg sync.WaitGroup + next := make(chan struct{}, 256) + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for range next { + if err := one(); err != nil { + b.Errorf("conn: %v", err) + return + } + } + }() + } + for range b.N { + next <- struct{}{} + } + close(next) + wg.Wait() + dl := time.Now().Add(60 * time.Second) + for e.Metrics().CloseCount-base < uint64(b.N) { + if time.Now().After(dl) { + b.Fatalf("the engine closed %d of %d connections", e.Metrics().CloseCount-base, b.N) + } + time.Sleep(50 * time.Microsecond) + } + b.StopTimer() + }) + } + } +} diff --git a/internal/engine/iouring/worker.go b/internal/engine/iouring/worker.go index c1f0fdbe0..b9b772ab2 100644 --- a/internal/engine/iouring/worker.go +++ b/internal/engine/iouring/worker.go @@ -2041,7 +2041,8 @@ func (w *Worker) run(ctx context.Context) { // legit traffic (observability+static_swagger_proxy) the // 0x3FF gate is too coarse and the walker times out before // the sweep notices. Tightening to 0x1F caps the worst-case - // close latency on a dropped arm to ~50ms. + // close latency on a dropped arm to 32 iterations: about 0.8 s + // on an idle worker (25 ms waits), less on a busy one. w.tickCounter++ gate := uint32(0x3FF) if w.detachedCount > 0 || w.cfg.ReadHeaderTimeout > 0 { @@ -2453,6 +2454,21 @@ func (w *Worker) handleHeaderTimer(fd int) { if cs.closing { return } + // A conn whose protocol is not decided yet has no H1 state: its deadline is + // detectDeadline, armed at accept (handleAccept), and cs.detected and the + // deadline are the worker's own, so no lock is needed (celeris#974). Past it + // the conn is closed as the sweep closes it; a timer that came early is armed + // again. Once the protocol is decided the deadline is the H1 state's, below. + if !cs.detected { + if dd := cs.detectDeadline; dd > 0 { + if mono.Now() >= dd { + w.closeConn(fd) + } else { + w.armHeaderTimerAt(cs, dd) + } + } + return + } // Snapshot under detachMu — same TOCTOU as checkTimeouts (celeris#548) and // the same TryLock for the same reason (celeris#593): this runs on the // LockOSThread'd worker, and runAsyncHandler holds detachMu for the whole @@ -2461,9 +2477,9 @@ func (w *Worker) handleHeaderTimer(fd int) { if !ok { // detachMu held ⇒ the conn is inside its handler, not waiting for a // request line, so the header deadline cannot be live. Dropping this - // timer costs nothing: checkTimeouts re-reads HeaderDeadlineNs on its - // ~50ms slowloris cadence and is the documented fallback for a timer - // that failed to arm. Re-arming here instead would spin — the spec's + // timer costs nothing: checkTimeouts re-reads HeaderDeadlineNs at its + // sweep (about every 0.8 s on an idle worker, sweepFloor on a busy + // one) and is the documented fallback for a timer that failed to arm. Re-arming here instead would spin — the spec's // deadline has already passed, so the fresh timer would fire at once. return } @@ -2771,6 +2787,16 @@ func (w *Worker) onAcceptedFD(ctx context.Context, newFD int, now int64, isFixed // checkTimeouts instead of living on the activity stamp each of its // bytes refreshes (celeris#974). cs.detectDeadline = now + int64(w.cfg.ReadHeaderTimeout) + // The kernel holds the deadline, as it does for a conn whose protocol is + // fixed (initProtocol): a sweep comes about every 0.8 s on an idle + // worker, so a conn that stays silent was closed up to that late + // (celeris#1189). The timer is the one initProtocol would arm at + // detection, for the same instant (ArmHeaderDeadlineAt(detectDeadline)), + // so an HTTP/1 conn costs no SQE more than before; an HTTP/2 conn costs + // one, which fires once and does nothing (handleHeaderTimer). With no SQE + // to be had the arm is skipped and the sweep is the fallback, as for a + // timer the ring dropped. + w.armHeaderTimerAt(cs, cs.detectDeadline) } if !w.prepareRecv(cs, cs.buf) { cs.needsRecv = true @@ -3830,6 +3856,13 @@ func (w *Worker) initProtocol(cs *connState) { // only stamped HeaderDeadlineNs, not the SQE. w.armHeaderTimer(cs) } + // The timer of the budget was armed at the accept (handleAccept), so the + // arms above found it in flight and did not look at the clock. Look, so + // that a budget that is spent by now ends the conn here, the last thing + // done to it, as the arm did when it was the first (celeris#974). + if dd := cs.detectDeadline; dd > 0 && cs.headerTimerArmed && mono.Now() >= dd { + w.closeConn(cs.fd) + } case engine.H2C: cs.h2State = conn.NewH2State(w.handler, w.h2cfg, cs.writeFn, w.wakeFD) cs.h2State.SetRemoteAddr(cs.remoteAddr) @@ -7819,8 +7852,19 @@ var testHookCheckTimeouts atomic.Pointer[func(*Worker)] // 2 s on a worker with no listener; the gate alone was 1.6 s with a detached // conn and about 102 s with none). While the worker waits without a timeout for a // SEND's completion (waitUntimed) no iteration runs; the standing timer -// (armSweepTick) ends that wait within sweepCeiling. This -// replaces the timer wheel: instead of allocating entries and updating maps +// (armSweepTick) ends that wait within sweepCeiling. +// +// What rides on the sweep, and so is enforced up to one sweep period late (a +// period of about 0.8 s on an idle worker: measured 0.83 s, celeris#1189): the +// idle, read and write timeouts, the WebSocket idle deadline, the HTTP/2 +// deadlines, and the closing drain. The HTTP/1 header deadline and the budget of +// a conn whose protocol is not decided yet (detectDeadline) have a timer of +// their own in the kernel (armHeaderTimer, armed at the accept for the latter) +// and are met within milliseconds; the sweep is their fallback for a timer the +// ring could not take. epoll's sweep is a 25 ms timerfd, so it is late by at most +// that when ReadHeaderTimeout is set. +// +// This replaces the timer wheel: instead of allocating entries and updating maps // on every recv/send, we store a single lastActivity timestamp on the // connState and scan here. // diff --git a/internal/mono/monotest/monotest.go b/internal/mono/monotest/monotest.go index 1e934824a..3f6568a22 100644 --- a/internal/mono/monotest/monotest.go +++ b/internal/mono/monotest/monotest.go @@ -228,6 +228,12 @@ func slowHeader(t *testing.T, start Start) { func silentConn(t *testing.T, start Start) { cfg := Config() cfg.ReadHeaderTimeout = time.Second + // Without the deferral the budget counts from the connect. With it the engine + // accepts the connection about a second after its SYN (TCP_DEFER_ACCEPT), and + // that second would make a deadline that fires at once, as a forward step makes + // a wall-clock one, indistinguishable from one on time: the lower bound below + // would not bite (celeris#1189: a forward step closed the connection at 1.05 s). + cfg.DisableDeferAccept = true addr := start(t, cfg, handler{}) c := dial(t, addr) expect(t, "silent conn (ReadHeaderTimeout 1 s)", c, time.Now(), 600*time.Millisecond, 7*time.Second) diff --git a/internal/mono/monotest/silent.go b/internal/mono/monotest/silent.go index aaf60c2e7..b3dbefe9f 100644 --- a/internal/mono/monotest/silent.go +++ b/internal/mono/monotest/silent.go @@ -29,8 +29,9 @@ const ( // idle backoff (1 ms, then 5 ms, then 25 ms waits) to have run its course. settle = 700 * time.Millisecond - // deferWindow is the longest a silent connection waits in the kernel, with - // TCP_DEFER_ACCEPT on, before the engine accepts it. + // deferWindow is what a silent connection is allowed to wait in the kernel, with + // TCP_DEFER_ACCEPT on, before the engine accepts it: the 1.5 s that + // deferlinger.DefaultLinger budgets for it (1.07 s was the most measured). deferWindow = 1500 * time.Millisecond ) From 5e6b51cc635846a423918db32ba75a589fc1e0c6 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 31 Oct 2026 20:07:27 +0100 Subject: [PATCH 3/5] iouring: pin that a connection closed by the header-timer arm at the accept is given no recv (Refs #1189) The arm at the accept closes the connection when its budget is already spent: the budget is taken from the CQE batch's clock, which is older than ReadHeaderTimeout when an OnConnect hook or a slow inline handler earlier in the batch ran long. onAcceptedFD goes on and arms a recv on the closed connection, which nothing accounts for (the fd-lifetime and cancel-then-release rules, celeris#685 and celeris#256). The test is the review's probe, made fast: ReadHeaderTimeout 200 ms, OnConnect sleeps 300 ms for the first connection, and after one second no connState of pendingRelease may have an op in flight and no closedOps entry. RED on 421a164: 5 of 5 runs, every look at the released connState finds the orphaned recv. --- .../close_at_accept_1189_linux_test.go | 112 ++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 internal/engine/iouring/close_at_accept_1189_linux_test.go diff --git a/internal/engine/iouring/close_at_accept_1189_linux_test.go b/internal/engine/iouring/close_at_accept_1189_linux_test.go new file mode 100644 index 000000000..720d03386 --- /dev/null +++ b/internal/engine/iouring/close_at_accept_1189_linux_test.go @@ -0,0 +1,112 @@ +//go:build linux + +package iouring + +import ( + "io" + "log/slog" + "net" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" + "github.com/goceleris/celeris/internal/resource" +) + +// A connection whose header timer finds its budget already spent when it is armed +// at the accept is closed there, and nothing may then be armed on it +// (celeris#1189, review of #1193). The budget is taken from the CQE batch's clock, +// so it is spent at the accept when the batch ran for longer than ReadHeaderTimeout +// before it reached the accept: an OnConnect hook, or a slow inline handler of an +// earlier CQE of the batch. Here OnConnect takes 300 ms against a ReadHeaderTimeout +// of 200 ms. A recv armed on the closed connection is a kernel op on a released +// descriptor number and on a connState that is queued for release (celeris#685, +// celeris#256), and nothing accounts for it: the connState is held by the 5 s +// pendingRelease backstop, which then releases it with a "kernel ops unaccounted" +// warning. The check is the sweep's own, so it does not wait for the backstop: no +// connState of pendingRelease may have an op in flight and no closedOps entry that +// will account for its completion. +func TestConnClosedByTheArmAtAcceptGetsNoRecv1189(t *testing.T) { + var ( + mu sync.Mutex + addr string + looked int + orphans int + ) + hook := func(w *Worker) { + mu.Lock() + defer mu.Unlock() + // The hook is the package's, not the engine's: workers of an earlier test may + // still be winding down. + if addr == "" || w.cfg.Addr != addr { + return + } + for i := range w.pendingRelease { + cs := w.pendingRelease[i].cs + looked++ + if cs.kernelInflight > 0 && w.closedOps[encodeConnOpKey(cs.fd, cs.generation)] == nil { + orphans++ + } + } + } + testHookCheckTimeouts.Store(&hook) + t.Cleanup(func() { testHookCheckTimeouts.Store(nil) }) + logs := &syncBuf806{} + var first atomic.Bool + r := startEngine806(t, loadHandler806{}, func(c *resource.Config) { + c.Protocol = engine.Auto + c.ReadHeaderTimeout = 200 * time.Millisecond + c.DisableDeferAccept = true + c.Logger = slog.New(slog.NewTextHandler(io.Writer(logs), &slog.HandlerOptions{Level: slog.LevelWarn})) + c.OnConnect = func(string) { + if first.CompareAndSwap(false, true) { + time.Sleep(300 * time.Millisecond) + } + } + }) + mu.Lock() + addr = r.addr + mu.Unlock() + + c, err := net.DialTimeout("tcp", r.addr, 3*time.Second) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + _ = c.SetReadDeadline(time.Now().Add(3 * time.Second)) + if _, err := c.Read(make([]byte, 1)); err != io.EOF { + t.Fatalf("the connection whose budget was spent at the accept: read = %v, want it closed (EOF)", err) + } + + // A healthy connection is served. + c2, err := net.DialTimeout("tcp", r.addr, 3*time.Second) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c2.Close() }() + _ = c2.SetDeadline(time.Now().Add(3 * time.Second)) + if _, err := io.WriteString(c2, "GET / HTTP/1.1\r\nHost: x\r\n\r\n"); err != nil { + t.Fatal(err) + } + buf := make([]byte, 12) + if _, err := io.ReadFull(c2, buf); err != nil || !strings.HasPrefix(string(buf), "HTTP/1.1 200") { + t.Fatalf("the healthy connection was not served: %q, %v", buf, err) + } + + // The sweeps come within about 70, 250 and 430 ms of the activity of an idle + // worker (its idle backoff starts again), so a second is several looks; the 5 s + // backstop is not waited for. + time.Sleep(time.Second) + mu.Lock() + defer mu.Unlock() + t.Logf("sweeps looked at a connState of pendingRelease %d times", looked) + if orphans > 0 { + t.Errorf("a connState queued for release had a kernel op in flight with no closedOps entry in %d of %d looks (a recv armed on a closed connection)", orphans, looked) + } + if l := logs.String(); strings.Contains(l, "unaccounted") { + t.Errorf("warning logged: %s", l) + } +} From 4d4ccd9df3ad959f1ec292077755df075640942e Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 31 Oct 2026 20:08:05 +0100 Subject: [PATCH 4/5] iouring: a connection closed by the header-timer arm at the accept is given no recv; tighten the bound test, correct the sweep doc (Refs #1189) onAcceptedFD armed a recv on a connection the new header-timer arm had just closed (the batch's clock older than ReadHeaderTimeout: a slow OnConnect or inline handler earlier in the batch). The recv named a released descriptor number and a connState queued for release and nothing accounted for it (celeris#685, celeris#256): the 5 s backstop released it with the "kernel ops unaccounted" warning, and in the worst case a later accept reusing the number would have its first bytes eaten. The accept now returns when the arm closed the connection, as handleRecv does after its own arm. The other outcomes of the arm leave nothing behind: a full SQ or no SQE for the timer returns unarmed with the connection untouched (the sweep is the fallback), and the dirty-list path is only reached by a connection that is still open. Tests: monotest's slack is 150 ms (six times the larger of the engines' measured lateness) and its doc says that the white-box test is the one that catches the sweep regression, since the black-box bound does not tell the sweep from the timer on every run (the sweep's phase decides). The ReadTimeout and ReadHeaderTimeout docs say that the HTTP/2 header-block bound is checked by the sweep and is late by up to a sweep period, and no longer carry the history of the fix. --- config.go | 13 +++++++------ internal/engine/iouring/worker.go | 9 +++++++++ internal/mono/monotest/silent.go | 15 +++++++++------ 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/config.go b/config.go index dc43b7d3a..a2f961c1a 100644 --- a/config.go +++ b/config.go @@ -280,8 +280,10 @@ type Config struct { // ReadHeaderTimeout off). With ReadHeaderTimeout off epoll has no such timer and // looks much more rarely (a 1 s IdleTimeout closed an idle connection at 3 to 6 s // in measurement). A timeout of a few seconds or less is approximate by that - // much on io_uring. ReadHeaderTimeout, which has a timer of its own on io_uring, - // is not. + // much on io_uring. The HTTP/1 header deadline, and the budget of a connection + // whose protocol is not decided yet, have a timer of their own on io_uring and + // are not; the HTTP/2 header-block bound of ReadHeaderTimeout is checked by the + // sweep and is late by up to a sweep period. ReadTimeout time.Duration // ReadHeaderTimeout caps the read of just the request line + headers // (separately from the request body, which ReadTimeout covers). This @@ -309,10 +311,9 @@ type Config struct { // and on io_uring (a timer of the connection's own), measured on docker // arm64 with 2 CPUs. With DisableDeferAccept the deferral is gone: a silent // connection is closed at ReadHeaderTimeout from its connect, and a - // ReadHeaderTimeout of one second closed it at 1.00 to 1.03 s. The io_uring - // engine used to leave the check of a connection that had sent nothing, or - // not enough to tell HTTP/1 from HTTP/2, to its sweep (0.8 s apart when idle), - // and closed a silent connection 0.5 to 0.9 s late. + // ReadHeaderTimeout of one second closed it at 1.00 to 1.03 s. The HTTP/2 + // header-block bound below is checked by the engine's sweep, not by a timer, + // and is late by up to a sweep period (see ReadTimeout). // // On HTTP/2 (epoll, io_uring, adaptive) it also bounds a header block: one // that begins with a HEADERS frame without END_HEADERS and is not complete this diff --git a/internal/engine/iouring/worker.go b/internal/engine/iouring/worker.go index b9b772ab2..8abd0381b 100644 --- a/internal/engine/iouring/worker.go +++ b/internal/engine/iouring/worker.go @@ -2797,6 +2797,15 @@ func (w *Worker) onAcceptedFD(ctx context.Context, newFD int, now int64, isFixed // to be had the arm is skipped and the sweep is the fallback, as for a // timer the ring dropped. w.armHeaderTimerAt(cs, cs.detectDeadline) + // The arm closes the conn when the budget is spent already: detectDeadline + // comes from the batch's clock, which is older than ReadHeaderTimeout when an + // OnConnect hook, or a slow inline handler earlier in the batch, ran long. A + // closed conn is given nothing: a recv on it would name a descriptor number + // that is released, on a connState queued for release, and nothing would + // account for it (celeris#685, celeris#256). As handleRecv does after its arm. + if w.conns[newFD] != cs || cs.closing { + return + } } if !w.prepareRecv(cs, cs.buf) { cs.needsRecv = true diff --git a/internal/mono/monotest/silent.go b/internal/mono/monotest/silent.go index b3dbefe9f..ad58e2fd4 100644 --- a/internal/mono/monotest/silent.go +++ b/internal/mono/monotest/silent.go @@ -19,11 +19,14 @@ const ( // silentSlack is what a silent connection may exceed its deadline by once it // has been accepted. epoll sweeps every 25 ms and io_uring arms a timer of its - // own, so a connection is closed within about 30 ms of its deadline (measured, - // docker arm64 with 2 CPUs, 20 runs per engine); 400 ms is more than ten times - // that, and is not tuned to the machine. The io_uring sweep that enforced the - // deadline before celeris#1189 came 0.54 to 0.86 s late (40 runs). - silentSlack = 400 * time.Millisecond + // own, so a connection is closed within 8 to 25 ms of its deadline (measured, + // docker arm64 with 2 CPUs, 20 runs per engine); 150 ms is six times the larger + // and is not tuned to the machine. It does not tell the io_uring sweep from the + // timer on every run: the sweep that enforced the deadline before celeris#1189 + // came 0.1 to 0.9 s late, depending on where its phase fell after the accept. + // The regression test for that is the io_uring package's + // TestUndecidedConnHasItsHeaderTimerArmed1189, which does not depend on timing. + silentSlack = 150 * time.Millisecond // settle is how long the engine idles before the connection, for its workers' // idle backoff (1 ms, then 5 ms, then 25 ms waits) to have run its course. @@ -39,7 +42,7 @@ const ( // TCP_DEFER_ACCEPT off, so the deadline counts from the connect, opens a connection // that sends nothing and another that sends one byte (the protocol is not decided // yet) and then nothing, and asserts each is closed no sooner than 900 ms and no later -// than 1.4 s after its connect. +// than 1.15 s after its connect. func SilentConnIsCutAtReadHeaderTimeout(t *testing.T, start Start) { cfg := Config() cfg.ReadHeaderTimeout = silentTimeout From 311274a1e12e7179eaec44448174da016d40fa77 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sat, 31 Oct 2026 23:04:16 +0100 Subject: [PATCH 5/5] iouring: the close-at-accept test proves the scenario happened (Refs #1189) TestConnClosedByTheArmAtAcceptGetsNoRecv1189 passed when nothing was closed at the accept (an OnConnect that does not outlast the budget, or a close by a timer or the sweep), because it then had nothing to check. It now records when the slow OnConnect returns and when OnDisconnect runs, and requires the close within 50 ms of the return: the arm's close is 6 to 75 microseconds after it (also under -race), a timer's would be ReadHeaderTimeout (200 ms) after the accept, the sweep's 100 ms or more. RED on 421a164 5 of 5 (the recv left on the closed connection), GREEN on the fix 10 of 10 and under -race 10 of 10. A mutant with an OnConnect of 10 ms, which does not outlast the budget, FAILS 3 of 3 (closed 190 ms after the return) where it passed vacuously before. --- .../close_at_accept_1189_linux_test.go | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/internal/engine/iouring/close_at_accept_1189_linux_test.go b/internal/engine/iouring/close_at_accept_1189_linux_test.go index 720d03386..8a9e373e1 100644 --- a/internal/engine/iouring/close_at_accept_1189_linux_test.go +++ b/internal/engine/iouring/close_at_accept_1189_linux_test.go @@ -55,7 +55,11 @@ func TestConnClosedByTheArmAtAcceptGetsNoRecv1189(t *testing.T) { testHookCheckTimeouts.Store(&hook) t.Cleanup(func() { testHookCheckTimeouts.Store(nil) }) logs := &syncBuf806{} - var first atomic.Bool + var ( + first atomic.Bool + connReturned atomic.Pointer[time.Time] // when the slow OnConnect returned + closedAt atomic.Pointer[time.Time] // when the first connection's OnDisconnect ran + ) r := startEngine806(t, loadHandler806{}, func(c *resource.Config) { c.Protocol = engine.Auto c.ReadHeaderTimeout = 200 * time.Millisecond @@ -64,8 +68,14 @@ func TestConnClosedByTheArmAtAcceptGetsNoRecv1189(t *testing.T) { c.OnConnect = func(string) { if first.CompareAndSwap(false, true) { time.Sleep(300 * time.Millisecond) + now := time.Now() + connReturned.Store(&now) } } + c.OnDisconnect = func(string) { + now := time.Now() + closedAt.CompareAndSwap(nil, &now) + } }) mu.Lock() addr = r.addr @@ -81,6 +91,22 @@ func TestConnClosedByTheArmAtAcceptGetsNoRecv1189(t *testing.T) { t.Fatalf("the connection whose budget was spent at the accept: read = %v, want it closed (EOF)", err) } + // The scenario happened: the arm closed the connection at the accept, as soon as + // OnConnect returned, not at a timer or a sweep after it. Without this the test + // passes when the OnConnect did not outlast the budget (nothing is closed at the + // accept), or when a timer or the sweep closed the connection, which have nothing + // to leave behind. The close is within microseconds of the return when it is the + // arm's (measured 6 to 75 microseconds, under -race too); a timer + // would be ReadHeaderTimeout (200 ms) after the accept, which is 200 ms after the return, and the sweep's 100 ms or more. + ret, closed := connReturned.Load(), closedAt.Load() + if ret == nil || closed == nil { + t.Fatalf("OnConnect returned: %v, OnDisconnect ran: %v: the scenario did not occur", ret != nil, closed != nil) + } + if gap := closed.Sub(*ret); gap < 0 || gap > 50*time.Millisecond { + t.Fatalf("the connection was closed %v after its slow OnConnect returned, want within 50ms: it was not the arm at the accept that closed it", gap) + } + t.Logf("closed %v after OnConnect returned", closed.Sub(*ret)) + // A healthy connection is served. c2, err := net.DialTimeout("tcp", r.addr, 3*time.Second) if err != nil {