From 99354c66e329c28f0262a0b461659db52351c452 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sun, 1 Nov 2026 09:09:41 +0100 Subject: [PATCH 1/3] tests: a healthy long batch is not a wedge; a dead loop leaves no stamp; the many-downloads cell asserts the condition (red on main) Refs #1170. Coverage root shard c, run 41246492268: TestH2GoTransportManyParallelDownloadsAreNotCut1170/adaptive had 7 of 8 downloads reset (loop-blocked 4, rate 3). The loop-stuck fallback judges a loop stuck from the wall-clock age of its current batch (max(WriteTimeout/2, 250 ms)); a healthy loop of an instrumented or overloaded host has batches of 0.4 to 1.9 s (one h2 connection's write-queue drain), measured at 7 of 72 cells with -race, -covermode=atomic and two busy loops on two CPUs. Unit tests, on mono.Manual (exact, no sleep): - loopStuckFor has a 2 s floor (1 s -> 2 s, 5 s -> 2.5 s, 60 s -> 30 s unchanged); - a batch of 0.8, 1.5 and 1.9 s at WriteTimeout 1 s is not a wedge, 2 s is; at 5 s and at the 60 s default the edge is 2.5 s and 30 s; a waiter that has not waited a WriteTimeout is never cut by it; - a loop that blocks mid-wait is cut within [threshold, threshold + WriteTimeout] (the positive control, with the bound stated); - a loop that ends without returning (runtime.Goexit, celeris#848) leaves its LoopWatch stamp set: every waiter of the engine would be cut as loop-blocked for ever (epoll and io_uring, in the 848 exit tests). Cells: the many-downloads cell runs the engines one at a time and asserts that no wait was reset as loop-blocked before it asserts the downloads; the wedge cell asserts the bound as a formula; the threshold pair blocks a loop for 2.5 s (over) and 0.1 s (under). Red on this commit's parent code: the floor, the healthy-batch cases, the stale stamp. Green: the bound, the 60 s edge, the cells. --- .../engine/epoll/loop_exit_848_linux_test.go | 10 ++ .../iouring/worker_exit_848_linux_test.go | 10 ++ .../h2/stream/loop_stuck_floor_1170_test.go | 107 ++++++++++++++++++ .../h2/stream/write_wait_1170_test.go | 14 +-- .../h2_pool_wait_cells_1170_linux_test.go | 24 ++-- .../h2_pool_wait_rate_1170_linux_test.go | 16 ++- 6 files changed, 160 insertions(+), 21 deletions(-) create mode 100644 internal/protocol/h2/stream/loop_stuck_floor_1170_test.go diff --git a/internal/engine/epoll/loop_exit_848_linux_test.go b/internal/engine/epoll/loop_exit_848_linux_test.go index bedbd8e74..1531b44b4 100644 --- a/internal/engine/epoll/loop_exit_848_linux_test.go +++ b/internal/engine/epoll/loop_exit_848_linux_test.go @@ -19,6 +19,7 @@ import ( "github.com/goceleris/celeris/internal/ctxkit" celerisengine "github.com/goceleris/celeris/internal/engine" + "github.com/goceleris/celeris/internal/mono" "github.com/goceleris/celeris/internal/protocol/h2/stream" "github.com/goceleris/celeris/internal/resource" ) @@ -176,6 +177,15 @@ func TestSyncHandlerGoexitEndsOneLoopLoudly848(t *testing.T) { for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl) && e.abnormalLoopExits.Load() == 0; time.Sleep(5 * time.Millisecond) { } time.Sleep(200 * time.Millisecond) + // The loop died inside a batch, with its stamp set (celeris#1170): a stamp that + // is left behind ages for ever, and every waiting pool handler of the engine would + // be cut as "loop-blocked" from then on. The exit clears it. + e.mu.Lock() + watch := e.loopWatch + e.mu.Unlock() + if watch.BlockedFor(mono.Now()+int64(time.Hour), 1) { + t.Error("the loop that ended without returning left its LoopWatch stamp set: it reads as stuck for ever") + } // Only one loop died: a conn on another loop is still served. var otherAnswered, others int for k, c := range conns { diff --git a/internal/engine/iouring/worker_exit_848_linux_test.go b/internal/engine/iouring/worker_exit_848_linux_test.go index 64b8364a1..b54f80b47 100644 --- a/internal/engine/iouring/worker_exit_848_linux_test.go +++ b/internal/engine/iouring/worker_exit_848_linux_test.go @@ -19,6 +19,7 @@ import ( "github.com/goceleris/celeris/internal/ctxkit" "github.com/goceleris/celeris/internal/engine" + "github.com/goceleris/celeris/internal/mono" "github.com/goceleris/celeris/internal/protocol/h2/stream" "github.com/goceleris/celeris/internal/resource" ) @@ -202,6 +203,15 @@ func TestSyncHandlerGoexitEndsOneWorkerLoudly848(t *testing.T) { for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl) && e.abnormalWorkerExits.Load() == 0; time.Sleep(5 * time.Millisecond) { } time.Sleep(200 * time.Millisecond) + // The worker died inside a batch, with its stamp set (celeris#1170): a stamp that + // is left behind ages for ever, and every waiting pool handler of the engine would + // be cut as "loop-blocked" from then on. The exit clears it. + e.mu.Lock() + watch := e.loopWatch + e.mu.Unlock() + if watch.BlockedFor(mono.Now()+int64(time.Hour), 1) { + t.Error("the worker that ended without returning left its LoopWatch stamp set: it reads as stuck for ever") + } // Only one worker died: a conn on another worker is still served. var otherAnswered, others int for k, c := range conns { diff --git a/internal/protocol/h2/stream/loop_stuck_floor_1170_test.go b/internal/protocol/h2/stream/loop_stuck_floor_1170_test.go new file mode 100644 index 000000000..b5b5a15fc --- /dev/null +++ b/internal/protocol/h2/stream/loop_stuck_floor_1170_test.go @@ -0,0 +1,107 @@ +package stream + +import ( + "testing" + "time" + + "github.com/goceleris/celeris/internal/mono" +) + +// celeris#1170, the loop-stuck fallback (LoopWatch): a loop is taken as stuck when it +// has been in ONE batch for max(WriteTimeout/2, floor). With the floor at 250 ms a +// healthy loop of a host that is overloaded or runs -race and a coverage profile (a +// batch of 0.4 to 1.9 s measured, mostly one h2 connection's write-queue drain) cut +// healthy waiters of 8 parallel downloads as "loop-blocked" (Coverage root shard c, +// run 41246492268; 7 of 72 cells with two busy loops on two CPUs). A blocked loop +// (a synchronous handler waiting for a lock a waiting pool handler holds) is never a +// batch that short, so the floor is 2 s. +// +// The clock is mono.Manual: the verdicts are exact, with no sleep and no margin for +// the speed of the runner. + +func TestLoopStuckThresholdHasAFloor1170(t *testing.T) { + for _, tc := range []struct{ wt, want time.Duration }{ + {100 * time.Millisecond, 2 * time.Second}, + {time.Second, 2 * time.Second}, + {4 * time.Second, 2 * time.Second}, + {5 * time.Second, 2500 * time.Millisecond}, + {10 * time.Second, 5 * time.Second}, + {60 * time.Second, 30 * time.Second}, // the default WriteTimeout: unchanged + } { + if got := time.Duration(loopStuckFor(int64(tc.wt))); got != tc.want { + t.Errorf("loopStuckFor(%v) = %v, want %v", tc.wt, got, tc.want) + } + } +} + +// A long batch that is not yet the threshold old is not a wedge; at the threshold it is. +func TestHealthyLongBatchIsNotAWedge1170(t *testing.T) { + for _, tc := range []struct { + name string + wt time.Duration + age time.Duration // how long the loop has been in its batch at the judgement + waited time.Duration // how long the waiter has waited + cut bool + }{ + {"WT 1s, batch of 0.8 s (healthy, instrumented)", time.Second, 800 * time.Millisecond, 20 * time.Second, false}, + {"WT 1s, batch of 1.5 s", time.Second, 1500 * time.Millisecond, 20 * time.Second, false}, + {"WT 1s, batch of 1.9 s", time.Second, 1900 * time.Millisecond, 20 * time.Second, false}, + {"WT 1s, batch of 2 s", time.Second, 2 * time.Second, 20 * time.Second, true}, + {"WT 1s, batch of 3 s, waited under a WriteTimeout", time.Second, 3 * time.Second, 500 * time.Millisecond, false}, + {"WT 5s, batch of 2.4 s", 5 * time.Second, 2400 * time.Millisecond, 20 * time.Second, false}, + {"WT 5s, batch of 2.6 s", 5 * time.Second, 2600 * time.Millisecond, 20 * time.Second, true}, + {"WT 60s (default), batch of 29 s", time.Minute, 29 * time.Second, 70 * time.Second, false}, + {"WT 60s (default), batch of 31 s", time.Minute, 31 * time.Second, 70 * time.Second, true}, + } { + t.Run(tc.name, func(t *testing.T) { + clk := mono.NewManual() + t.Cleanup(mono.SetManual(clk)) + m, s := waitRig1170(t, tc.wt, 0) + lw := NewLoopWatch() + lw.Slot().Enter(mono.Now() - int64(tc.age)) + w := SendWait{Loops: lw, start: mono.Now() - int64(tc.waited)} + v, _ := m.judgeWait(s, s.ID, &w, 4<<20) + if cut := v == waitExpired && w.BlockedLoop(); cut != tc.cut { + t.Fatalf("verdict %v, loop-blocked %v: want cut=%v", v, w.BlockedLoop(), tc.cut) + } + }) + } +} + +// The bound of a wedge: a loop that blocks while a waiter is waiting is cut at the first +// judgement (one per WriteTimeout window) at which its batch is max(WT/2, 2 s) old and +// the wait has lasted a WriteTimeout, so at most that plus one WriteTimeout after the +// loop blocked. Positive control: the fallback still cuts, within the stated bound. +func TestBlockedLoopIsCutWithinTheBound1170(t *testing.T) { + for _, wt := range []time.Duration{time.Second, 5 * time.Second, 60 * time.Second} { + t.Run(wt.String(), func(t *testing.T) { + clk := mono.NewManual() + t.Cleanup(mono.SetManual(clk)) + m, s := waitRig1170(t, wt, 0) + lw := NewLoopWatch() + slot := lw.Slot() + start := mono.Now() // the waiter blocks + clk.Advance(wt * 7 / 10) + slot.Enter(mono.Now()) // the loop blocks 0.7 WT into the wait and never returns + w := SendWait{Loops: lw, start: start} + var cutAt time.Duration + for k := 1; k <= 400 && cutAt == 0; k++ { // the waiter's timer: one judgement per window + clk.Advance(wt) + m.dropWait(s.ID) // the rate rule is not under test: the record is made afresh + if v, _ := m.judgeWait(s, s.ID, &w, 4<<20); v == waitExpired { + if !w.BlockedLoop() { + t.Fatalf("cut by the rate rule at judgement %d", k) + } + cutAt = time.Duration(mono.Now() - slot.since.Load()) // how long the loop had been blocked + } + } + bound := time.Duration(loopStuckFor(int64(wt))) + wt // threshold, then one window to the next judgement + if cutAt == 0 { + t.Fatalf("a blocked loop was never cut") + } + if cutAt < time.Duration(loopStuckFor(int64(wt))) || cutAt > bound { + t.Fatalf("cut %v after the loop blocked, want within [%v, %v]", cutAt, time.Duration(loopStuckFor(int64(wt))), bound) + } + }) + } +} diff --git a/internal/protocol/h2/stream/write_wait_1170_test.go b/internal/protocol/h2/stream/write_wait_1170_test.go index f6d5740ee..8c9011d9e 100644 --- a/internal/protocol/h2/stream/write_wait_1170_test.go +++ b/internal/protocol/h2/stream/write_wait_1170_test.go @@ -516,7 +516,7 @@ func TestIdleAllowanceIsFixedWhenTheRunBegins1170(t *testing.T) { } // While an event loop is stuck in one batch for half a WriteTimeout (and at least -// 250 ms) a waiter that has waited a whole WriteTimeout is reset at its next +// 2 s) a waiter that has waited a whole WriteTimeout is reset at its next // judgement, whatever its seed and credit say; just under the threshold it is not. func TestWaiterLoopBlockedFallback1170(t *testing.T) { for _, tc := range []struct { @@ -524,8 +524,8 @@ func TestWaiterLoopBlockedFallback1170(t *testing.T) { stuck time.Duration // how long the loop has been in its batch when the waiter first blocks blocks bool }{ - {"just under the threshold", 200 * time.Millisecond, false}, - {"just over the threshold", 400 * time.Millisecond, true}, + {"just under the threshold", 1500 * time.Millisecond, false}, + {"just over the threshold", 2100 * time.Millisecond, true}, {"no loop stuck", 0, false}, } { t.Run(tc.name, func(t *testing.T) { @@ -587,8 +587,8 @@ func TestLoopWatch1170(t *testing.T) { var nilSlot *LoopSlot nilSlot.Enter(1) nilSlot.Leave() // a loop without a slot (a test) is not a crash - if loopStuckFor(int64(time.Second)) != int64(500*time.Millisecond) || loopStuckFor(int64(100*time.Millisecond)) != int64(250*time.Millisecond) { - t.Fatal("the threshold is half a WriteTimeout, at least 250 ms") + if loopStuckFor(int64(time.Second)) != int64(2*time.Second) || loopStuckFor(int64(10*time.Second)) != int64(5*time.Second) { + t.Fatal("the threshold is half a WriteTimeout, at least 2 s") } } @@ -716,7 +716,7 @@ func TestLoopWatchTellsAnEpisodeOnce1170(t *testing.T) { wt := int64(time.Second) now := mono.Now() for i := 0; i < 5; i++ { // five judgements that see the loop stuck: one line - slot.Enter(now - int64(time.Second)) + slot.Enter(now - int64(3*time.Second)) if idx, _, ok := lw.stuck(now, loopStuckFor(wt)); ok { lw.noteStuck(idx, 0, wt) } @@ -726,7 +726,7 @@ func TestLoopWatchTellsAnEpisodeOnce1170(t *testing.T) { } slot.Leave() // it ends: a judgement that finds no loop stuck ends the episode lw.noteClear() - slot.Enter(now - int64(time.Second)) + slot.Enter(now - int64(3*time.Second)) lw.noteStuck(0, 0, wt) if got := h.n.Load(); got != 2 { t.Fatalf("%d lines after a second episode, want 2", got) diff --git a/test/server/h2_pool_wait_cells_1170_linux_test.go b/test/server/h2_pool_wait_cells_1170_linux_test.go index ec9a2a557..e796c1382 100644 --- a/test/server/h2_pool_wait_cells_1170_linux_test.go +++ b/test/server/h2_pool_wait_cells_1170_linux_test.go @@ -350,7 +350,9 @@ func TestH2PoolWaiterWithoutAWriteTimeoutKeepsTheOldClocks1170(t *testing.T) { // if it is another loop (the case the fallback is for) only the fallback to WriteTimeout // in all can: the trial is repeated on fresh connections, one locker each, so that a // server without the fallback fails it with the probability of a coin in every trial. -// Every locker is answered within 1.5 WriteTimeouts plus slack. +// Every locker is answered within the bound of the fallback, max(WriteTimeout/2, 2 s) +// + WriteTimeout (a loop is taken as stuck after half a WriteTimeout, and at least 2 s: +// a healthy loop of a loaded host has batches of a second or more), plus slack. func TestH2PoolWaiterHoldingALockDoesNotWedgeTheServer1170(t *testing.T) { const timeout = time.Second for _, e := range nativeEngines889() { @@ -418,8 +420,8 @@ func TestH2PoolWaiterHoldingALockDoesNotWedgeTheServer1170(t *testing.T) { if n != 1 { t.Errorf("trial %d: /locked not answered: the loop waited for the lock a compliant reader's download held", i) } - if took > 3*timeout { - t.Errorf("trial %d: the locker waited %v, want about 1.5 x WriteTimeout (%v) plus slack", i, took, timeout) + if bound := max(timeout/2, 2*time.Second) + timeout + time.Second; took > bound { + t.Errorf("trial %d: the locker waited %v, want within max(WriteTimeout/2, 2 s) + WriteTimeout (WriteTimeout %v) plus 1 s of slack = %v", i, took, timeout, bound) } } for i := 0; i < 8; i++ { @@ -430,11 +432,13 @@ func TestH2PoolWaiterHoldingALockDoesNotWedgeTheServer1170(t *testing.T) { } // TestH2PoolWaiterIsNotCutWhileALoopIsBlockedBelowTheThreshold1170 and ...Above...: -// the fallback is server-wide and has a threshold (half a WriteTimeout, at least 250 -// ms). A route that blocks one loop for 0.1 s at a time (under it) leaves Go's -// Transport 4 x 8 MiB at WriteTimeout 1 s as green as without it; one that blocks 0.8 +// the fallback is server-wide and has a threshold (half a WriteTimeout, at least 2 +// s). A route that blocks one loop for 0.1 s at a time (under it) leaves Go's +// Transport 4 x 8 MiB at WriteTimeout 1 s as green as without it; one that blocks 2.5 // s (over) switches the fallback on, and the downloads are bounded by WriteTimeout in -// all, as before celeris#1170: some are reset, as documented. +// all, as before celeris#1170: some are reset, as documented. (A batch of 0.8 to 1.9 s +// is not a wedge: that is what a healthy loop of a loaded host has, and the unit tests +// of the threshold pin it at the edge, on a stepped clock.) func TestH2PoolWaiterLoopBlockedThreshold1170(t *testing.T) { // "under" and "over" one after the other, the engines of one side by side: four // servers with their sleepers and downloads at once under -race or coverage on a @@ -444,7 +448,7 @@ func TestH2PoolWaiterLoopBlockedThreshold1170(t *testing.T) { name string block int // ms a batch blocks cut bool - }{{"under", 100, false}, {"over", 800, true}} { + }{{"under", 100, false}, {"over", 2500, true}} { t.Run(tc.name, func(t *testing.T) { for _, e := range nativeEngines889() { if e.name == "io_uring" { @@ -479,10 +483,10 @@ func TestH2PoolWaiterLoopBlockedThreshold1170(t *testing.T) { } t.Logf("LB1170 %s/%s: %d cut: %q at %v; %s", e.name, tc.name, cut, ends, at, p.logs.summary()) if tc.cut && (cut == 0 || p.logs.resets()["loop-blocked"] == 0 || p.logs.episodes() == 0) { - t.Fatalf("%d cut although a loop was blocked for 0.8 s at a time: the fallback to WriteTimeout in all must cut some, say so (%s)", cut, p.logs.summary()) + t.Fatalf("%d cut although a loop was blocked for 2.5 s at a time: the fallback to WriteTimeout in all must cut some, say so (%s)", cut, p.logs.summary()) } if !tc.cut && (cut != 0 || p.logs.resets()["loop-blocked"] != 0) { - t.Fatalf("%d downloads were cut although the loops were blocked for only 0.1 s at a time (under the threshold of 0.5 s): %s", cut, p.logs.summary()) + t.Fatalf("%d downloads were cut although the loops were blocked for only 0.1 s at a time (under the threshold of 2 s): %s", cut, p.logs.summary()) } }) } diff --git a/test/server/h2_pool_wait_rate_1170_linux_test.go b/test/server/h2_pool_wait_rate_1170_linux_test.go index be1b0bc1f..c69dcba4e 100644 --- a/test/server/h2_pool_wait_rate_1170_linux_test.go +++ b/test/server/h2_pool_wait_rate_1170_linux_test.go @@ -113,9 +113,9 @@ func downloadsOn1170(t *testing.T, cl *http.Client, addr, path string, n, size i func TestH2GoTransportParallelDownloadsOnASlowLinkAreNotCut1170(t *testing.T) { // One WriteTimeout at a time, and at 1 s one engine at a time: a few of these cells // at once under -race or a coverage profile starve each other's event loops (CI saw - // it), and a loop that is stuck in one batch for half a WriteTimeout switches the - // rule off for the server (the fallback), which at 1 s is a cell's flake, not a - // finding. At 5 s the threshold is 2.5 s and the engines run side by side. + // it), and a loop that is stuck in one batch for half a WriteTimeout (at least 2 s) + // switches the rule off for the server (the fallback), which at 1 s is a cell's flake, + // not a finding. At 5 s the threshold is 2.5 s and the engines run side by side. for _, wt := range []time.Duration{time.Second, 5 * time.Second} { t.Run(fmt.Sprintf("WT%v", wt), func(t *testing.T) { for _, e := range nativeEngines889() { @@ -151,16 +151,24 @@ func TestH2GoTransportParallelDownloadsOnASlowLinkAreNotCut1170(t *testing.T) { // = 14 s after its wait began, 14 windows (the idle allowance covers it, not the 3 of // the rule before celeris#1170). func TestH2GoTransportManyParallelDownloadsAreNotCut1170(t *testing.T) { + // One engine at a time: both side by side under -race and a coverage profile on two + // CPUs gave each other's event loops batches of a second or more (Coverage root shard + // c, run 41246492268), which is a cell's flake and not a finding. Each is its own + // server, and the property is per server: eight downloads on one connection are not + // cut by the rate rule. for _, e := range nativeEngines889() { if e.name == "io_uring" { continue // its own send clock at WriteTimeout 1 s, see above } t.Run(e.name, func(t *testing.T) { - t.Parallel() var p pool1170 addr := p.server(t, e.eng, -1, -1, time.Second) ends, at := downloads1170(t, addr, "/pool", 8, 8<<20, 2e6, 90*time.Second) t.Logf("GT1170 many %s: %q at %v; %s", e.name, ends, at, p.logs.summary()) + // The condition, before the timing: the healthy loop was not taken as a wedge. + if n := p.logs.resets()["loop-blocked"]; n != 0 || p.logs.episodes() != 0 { + t.Errorf("%d waits were reset as loop-blocked (%d episodes): a healthy loop of a loaded host is not a wedge; %s", n, p.logs.episodes(), p.logs.summary()) + } for i, r := range ends { if r != "complete" { t.Fatalf("download %d: %s at %v (all eight must complete); %s", i, r, at[i].Round(time.Millisecond), p.logs.summary()) From 2866411ee9cb32ba26fbfcc829f0b04f1fd6c92d Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sun, 1 Nov 2026 09:10:20 +0100 Subject: [PATCH 2/3] h2: a loop is taken as stuck after half a WriteTimeout and at least 2 s; a loop that dies leaves no stamp Refs #1170. The wedge fallback (LoopWatch) cuts every pool-handler wait that has lasted a WriteTimeout while any event loop has been in one batch for max(WriteTimeout/2, 250 ms). A healthy loop of an instrumented or overloaded host has batches of 0.4 to 1.9 s (one h2 connection's write-queue drain), so the cell of eight parallel downloads at WriteTimeout 1 s lost streams as loop-blocked (Coverage root shard c, run 41246492268). The floor is now 2 s (minLoopStuck): unchanged for a WriteTimeout of 4 s or more (the 60 s default keeps its 30 s), and a wedge is cut at most max(WriteTimeout/2, 2 s) + WriteTimeout after the loop blocks (3 s at WriteTimeout 1 s, was 1.5 WriteTimeouts). No hot-path cost. A loop or worker that ends without returning (runtime.Goexit in a sync handler, celeris#848) now clears its LoopWatch stamp on the way out: it was left set, and every waiting handler of the engine was cut as loop-blocked from then on. Docs: Config.WriteTimeout, the OutboundBudget comment and H2Config.LoopWatch state the threshold and bound. --- config.go | 14 +++++++------ internal/conn/h2.go | 5 +++-- internal/engine/epoll/loop_exit_848_linux.go | 4 ++++ .../engine/iouring/worker_exit_848_linux.go | 4 ++++ internal/protocol/h2/stream/outbound.go | 2 +- internal/protocol/h2/stream/write_wait.go | 20 ++++++++++++++----- 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/config.go b/config.go index ec8bde018..33aa22990 100644 --- a/config.go +++ b/config.go @@ -470,12 +470,14 @@ type Config struct { // block): its write returns an error that wraps os.ErrDeadlineExceeded when the // stream is reset. While an event loop of the server has been in one batch of // events (all the handlers it runs, not one) for half a WriteTimeout (at least - // 250 ms, which matters only for a WriteTimeout under 500 ms: at the 60s default - // the threshold is 30s), as a synchronous handler that waits for a lock a waiting - // handler holds across its response write makes it, or any slow blocking call, - // every such wait that has lasted a WriteTimeout is bounded by WriteTimeout in - // all, as before: about 1.5 WriteTimeouts from the start of the freeze, whatever - // the client reads. The server says so once per episode, in a Warn log line + // 2s, which matters only for a WriteTimeout under 4s: at the 60s default the + // threshold is 30s; a healthy loop of an overloaded or instrumented host has + // batches of a second or two), as a synchronous handler that waits for a lock a + // waiting handler holds across its response write makes it, or any slow blocking + // call, every such wait that has lasted a WriteTimeout is bounded by WriteTimeout + // in all, as before: at most max(WriteTimeout/2, 2s) + WriteTimeout from the start + // of the freeze (1.5 WriteTimeouts for a WriteTimeout of 4s or more), whatever the + // client reads. The server says so once per episode, in a Warn log line // ("the minimum rate of a pool handler's wait for window is suspended"), and // logs the reason of each such reset at Debug level (rate, or loop-blocked). A // client that stops reading altogether for a WriteTimeout (a pause of 1s at diff --git a/internal/conn/h2.go b/internal/conn/h2.go index 4338fef65..db44cdfd0 100644 --- a/internal/conn/h2.go +++ b/internal/conn/h2.go @@ -70,8 +70,9 @@ type H2Config struct { // WriteTimeout. 0 = no bound. WriteTimeout time.Duration // LoopWatch is the engine's watch on its event loops: while one of them is - // stuck in a batch for half a WriteTimeout, a pool handler's wait for window is - // bounded by WriteTimeout in all (celeris#1170, stream.LoopWatch). Nil: never. + // stuck in a batch for half a WriteTimeout (at least 2 s), a pool handler's wait + // for window is bounded by WriteTimeout in all (celeris#1170, stream.LoopWatch). + // Nil: never. LoopWatch *stream.LoopWatch // ReadTimeout is the request deadline of a stream (the server's // ReadTimeout): a request that is not whole this long after its HEADERS is diff --git a/internal/engine/epoll/loop_exit_848_linux.go b/internal/engine/epoll/loop_exit_848_linux.go index c943503aa..b52239417 100644 --- a/internal/engine/epoll/loop_exit_848_linux.go +++ b/internal/engine/epoll/loop_exit_848_linux.go @@ -50,6 +50,10 @@ func (e *Engine) runWatched(ctx context.Context, l *Loop) { // PauseAccept that checked in between. l.exited.Store(true) l.pause.Notify() + // A loop that ended inside a batch (a Goexit in a sync handler) has its stamp + // set: left, it ages for ever, and every waiting pool handler of the engine + // is then bounded by WriteTimeout in all as for a wedge (celeris#1170). + l.loopSlot.Leave() }() l.run(ctx) returned = true diff --git a/internal/engine/iouring/worker_exit_848_linux.go b/internal/engine/iouring/worker_exit_848_linux.go index acf15ee72..2366f2c74 100644 --- a/internal/engine/iouring/worker_exit_848_linux.go +++ b/internal/engine/iouring/worker_exit_848_linux.go @@ -51,6 +51,10 @@ func (e *Engine) runWatched(ctx context.Context, w *Worker) { // PauseAccept that checked in between. w.exited.Store(true) w.pause.Notify() + // A worker that ended inside a batch (a Goexit in a sync handler) has its stamp + // set: left, it ages for ever, and every waiting pool handler of the engine + // is then bounded by WriteTimeout in all as for a wedge (celeris#1170). + w.loopSlot.Leave() }() w.run(ctx) returned = true diff --git a/internal/protocol/h2/stream/outbound.go b/internal/protocol/h2/stream/outbound.go index 276964f2e..0b7c8a95f 100644 --- a/internal/protocol/h2/stream/outbound.go +++ b/internal/protocol/h2/stream/outbound.go @@ -54,7 +54,7 @@ import ( // requests. On epoll, io_uring and adaptive a sync handler that waits for it holds // its event loop, and so every connection on that loop, and with a rate a reader // that takes its floor would hold it for the whole download: so while an event loop -// of the server has been in one batch for half a WriteTimeout (at least 250 ms) +// of the server has been in one batch for half a WriteTimeout (at least 2 s) // every waiting handler that has waited a WriteTimeout is bounded by WriteTimeout in // all, as before (LoopWatch). With WriteTimeout disabled there is no bound: the wait // lasts until the peer grants window or the connection closes, and never ends if the diff --git a/internal/protocol/h2/stream/write_wait.go b/internal/protocol/h2/stream/write_wait.go index c4d164d7b..4db1683c4 100644 --- a/internal/protocol/h2/stream/write_wait.go +++ b/internal/protocol/h2/stream/write_wait.go @@ -381,7 +381,7 @@ func (w *LoopWatch) SetLogger(l *slog.Logger) { w.log.Store(l) } func (w *LoopWatch) noteStuck(idx int, age, wt int64) { if w.episode.CompareAndSwap(false, true) { if l := w.log.Load(); l != nil { - l.Warn("h2: the minimum rate of a pool handler's wait for window is suspended: an event loop has been in one batch for half a WriteTimeout, so such waits are bounded by WriteTimeout in all until it ends", + l.Warn("h2: the minimum rate of a pool handler's wait for window is suspended: an event loop has been in one batch for half a WriteTimeout (at least 2 s), so such waits are bounded by WriteTimeout in all until it ends", "loop", idx, "stuck_for", time.Duration(age), "write_timeout", time.Duration(wt), "threshold", time.Duration(loopStuckFor(wt))) } } @@ -452,12 +452,22 @@ func (w *LoopWatch) stuck(now, d int64) (idx int, age int64, ok bool) { return 0, 0, false } -// minLoopStuck is the shortest a loop's batch is taken as stuck: ordinary batches -// of a loaded loop are not. -const minLoopStuck = 250 * time.Millisecond +// minLoopStuck is the shortest a loop's batch is taken as stuck. A batch is all that +// one wake of the loop does (its events, the write-queue drain of its h2 connections), +// and on an overloaded or instrumented host a healthy loop's batch reaches a second or +// two: measured 0.4 to 1.9 s with -race and a coverage profile on two starved CPUs +// (0.3 s without), almost all of it one h2 connection's drain. A blocked loop (a +// handler waiting for a lock a waiting pool handler holds) is not a batch that short: +// it stays in the batch until the lock is released. At 250 ms the fallback cut healthy +// waiters of eight parallel downloads (celeris#1170, Coverage root shard c). Only a +// WriteTimeout under 4 s sees the floor; at the 60 s default the threshold is 30 s. +const minLoopStuck = 2 * time.Second // loopStuckFor is how long a loop must have been in one batch to switch the -// fallback on: half a WriteTimeout, and at least minLoopStuck. +// fallback on: half a WriteTimeout, and at least minLoopStuck. A wedge is then cut at +// the first judgement (one per WriteTimeout window) after that, so within +// loopStuckFor(wt) + wt of the loop's block: 1.5 WriteTimeouts for a WriteTimeout of +// 4 s or more, WriteTimeout + 2 s below. func loopStuckFor(wt int64) int64 { return max(wt/2, int64(minLoopStuck)) } From ab3e408b86969dbed62ebcdaaec444f71a32d311 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sun, 1 Nov 2026 15:23:25 +0100 Subject: [PATCH 3/3] engines: the dead-loop stamp test reads the dead loop's own slot; the exit clears the stamp first Refs #1170. Review r1 of #1202: - MINOR-1: BlockedFor(now+1h, 1) is true when ANY slot of the engine is stamped, so a live loop caught in a batch at that instant read as the dead loop's stamp (a flake shape on a starved runner). The tests find the loop or worker the /boom ran on (boomID848) and check its own slot (LoopSlot.Stamped). - NIT-1: the Leave ran after reportLoopExit/reportWorkerExit, whose Logger can block while the stamp ages and the engine's waiters are cut as loop-blocked. It is now the first statement of the deferred function. --- internal/engine/epoll/loop_exit_848_linux.go | 9 +++++---- internal/engine/epoll/loop_exit_848_linux_test.go | 14 +++++++++++--- internal/engine/iouring/worker_exit_848_linux.go | 9 +++++---- .../engine/iouring/worker_exit_848_linux_test.go | 14 +++++++++++--- internal/protocol/h2/stream/write_wait.go | 3 +++ 5 files changed, 35 insertions(+), 14 deletions(-) diff --git a/internal/engine/epoll/loop_exit_848_linux.go b/internal/engine/epoll/loop_exit_848_linux.go index b52239417..860f35d42 100644 --- a/internal/engine/epoll/loop_exit_848_linux.go +++ b/internal/engine/epoll/loop_exit_848_linux.go @@ -42,6 +42,11 @@ import ( func (e *Engine) runWatched(ctx context.Context, l *Loop) { returned := false defer func() { + // First: a loop that ended inside a batch (a Goexit in a sync handler) has its + // stamp set, and every statement below, a blocking Logger among them, lets it age + // while every waiting pool handler of the engine is bounded by WriteTimeout in + // all as for a wedge (celeris#1170). + l.loopSlot.Leave() if !returned { e.reportLoopExit(l) } @@ -50,10 +55,6 @@ func (e *Engine) runWatched(ctx context.Context, l *Loop) { // PauseAccept that checked in between. l.exited.Store(true) l.pause.Notify() - // A loop that ended inside a batch (a Goexit in a sync handler) has its stamp - // set: left, it ages for ever, and every waiting pool handler of the engine - // is then bounded by WriteTimeout in all as for a wedge (celeris#1170). - l.loopSlot.Leave() }() l.run(ctx) returned = true diff --git a/internal/engine/epoll/loop_exit_848_linux_test.go b/internal/engine/epoll/loop_exit_848_linux_test.go index 1531b44b4..b62c98b89 100644 --- a/internal/engine/epoll/loop_exit_848_linux_test.go +++ b/internal/engine/epoll/loop_exit_848_linux_test.go @@ -19,7 +19,6 @@ import ( "github.com/goceleris/celeris/internal/ctxkit" celerisengine "github.com/goceleris/celeris/internal/engine" - "github.com/goceleris/celeris/internal/mono" "github.com/goceleris/celeris/internal/protocol/h2/stream" "github.com/goceleris/celeris/internal/resource" ) @@ -180,10 +179,19 @@ func TestSyncHandlerGoexitEndsOneLoopLoudly848(t *testing.T) { // The loop died inside a batch, with its stamp set (celeris#1170): a stamp that // is left behind ages for ever, and every waiting pool handler of the engine would // be cut as "loop-blocked" from then on. The exit clears it. + // (Its own slot: a live loop caught in a batch at this instant is not the one.) e.mu.Lock() - watch := e.loopWatch + var dead *Loop + for _, x := range e.loops { + if int64(x.id) == boomID848.Load() { + dead = x + } + } e.mu.Unlock() - if watch.BlockedFor(mono.Now()+int64(time.Hour), 1) { + if dead == nil { + t.Fatalf("PREMISE: no loop with id %d", boomID848.Load()) + } + if dead.loopSlot.Stamped() { t.Error("the loop that ended without returning left its LoopWatch stamp set: it reads as stuck for ever") } // Only one loop died: a conn on another loop is still served. diff --git a/internal/engine/iouring/worker_exit_848_linux.go b/internal/engine/iouring/worker_exit_848_linux.go index 2366f2c74..04a253bc5 100644 --- a/internal/engine/iouring/worker_exit_848_linux.go +++ b/internal/engine/iouring/worker_exit_848_linux.go @@ -43,6 +43,11 @@ import ( func (e *Engine) runWatched(ctx context.Context, w *Worker) { returned := false defer func() { + // First: a worker that ended inside a batch (a Goexit in a sync handler) has its + // stamp set, and every statement below, a blocking Logger among them, lets it age + // while every waiting pool handler of the engine is bounded by WriteTimeout in + // all as for a wedge (celeris#1170). + w.loopSlot.Leave() if !returned { e.reportWorkerExit(w) } @@ -51,10 +56,6 @@ func (e *Engine) runWatched(ctx context.Context, w *Worker) { // PauseAccept that checked in between. w.exited.Store(true) w.pause.Notify() - // A worker that ended inside a batch (a Goexit in a sync handler) has its stamp - // set: left, it ages for ever, and every waiting pool handler of the engine - // is then bounded by WriteTimeout in all as for a wedge (celeris#1170). - w.loopSlot.Leave() }() w.run(ctx) returned = true diff --git a/internal/engine/iouring/worker_exit_848_linux_test.go b/internal/engine/iouring/worker_exit_848_linux_test.go index b54f80b47..4308554eb 100644 --- a/internal/engine/iouring/worker_exit_848_linux_test.go +++ b/internal/engine/iouring/worker_exit_848_linux_test.go @@ -19,7 +19,6 @@ import ( "github.com/goceleris/celeris/internal/ctxkit" "github.com/goceleris/celeris/internal/engine" - "github.com/goceleris/celeris/internal/mono" "github.com/goceleris/celeris/internal/protocol/h2/stream" "github.com/goceleris/celeris/internal/resource" ) @@ -206,10 +205,19 @@ func TestSyncHandlerGoexitEndsOneWorkerLoudly848(t *testing.T) { // The worker died inside a batch, with its stamp set (celeris#1170): a stamp that // is left behind ages for ever, and every waiting pool handler of the engine would // be cut as "loop-blocked" from then on. The exit clears it. + // (Its own slot: a live worker caught in a batch at this instant is not the one.) e.mu.Lock() - watch := e.loopWatch + var dead *Worker + for _, x := range e.workers { + if int64(x.id) == boomID848.Load() { + dead = x + } + } e.mu.Unlock() - if watch.BlockedFor(mono.Now()+int64(time.Hour), 1) { + if dead == nil { + t.Fatalf("PREMISE: no worker with id %d", boomID848.Load()) + } + if dead.loopSlot.Stamped() { t.Error("the worker that ended without returning left its LoopWatch stamp set: it reads as stuck for ever") } // Only one worker died: a conn on another worker is still served. diff --git a/internal/protocol/h2/stream/write_wait.go b/internal/protocol/h2/stream/write_wait.go index 4db1683c4..a62dc308b 100644 --- a/internal/protocol/h2/stream/write_wait.go +++ b/internal/protocol/h2/stream/write_wait.go @@ -431,6 +431,9 @@ func (s *LoopSlot) Leave() { } } +// Stamped reports whether the loop is inside a batch (its stamp is set). Tests. +func (s *LoopSlot) Stamped() bool { return s != nil && s.since.Load() != 0 } + // BlockedFor reports whether some loop has been in one batch since at least d // before now. func (w *LoopWatch) BlockedFor(now, d int64) bool {