Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions config.go
Original file line number Diff line number Diff line change
Expand Up @@ -470,12 +470,14 @@ type Config struct {
// block): its write returns an error that wraps os.ErrDeadlineExceeded when the
// stream is reset. While an event loop of the server has been in one batch of
// events (all the handlers it runs, not one) for half a WriteTimeout (at least
// 250 ms, which matters only for a WriteTimeout under 500 ms: at the 60s default
// the threshold is 30s), as a synchronous handler that waits for a lock a waiting
// handler holds across its response write makes it, or any slow blocking call,
// every such wait that has lasted a WriteTimeout is bounded by WriteTimeout in
// all, as before: about 1.5 WriteTimeouts from the start of the freeze, whatever
// the client reads. The server says so once per episode, in a Warn log line
// 2s, which matters only for a WriteTimeout under 4s: at the 60s default the
// threshold is 30s; a healthy loop of an overloaded or instrumented host has
// batches of a second or two), as a synchronous handler that waits for a lock a
// waiting handler holds across its response write makes it, or any slow blocking
// call, every such wait that has lasted a WriteTimeout is bounded by WriteTimeout
// in all, as before: at most max(WriteTimeout/2, 2s) + WriteTimeout from the start
// of the freeze (1.5 WriteTimeouts for a WriteTimeout of 4s or more), whatever the
// client reads. The server says so once per episode, in a Warn log line
// ("the minimum rate of a pool handler's wait for window is suspended"), and
// logs the reason of each such reset at Debug level (rate, or loop-blocked). A
// client that stops reading altogether for a WriteTimeout (a pause of 1s at
Expand Down
5 changes: 3 additions & 2 deletions internal/conn/h2.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,9 @@ type H2Config struct {
// WriteTimeout. 0 = no bound.
WriteTimeout time.Duration
// LoopWatch is the engine's watch on its event loops: while one of them is
// stuck in a batch for half a WriteTimeout, a pool handler's wait for window is
// bounded by WriteTimeout in all (celeris#1170, stream.LoopWatch). Nil: never.
// stuck in a batch for half a WriteTimeout (at least 2 s), a pool handler's wait
// for window is bounded by WriteTimeout in all (celeris#1170, stream.LoopWatch).
// Nil: never.
LoopWatch *stream.LoopWatch
// ReadTimeout is the request deadline of a stream (the server's
// ReadTimeout): a request that is not whole this long after its HEADERS is
Expand Down
5 changes: 5 additions & 0 deletions internal/engine/epoll/loop_exit_848_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,11 @@ import (
func (e *Engine) runWatched(ctx context.Context, l *Loop) {
returned := false
defer func() {
// First: a loop that ended inside a batch (a Goexit in a sync handler) has its
// stamp set, and every statement below, a blocking Logger among them, lets it age
// while every waiting pool handler of the engine is bounded by WriteTimeout in
// all as for a wedge (celeris#1170).
l.loopSlot.Leave()
if !returned {
e.reportLoopExit(l)
}
Expand Down
18 changes: 18 additions & 0 deletions internal/engine/epoll/loop_exit_848_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,24 @@ func TestSyncHandlerGoexitEndsOneLoopLoudly848(t *testing.T) {
for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl) && e.abnormalLoopExits.Load() == 0; time.Sleep(5 * time.Millisecond) {
}
time.Sleep(200 * time.Millisecond)
// The loop died inside a batch, with its stamp set (celeris#1170): a stamp that
// is left behind ages for ever, and every waiting pool handler of the engine would
// be cut as "loop-blocked" from then on. The exit clears it.
// (Its own slot: a live loop caught in a batch at this instant is not the one.)
e.mu.Lock()
var dead *Loop
for _, x := range e.loops {
if int64(x.id) == boomID848.Load() {
dead = x
}
}
e.mu.Unlock()
if dead == nil {
t.Fatalf("PREMISE: no loop with id %d", boomID848.Load())
}
if dead.loopSlot.Stamped() {
t.Error("the loop that ended without returning left its LoopWatch stamp set: it reads as stuck for ever")
}
// Only one loop died: a conn on another loop is still served.
var otherAnswered, others int
for k, c := range conns {
Expand Down
5 changes: 5 additions & 0 deletions internal/engine/iouring/worker_exit_848_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@ import (
func (e *Engine) runWatched(ctx context.Context, w *Worker) {
returned := false
defer func() {
// First: a worker that ended inside a batch (a Goexit in a sync handler) has its
// stamp set, and every statement below, a blocking Logger among them, lets it age
// while every waiting pool handler of the engine is bounded by WriteTimeout in
// all as for a wedge (celeris#1170).
w.loopSlot.Leave()
if !returned {
e.reportWorkerExit(w)
}
Expand Down
18 changes: 18 additions & 0 deletions internal/engine/iouring/worker_exit_848_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,24 @@ func TestSyncHandlerGoexitEndsOneWorkerLoudly848(t *testing.T) {
for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl) && e.abnormalWorkerExits.Load() == 0; time.Sleep(5 * time.Millisecond) {
}
time.Sleep(200 * time.Millisecond)
// The worker died inside a batch, with its stamp set (celeris#1170): a stamp that
// is left behind ages for ever, and every waiting pool handler of the engine would
// be cut as "loop-blocked" from then on. The exit clears it.
// (Its own slot: a live worker caught in a batch at this instant is not the one.)
e.mu.Lock()
var dead *Worker
for _, x := range e.workers {
if int64(x.id) == boomID848.Load() {
dead = x
}
}
e.mu.Unlock()
if dead == nil {
t.Fatalf("PREMISE: no worker with id %d", boomID848.Load())
}
if dead.loopSlot.Stamped() {
t.Error("the worker that ended without returning left its LoopWatch stamp set: it reads as stuck for ever")
}
// Only one worker died: a conn on another worker is still served.
var otherAnswered, others int
for k, c := range conns {
Expand Down
107 changes: 107 additions & 0 deletions internal/protocol/h2/stream/loop_stuck_floor_1170_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package stream

import (
"testing"
"time"

"github.com/goceleris/celeris/internal/mono"
)

// celeris#1170, the loop-stuck fallback (LoopWatch): a loop is taken as stuck when it
// has been in ONE batch for max(WriteTimeout/2, floor). With the floor at 250 ms a
// healthy loop of a host that is overloaded or runs -race and a coverage profile (a
// batch of 0.4 to 1.9 s measured, mostly one h2 connection's write-queue drain) cut
// healthy waiters of 8 parallel downloads as "loop-blocked" (Coverage root shard c,
// run 41246492268; 7 of 72 cells with two busy loops on two CPUs). A blocked loop
// (a synchronous handler waiting for a lock a waiting pool handler holds) is never a
// batch that short, so the floor is 2 s.
//
// The clock is mono.Manual: the verdicts are exact, with no sleep and no margin for
// the speed of the runner.

func TestLoopStuckThresholdHasAFloor1170(t *testing.T) {
for _, tc := range []struct{ wt, want time.Duration }{
{100 * time.Millisecond, 2 * time.Second},
{time.Second, 2 * time.Second},
{4 * time.Second, 2 * time.Second},
{5 * time.Second, 2500 * time.Millisecond},
{10 * time.Second, 5 * time.Second},
{60 * time.Second, 30 * time.Second}, // the default WriteTimeout: unchanged
} {
if got := time.Duration(loopStuckFor(int64(tc.wt))); got != tc.want {
t.Errorf("loopStuckFor(%v) = %v, want %v", tc.wt, got, tc.want)
}
}
}

// A long batch that is not yet the threshold old is not a wedge; at the threshold it is.
func TestHealthyLongBatchIsNotAWedge1170(t *testing.T) {
for _, tc := range []struct {
name string
wt time.Duration
age time.Duration // how long the loop has been in its batch at the judgement
waited time.Duration // how long the waiter has waited
cut bool
}{
{"WT 1s, batch of 0.8 s (healthy, instrumented)", time.Second, 800 * time.Millisecond, 20 * time.Second, false},
{"WT 1s, batch of 1.5 s", time.Second, 1500 * time.Millisecond, 20 * time.Second, false},
{"WT 1s, batch of 1.9 s", time.Second, 1900 * time.Millisecond, 20 * time.Second, false},
{"WT 1s, batch of 2 s", time.Second, 2 * time.Second, 20 * time.Second, true},
{"WT 1s, batch of 3 s, waited under a WriteTimeout", time.Second, 3 * time.Second, 500 * time.Millisecond, false},
{"WT 5s, batch of 2.4 s", 5 * time.Second, 2400 * time.Millisecond, 20 * time.Second, false},
{"WT 5s, batch of 2.6 s", 5 * time.Second, 2600 * time.Millisecond, 20 * time.Second, true},
{"WT 60s (default), batch of 29 s", time.Minute, 29 * time.Second, 70 * time.Second, false},
{"WT 60s (default), batch of 31 s", time.Minute, 31 * time.Second, 70 * time.Second, true},
} {
t.Run(tc.name, func(t *testing.T) {
clk := mono.NewManual()
t.Cleanup(mono.SetManual(clk))
m, s := waitRig1170(t, tc.wt, 0)
lw := NewLoopWatch()
lw.Slot().Enter(mono.Now() - int64(tc.age))
w := SendWait{Loops: lw, start: mono.Now() - int64(tc.waited)}
v, _ := m.judgeWait(s, s.ID, &w, 4<<20)
if cut := v == waitExpired && w.BlockedLoop(); cut != tc.cut {
t.Fatalf("verdict %v, loop-blocked %v: want cut=%v", v, w.BlockedLoop(), tc.cut)
}
})
}
}

// The bound of a wedge: a loop that blocks while a waiter is waiting is cut at the first
// judgement (one per WriteTimeout window) at which its batch is max(WT/2, 2 s) old and
// the wait has lasted a WriteTimeout, so at most that plus one WriteTimeout after the
// loop blocked. Positive control: the fallback still cuts, within the stated bound.
func TestBlockedLoopIsCutWithinTheBound1170(t *testing.T) {
for _, wt := range []time.Duration{time.Second, 5 * time.Second, 60 * time.Second} {
t.Run(wt.String(), func(t *testing.T) {
clk := mono.NewManual()
t.Cleanup(mono.SetManual(clk))
m, s := waitRig1170(t, wt, 0)
lw := NewLoopWatch()
slot := lw.Slot()
start := mono.Now() // the waiter blocks
clk.Advance(wt * 7 / 10)
slot.Enter(mono.Now()) // the loop blocks 0.7 WT into the wait and never returns
w := SendWait{Loops: lw, start: start}
var cutAt time.Duration
for k := 1; k <= 400 && cutAt == 0; k++ { // the waiter's timer: one judgement per window
clk.Advance(wt)
m.dropWait(s.ID) // the rate rule is not under test: the record is made afresh
if v, _ := m.judgeWait(s, s.ID, &w, 4<<20); v == waitExpired {
if !w.BlockedLoop() {
t.Fatalf("cut by the rate rule at judgement %d", k)
}
cutAt = time.Duration(mono.Now() - slot.since.Load()) // how long the loop had been blocked
}
}
bound := time.Duration(loopStuckFor(int64(wt))) + wt // threshold, then one window to the next judgement
if cutAt == 0 {
t.Fatalf("a blocked loop was never cut")
}
if cutAt < time.Duration(loopStuckFor(int64(wt))) || cutAt > bound {
t.Fatalf("cut %v after the loop blocked, want within [%v, %v]", cutAt, time.Duration(loopStuckFor(int64(wt))), bound)
}
})
}
}
2 changes: 1 addition & 1 deletion internal/protocol/h2/stream/outbound.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ import (
// requests. On epoll, io_uring and adaptive a sync handler that waits for it holds
// its event loop, and so every connection on that loop, and with a rate a reader
// that takes its floor would hold it for the whole download: so while an event loop
// of the server has been in one batch for half a WriteTimeout (at least 250 ms)
// of the server has been in one batch for half a WriteTimeout (at least 2 s)
// every waiting handler that has waited a WriteTimeout is bounded by WriteTimeout in
// all, as before (LoopWatch). With WriteTimeout disabled there is no bound: the wait
// lasts until the peer grants window or the connection closes, and never ends if the
Expand Down
23 changes: 18 additions & 5 deletions internal/protocol/h2/stream/write_wait.go
Original file line number Diff line number Diff line change
Expand Up @@ -381,7 +381,7 @@ func (w *LoopWatch) SetLogger(l *slog.Logger) { w.log.Store(l) }
func (w *LoopWatch) noteStuck(idx int, age, wt int64) {
if w.episode.CompareAndSwap(false, true) {
if l := w.log.Load(); l != nil {
l.Warn("h2: the minimum rate of a pool handler's wait for window is suspended: an event loop has been in one batch for half a WriteTimeout, so such waits are bounded by WriteTimeout in all until it ends",
l.Warn("h2: the minimum rate of a pool handler's wait for window is suspended: an event loop has been in one batch for half a WriteTimeout (at least 2 s), so such waits are bounded by WriteTimeout in all until it ends",
"loop", idx, "stuck_for", time.Duration(age), "write_timeout", time.Duration(wt), "threshold", time.Duration(loopStuckFor(wt)))
}
}
Expand Down Expand Up @@ -431,6 +431,9 @@ func (s *LoopSlot) Leave() {
}
}

// Stamped reports whether the loop is inside a batch (its stamp is set). Tests.
func (s *LoopSlot) Stamped() bool { return s != nil && s.since.Load() != 0 }

// BlockedFor reports whether some loop has been in one batch since at least d
// before now.
func (w *LoopWatch) BlockedFor(now, d int64) bool {
Expand All @@ -452,12 +455,22 @@ func (w *LoopWatch) stuck(now, d int64) (idx int, age int64, ok bool) {
return 0, 0, false
}

// minLoopStuck is the shortest a loop's batch is taken as stuck: ordinary batches
// of a loaded loop are not.
const minLoopStuck = 250 * time.Millisecond
// minLoopStuck is the shortest a loop's batch is taken as stuck. A batch is all that
// one wake of the loop does (its events, the write-queue drain of its h2 connections),
// and on an overloaded or instrumented host a healthy loop's batch reaches a second or
// two: measured 0.4 to 1.9 s with -race and a coverage profile on two starved CPUs
// (0.3 s without), almost all of it one h2 connection's drain. A blocked loop (a
// handler waiting for a lock a waiting pool handler holds) is not a batch that short:
// it stays in the batch until the lock is released. At 250 ms the fallback cut healthy
// waiters of eight parallel downloads (celeris#1170, Coverage root shard c). Only a
// WriteTimeout under 4 s sees the floor; at the 60 s default the threshold is 30 s.
const minLoopStuck = 2 * time.Second

// loopStuckFor is how long a loop must have been in one batch to switch the
// fallback on: half a WriteTimeout, and at least minLoopStuck.
// fallback on: half a WriteTimeout, and at least minLoopStuck. A wedge is then cut at
// the first judgement (one per WriteTimeout window) after that, so within
// loopStuckFor(wt) + wt of the loop's block: 1.5 WriteTimeouts for a WriteTimeout of
// 4 s or more, WriteTimeout + 2 s below.
func loopStuckFor(wt int64) int64 {
return max(wt/2, int64(minLoopStuck))
}
14 changes: 7 additions & 7 deletions internal/protocol/h2/stream/write_wait_1170_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -516,16 +516,16 @@ func TestIdleAllowanceIsFixedWhenTheRunBegins1170(t *testing.T) {
}

// While an event loop is stuck in one batch for half a WriteTimeout (and at least
// 250 ms) a waiter that has waited a whole WriteTimeout is reset at its next
// 2 s) a waiter that has waited a whole WriteTimeout is reset at its next
// judgement, whatever its seed and credit say; just under the threshold it is not.
func TestWaiterLoopBlockedFallback1170(t *testing.T) {
for _, tc := range []struct {
name string
stuck time.Duration // how long the loop has been in its batch when the waiter first blocks
blocks bool
}{
{"just under the threshold", 200 * time.Millisecond, false},
{"just over the threshold", 400 * time.Millisecond, true},
{"just under the threshold", 1500 * time.Millisecond, false},
{"just over the threshold", 2100 * time.Millisecond, true},
{"no loop stuck", 0, false},
} {
t.Run(tc.name, func(t *testing.T) {
Expand Down Expand Up @@ -587,8 +587,8 @@ func TestLoopWatch1170(t *testing.T) {
var nilSlot *LoopSlot
nilSlot.Enter(1)
nilSlot.Leave() // a loop without a slot (a test) is not a crash
if loopStuckFor(int64(time.Second)) != int64(500*time.Millisecond) || loopStuckFor(int64(100*time.Millisecond)) != int64(250*time.Millisecond) {
t.Fatal("the threshold is half a WriteTimeout, at least 250 ms")
if loopStuckFor(int64(time.Second)) != int64(2*time.Second) || loopStuckFor(int64(10*time.Second)) != int64(5*time.Second) {
t.Fatal("the threshold is half a WriteTimeout, at least 2 s")
}
}

Expand Down Expand Up @@ -716,7 +716,7 @@ func TestLoopWatchTellsAnEpisodeOnce1170(t *testing.T) {
wt := int64(time.Second)
now := mono.Now()
for i := 0; i < 5; i++ { // five judgements that see the loop stuck: one line
slot.Enter(now - int64(time.Second))
slot.Enter(now - int64(3*time.Second))
if idx, _, ok := lw.stuck(now, loopStuckFor(wt)); ok {
lw.noteStuck(idx, 0, wt)
}
Expand All @@ -726,7 +726,7 @@ func TestLoopWatchTellsAnEpisodeOnce1170(t *testing.T) {
}
slot.Leave() // it ends: a judgement that finds no loop stuck ends the episode
lw.noteClear()
slot.Enter(now - int64(time.Second))
slot.Enter(now - int64(3*time.Second))
lw.noteStuck(0, 0, wt)
if got := h.n.Load(); got != 2 {
t.Fatalf("%d lines after a second episode, want 2", got)
Expand Down
Loading
Loading