diff --git a/.github/root-test-shards.txt b/.github/root-test-shards.txt index b24089817..b4658a436 100644 --- a/.github/root-test-shards.txt +++ b/.github/root-test-shards.txt @@ -18,6 +18,7 @@ # Add a test when it takes 3 s or more; keep the shards within ~10% of each other (the CI seconds below). # # shard a: ~221 s of CI test time, shard b: ~218 s; the rest: ~114 s (incl. TestAdaptiveSettledRouteRetime592, 37 s). +# After the write-deadline cells of celeris#889 and #1170: shards a and b ~360 s, shard c ~370 s (unit run 41114943307 had a and b at 519 and 497 s of 660 s before shard c). a TestUnmatchedChainGoexitIsLoud848 a TestShutdownHooksRunAfterTheDrain @@ -73,6 +74,20 @@ b TestH2ConnWindowStarvedStreamsAreJudgedPerConnection889 b TestH2FlowingDownloadAndQuietStreamAreNotCut889 b TestH2SilentClientWithAStalledResponseKeepsTheConnUntilTheStreamIsReset889 +# The pool handler's wait for window as a minimum rate (celeris#1170, stage 3), a shard of its own: 5 to 136 s each, 430 s together. +# (Shards a and b ran at 79% of their timeout with these in them: review r11 N4.) +c TestH2GoTransportParallelDownloadsOnASlowLinkAreNotCut1170 +c TestH2PoolWaiterHoldingALockDoesNotWedgeTheServer1170 +c TestH2PoolWaiterThatGrantsNothingIsReset1170 +c TestH2PoolWaiterLoopBlockedThreshold1170 +c TestH2GoTransportManyParallelDownloadsAreNotCut1170 +c TestH2GoTransportQueueBeyondTheCapIsCut1170 +c TestH2PoolWaiterBurstyReaderIsNotClosedByReadTimeout1170 +c TestH2PoolWaiterOfASilentClientEndsTheConnection1170 +c TestH2PoolWaiterDribblerIsReset1170 +c TestH2PoolStreamWriterDribblerIsReset1170 +c TestH2PoolWaiterAfterTheLoopsParked1170 + # The rest step of the Unit job (test/server, -race, -timeout=540s) was at 539.2 s on the merge of #1169 (main alone: 501 s), 0.8 s from its timeout: # the 14 slowest tests of the root packages still in it (8 to 34 s each, 216 s of CI test time) move to the shards, which have 300 s of room. a TestShutdownConnCloseContract884 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7a7c9252..ff5f4fe62 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1463,8 +1463,10 @@ jobs: # the runner today), because a skip is absent, not a pass. # # -timeout=660s: the shards took 214.7 s (a) and 225.5 s (b) on the runner (run - # 38348908017; 215.3 s and 220.5 s on a linux/arm64 container with 4 CPUs), so - # 660 s keeps a margin of 2.9x or more. + # 38348908017; 215.3 s and 220.5 s on a linux/arm64 container with 4 CPUs). With the + # write-deadline cells (celeris#889, #1170) in a and b they ran at 519 s and 497 s + # (79% of the timeout); the cells of #1170 have a shard of their own (c) now, and + # a, b and c are about 360 s each, a margin of 1.8x. unit-root-shard: name: Unit root shard ${{ matrix.shard }} runs-on: ubuntu-latest @@ -1474,7 +1476,7 @@ jobs: matrix: # Every shard named in .github/root-test-shards.txt (the guard in the # `unit` job and in test-coverage.yml fails when the two differ). - shard: [a, b] + shard: [a, b, c] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/test-coverage.yml b/.github/workflows/test-coverage.yml index e4be778c1..60705e0b2 100644 --- a/.github/workflows/test-coverage.yml +++ b/.github/workflows/test-coverage.yml @@ -286,11 +286,13 @@ jobs: # own 8 MiB memlock). Each shard writes a profile of the root package for its # tests and uploads it under the `root` flag; the `coverage` job uploads the # rest under the same flag, and Codecov adds the uploads of a commit together - # (codecov.yml waits for all seven). The shard's tests must each run and PASS + # (codecov.yml waits for all eight). The shard's tests must each run and PASS # by name, as in ci.yml. Not a required check, like the job above. # # -timeout=660s: the shards took 215.5 s (a) and 221.2 s (b) under coverage on - # the runner (run 38348908034), so 660 s keeps a margin of 3x. + # the runner (run 38348908034). With the write-deadline cells (celeris#889, #1170) in + # a and b they ran at 523 s and 497 s (79% of the timeout); the cells of #1170 have + # a shard of their own (c) now, and a, b and c are about 360 s each, a margin of 1.8x. coverage-root-shard: name: Coverage root shard ${{ matrix.shard }} runs-on: ubuntu-latest @@ -300,7 +302,7 @@ jobs: matrix: # Every shard named in .github/root-test-shards.txt (the guard in the # `coverage` job's root step fails when the two differ). - shard: [a, b] + shard: [a, b, c] permissions: contents: read # checkout id-token: write # Codecov upload over OIDC, as the job above diff --git a/codecov.yml b/codecov.yml index 1901d974c..ff57ea416 100644 --- a/codecov.yml +++ b/codecov.yml @@ -14,13 +14,13 @@ codecov: require_ci_to_pass: false notify: wait_for_ci: false - # Seven uploads per commit: root (the coverage job's), the two root-package + # Eight uploads per commit: root (the coverage job's), the three root-package # shards (coverage-root-shard jobs, also flagged root; celeris#890), - # compress, metrics, otel, protobuf. Waiting for all seven keeps Codecov + # compress, metrics, otel, protobuf. Waiting for all eight keeps Codecov # from posting a partial, misleading number. The root upload carries two # profile files (the root step's and ./test/server's, which run with # -coverpkg on the root package); that is still one upload. - after_n_builds: 7 + after_n_builds: 8 coverage: precision: 2 @@ -142,9 +142,9 @@ comment: # One comment per pull request, edited in place on every push. behavior: default require_changes: false - # The same seven uploads as codecov.notify.after_n_builds above: a comment + # The same eight uploads as codecov.notify.after_n_builds above: a comment # posted after five would show a partial number. - after_n_builds: 7 + after_n_builds: 8 github_checks: # No per-line "not covered" annotations in the Files tab. Many io_uring and diff --git a/config.go b/config.go index 54a1c5c18..11ac70efd 100644 --- a/config.go +++ b/config.go @@ -376,8 +376,10 @@ type Config struct { // client's HTTP/2 flow-control window is not parked (nothing is queued on // the socket); WriteTimeout bounds its stream in its own right, as a // minimum rate, on the native engines: in each WriteTimeout window a stream - // that holds buffered bytes must have had at least 64 KiB of them (or all - // it held, if less) sent, released by the client's WINDOW_UPDATEs, or it is + // that holds bytes waiting for the window, buffered or still held by a handler + // on the worker pool that is blocked in its write, must have had at least + // 64 KiB of them (or all it held, if less) sent, released by the client's + // WINDOW_UPDATEs, or it is // reset with RST_STREAM(INTERNAL_ERROR), as net/http resets a stream at its // write deadline. There is no ceiling, where std's deadline is absolute: a // stream that releases at least that on average, lagging the rate by no @@ -389,22 +391,39 @@ type Config struct { // updates stays under about five WriteTimeouts: half the window divided by the // rate it reads at. When the connection as a whole releases at least 64 KiB // in a window, a stream that takes less (parallel downloads share the - // client's socket) is held while it takes half its fair share of it, and for - // three windows in a row when it takes less: a download queued behind its - // siblings' bytes in the client's socket is cut where the bytes ahead of it - // take longer than three WriteTimeouts to read (N streams of W bytes in flight - // on a link of L bytes/s: N x W / L), so a server whose clients are on slow - // links should set WriteTimeout to 5s or more. When the connection itself - // releases less than 64 KiB while it holds more, the first window is the - // verdict. The end of a response is given grace: a stream that holds less than + // client's socket) is held while it takes half its fair share of it, and, + // when it takes less, for as many windows in a row as the client needs to read + // what is in flight on the connection at the rate the connection moves (the + // bytes sent and not yet granted back at the connection level over what the + // connection released in the window), at least three and at most twelve. The + // connection counts as moving for four windows after its other streams last + // released 64 KiB, so a connection whose other streams release a floor only once + // in four windows keeps its stalled streams for as long as the allowance lasts: a + // client that returns window in big steps leaves some windows with neither a grant + // nor a send. Those windows count as idle ones, so the bound below holds. A + // download queued behind its siblings' bytes in the client's socket is cut where + // the bytes ahead of it take longer than that to read (N streams of W bytes in + // flight on a link of L bytes/s: N x W / L over about 16 WriteTimeouts, the cut + // beginning between 16.8 and 21), so a server whose clients are on slow links + // should set WriteTimeout to 2s or more, and 5s where the links are slower than a + // megabyte a second. Measured with Go's HTTP/2 client on epoll and adaptive: 4 x + // 8 MiB on 1 MB/s and 8 x 8 MiB on 2 MB/s complete at 1s, 2s and 5s, with about one + // window of headroom; on io_uring the engine's own send clock closes such a + // connection at 1s, so io_uring is measured from 5s. A stream that is paused, or + // dribbles one byte at a time, beside siblings that keep the connection moving is + // held for as long: up to 17 windows in all (13 for the default window), against 8 + // when little is in flight. When the connection itself + // releases less than 64 KiB while it holds more, for more windows than the + // bank covers, the verdict is the first window that does not move. The end of a response is given grace: a stream that holds less than // 64 KiB, and has had half of it sent in the window, or any stream while the // connection as a whole holds less than 64 KiB (the last bytes of the transfer // are on the wire, and the grant for them can wait behind its siblings' bytes in // the client's socket for more than a window), is held for three windows more // and reset in the fourth if nothing moves it. A stream the client never // grants is reset after 1+k windows alone, or 4+k with siblings that keep the - // connection moving, k being the number of whole 64 KiB in the bytes in flight - // when it stalled (about the peer's initial window), at most four: 1 and 4 + // connection moving and little in flight on it, k being the number of whole + // 64 KiB in the bytes in flight when it stalled (about the peer's initial + // window), at most four: 1 and 4 // WriteTimeouts for the default 64 KiB window, 5 and 8 for a window of 256 KiB or // more. When what it has left to send is under 64 KiB (the response is a // little longer than the window), the bytes in flight are spent a whole floor a @@ -418,10 +437,28 @@ type Config struct { // (WriteTimeout is its bound), so a silent client's unread response, and the // buffer it pins, lives for up to 8 WriteTimeouts (5, with the default window) // where it used to be dropped with the whole connection at ReadTimeout. A - // handler blocked waiting for window keeps its own, absolute, bound of - // WriteTimeout from the start of the wait, whatever the client reads: parallel - // downloads on one connection whose responses exceed the connection's 4 MiB - // output budget can lose the later ones to it. A request error read while a + // handler on the worker pool that blocks in its write for the client's window + // (a response over the connection's 4 MiB output budget, a StreamWriter's + // Write) is bounded by the same rule, judged by its own timer from the moment it + // first blocks, so that parallel downloads on one connection are not lost to a + // deadline that is absolute (celeris#906 had WriteTimeout in all from the first + // block): its write returns an error that wraps os.ErrDeadlineExceeded when the + // stream is reset. While an event loop of the server has been in one batch of + // events (all the handlers it runs, not one) for half a WriteTimeout (at least + // 250 ms, which matters only for a WriteTimeout under 500 ms: at the 60s default + // the threshold is 30s), as a synchronous handler that waits for a lock a waiting + // handler holds across its response write makes it, or any slow blocking call, + // every such wait that has lasted a WriteTimeout is bounded by WriteTimeout in + // all, as before: about 1.5 WriteTimeouts from the start of the freeze, whatever + // the client reads. The server says so once per episode, in a Warn log line + // ("the minimum rate of a pool handler's wait for window is suspended"), and + // logs the reason of each such reset at Debug level (rate, or loop-blocked). A + // client that stops reading altogether for a WriteTimeout (a pause of 1s at + // WriteTimeout 1s) loses its connection to the engines' write clock on epoll and + // io_uring alike, with or without this rule, and on io_uring so does a client that + // reads at 1 MB/s at WriteTimeout 1s (a send that does not complete for a + // WriteTimeout); use 2s or more for clients that pause, 5s for io_uring on slow + // links. A request error read while a // response is queued (a malformed request) puts the connection on the // closing drain below, which counts each byte the client acknowledges and // has no floor: a client that sends one and then takes a few bytes per diff --git a/internal/conn/h2.go b/internal/conn/h2.go index 5aa70a71f..4338fef65 100644 --- a/internal/conn/h2.go +++ b/internal/conn/h2.go @@ -69,6 +69,10 @@ type H2Config struct { // wrapping os.ErrDeadlineExceeded. The engines pass the server's // WriteTimeout. 0 = no bound. WriteTimeout time.Duration + // LoopWatch is the engine's watch on its event loops: while one of them is + // stuck in a batch for half a WriteTimeout, a pool handler's wait for window is + // bounded by WriteTimeout in all (celeris#1170, stream.LoopWatch). Nil: never. + LoopWatch *stream.LoopWatch // ReadTimeout is the request deadline of a stream (the server's // ReadTimeout): a request that is not whole this long after its HEADERS is // answered 408 and reset with RST_STREAM(NO_ERROR) (celeris#889). Absolute, @@ -626,6 +630,8 @@ func NewH2State(handler stream.Handler, cfg H2Config, write func([]byte), wake * proc.GetManager().SetHints(&s.hints) proc.GetManager().SetRequestTimeout(cfg.ReadTimeout) proc.GetManager().SetWriteTimeout(cfg.WriteTimeout) + s.adapter.loops = cfg.LoopWatch + s.adapter.log = cfg.Logger proc.SetHeaderBlockTimeout(cfg.ReadHeaderTimeout) if lg := cfg.Logger; lg != nil { proc.OnHandlerPanic = func(id uint32, r any, stack []byte) { diff --git a/internal/conn/h2_sweep_889_test.go b/internal/conn/h2_sweep_889_test.go index 0c1a297bb..14669b00e 100644 --- a/internal/conn/h2_sweep_889_test.go +++ b/internal/conn/h2_sweep_889_test.go @@ -573,6 +573,7 @@ func TestSweepResetsAStalledStreamWithInternalError889(t *testing.T) { for w := 1; w <= 4; w++ { b.SetWindowSize(65536) m.FlushOutbound(b, func(uint32, bool, []byte) {}) + m.UpdateConnectionWindow(65536) // the client reads, and returns the connection window: nothing queued ahead res = r.st.Sweep(now+int64(w)*int64(1100*time.Millisecond), r.write) if (w < 4) != (res.Expired == 0) { t.Fatalf("window %d: %+v", w, res) diff --git a/internal/conn/h2_wait_rate_1170_test.go b/internal/conn/h2_wait_rate_1170_test.go new file mode 100644 index 000000000..c6aa916d4 --- /dev/null +++ b/internal/conn/h2_wait_rate_1170_test.go @@ -0,0 +1,39 @@ +package conn + +import ( + "testing" + "time" + + "golang.org/x/net/http2" +) + +// celeris#1170 (review r2 F4): the reset of a wait that was judged goes only if the +// call cancelled the stream. A stream the peer reset between the verdict and the +// enqueue (its RST_STREAM was processed in between) gets no second RST_STREAM +// (RFC 9113 §5.1), which #906's expiry sent. +func TestWaitExpiryAfterThePeersResetSendsNoSecondRST1170(t *testing.T) { + for _, tc := range []struct { + name string + peerFirst bool + wantResets int + }{{"the server alone", false, 1}, {"the peer reset it first", true, 0}} { + t.Run(tc.name, func(t *testing.T) { + w := newWaitBound893(t, "async", time.Hour) // the handler waits; its own timer will not fire + time.Sleep(150 * time.Millisecond) // it is blocked in sendRest + s, ok := w.st.processor.GetManager().GetStream(1) + if !ok { + t.Fatal("no stream 1") + } + if tc.peerFirst { + s.Cancel() // the peer's RST_STREAM, as the processor applies it + } + w.st.adapter.resetStreamOnce(s, http2.ErrCodeInternal) + <-w.done // the cancel ends the handler's wait + w.st.DrainWriteQueue(w.write) + _, _, _, rst := w.wire1() + if len(rst) != tc.wantResets { + t.Fatalf("%d RST_STREAM frames on the wire (%v), want %d", len(rst), rst, tc.wantResets) + } + }) + } +} diff --git a/internal/conn/response.go b/internal/conn/response.go index e5bad053b..60e0a53ad 100644 --- a/internal/conn/response.go +++ b/internal/conn/response.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "errors" "fmt" + "log/slog" "math" "math/bits" "net" @@ -810,6 +811,13 @@ type h2ResponseAdapter struct { // writeTimeout bounds how long sendRest waits for the peer's window // (H2Config.WriteTimeout; 0 = no bound). writeTimeout time.Duration + // loops is the engine's watch on its event loops (H2Config.LoopWatch): a pool + // handler's wait is bounded by WriteTimeout in all while one is stuck. + loops *stream.LoopWatch + // log is the server's logger (H2Config.Logger): the reason of each reset of a + // pool handler's wait is told at Debug level (rate, or an event loop stuck), so + // that a stream cut by it can be told from one cut by anything else. + log *slog.Logger } // peerMaxFrame returns the peer's currently-advertised SETTINGS_MAX_FRAME_SIZE, @@ -1027,28 +1035,35 @@ func (a *h2ResponseAdapter) finishBody(s *stream.Stream, rest []byte, maxFrame u // Stream.TryBufferOutbound). // // It gives up, sending nothing more, when the stream is reset or its -// connection closes. And it waits no longer than writeTimeout from now, the -// server's WriteTimeout, in all: then it resets the stream with -// INTERNAL_ERROR, as net/http does at a stream's write deadline, and returns -// an error that wraps os.ErrDeadlineExceeded. The bound is a timer, so it -// holds when the event loop cannot run: on the native engines a sync handler -// waiting for this one (a lock it holds across this write, a coalesced call it -// leads) holds its loop until this handler returns. +// connection closes. And it is bounded by the server's WriteTimeout as a +// minimum rate, the rule of every stream that waits for the peer's window +// (stream.AwaitSendWindowRate, celeris#1170): the client must let at least 64 KiB +// of the rest out per WriteTimeout window, with the carry-over, the connection's +// credit and the grace the rule gives; when it does not, the stream is reset with +// INTERNAL_ERROR, as net/http does at a stream's write deadline, and sendRest +// returns an error that wraps os.ErrDeadlineExceeded. Before celeris#1170 the +// bound was WriteTimeout in all from the first wait (#906), which cut a client +// that read at a healthy rate when the bytes ahead of this stream in its socket +// took longer. The bound is a timer of this handler's, so it holds when the event +// loop cannot run: on the native engines a sync handler waiting for this one (a +// lock it holds across this write, a coalesced call it leads) holds its loop until +// this handler returns; while a loop is stuck like that the wait is bounded by +// WriteTimeout in all, as before. func (a *h2ResponseAdapter) sendRest(s *stream.Stream, rest []byte, maxFrame uint32) error { - var deadline time.Time - if a.writeTimeout > 0 { - deadline = time.Now().Add(a.writeTimeout) - } + w := stream.SendWait{Loops: a.loops} + gen := s.Gen() + defer w.End(s, gen) for { - if err := s.AwaitSendWindow(deadline); err != nil { + if err := s.AwaitSendWindowRate(&w, len(rest)); err != nil { if !errors.Is(err, os.ErrDeadlineExceeded) { return nil // reset by the peer, or the connection closed } - a.resetStream(s, http2.ErrCodeInternal) - return fmt.Errorf("h2: stream %d reset: %d bytes of the response still waited for the peer's window at WriteTimeout (%v): %w", - s.ID, len(rest), a.writeTimeout, err) + a.resetStreamOnce(s, http2.ErrCodeInternal) + a.logWaitReset(s.ID, len(rest), &w) + return waitRateError(s.ID, len(rest), "the response", a.writeTimeout, &w, err) } if n := h2ReserveSend(a.manager, s, len(rest)); n > 0 { + a.manager.NoteWaitSent(s.ID, n) isEnd := n == len(rest) pooled := getH2FrameBuf() frameBuf := (*pooled)[:0] @@ -1066,6 +1081,44 @@ func (a *h2ResponseAdapter) sendRest(s *stream.Stream, rest []byte, maxFrame uin } } +// waitRateError is the error of a handler whose wait for the peer's window was +// judged not to make the minimum rate of WriteTimeout (wraps err, an +// os.ErrDeadlineExceeded). +func waitRateError(id uint32, left int, what string, wt time.Duration, w *stream.SendWait, err error) error { + if w.BlockedLoop() { + return fmt.Errorf("h2: stream %d reset: %d bytes of %s still waited for the peer's window after WriteTimeout (%v) while an event loop of the server was blocked: %w", + id, left, what, wt, err) + } + return fmt.Errorf("h2: stream %d reset: %d bytes of %s still waited for the peer's window and the client did not take the minimum rate of WriteTimeout (%v per window): %w", + id, left, what, wt, err) +} + +// logWaitReset tells why a pool handler's wait was reset (Debug). +func (a *h2ResponseAdapter) logWaitReset(id uint32, left int, w *stream.SendWait) { + if a.log == nil { + return + } + reason := "rate" + if w.BlockedLoop() { + reason = "loop-blocked" + } + a.log.Debug("h2: pool handler wait reset", "stream", id, "reason", reason, "left", left, "write_timeout", a.writeTimeout) +} + +// resetStreamOnce is resetStream for a wait that was judged: the stream is cancelled +// first, and the RST_STREAM goes only if this call cancelled it. A stream the peer +// reset between the verdict and here gets no second reset (RFC 9113 §5.1). +func (a *h2ResponseAdapter) resetStreamOnce(s *stream.Stream, code http2.ErrCode) { + if !s.CancelFirst() { + return + } + var payload [4]byte + binary.BigEndian.PutUint32(payload[:], uint32(code)) + pooled := getH2FrameBuf() + *pooled = appendH2Frame((*pooled)[:0], h2FrameRSTStream, 0, s.ID, payload[:]) + a.writeQueue.Enqueue(s.ID, pooled) +} + // resetStream queues a RST_STREAM for s behind the frames s has queued (all of // a stream's frames go to one shard, in order) and cancels s, so its handler's // context is done and nothing more of it is sent. A pool handler cannot use @@ -1094,7 +1147,7 @@ func (a *h2ResponseAdapter) resetStreamUse(s *stream.Stream, gen uint64, code ht return } defer s.UseUnlock() - a.resetStream(s, code) + a.resetStreamOnce(s, code) } // SendGoAway writes a GOAWAY frame via the shared writer. @@ -1576,18 +1629,21 @@ func (a *h2ResponseAdapter) WriteUse(s *stream.Stream, gen uint64, data []byte) if !onPool { return nil // taken whole: sent, buffered behind the DATA already sent, or a HEAD's, dropped } - // A pool goroutine: wait for the windows and send as they allow. The - // deadline starts when the call first has to wait, not before: a Write - // that finds the windows open reads no clock, and takes one lock. - var deadline time.Time + // A pool goroutine: wait for the windows and send as they allow. The wait + // is judged as a minimum rate of WriteTimeout (stream.AwaitSendWindowRateUse), + // from the moment the call first has to wait: a Write that finds the windows + // open reads no clock, and takes one lock. One wait per Write, not per + // response: a stream (SSE) is meant to outlast any one write. + w := stream.SendWait{Loops: a.loops} + defer w.End(s, gen) for len(data) > 0 { - if err := s.AwaitSendWindowUse(gen, a.writeTimeout, &deadline); err != nil { + if err := s.AwaitSendWindowRateUse(gen, &w, len(data)); err != nil { if !errors.Is(err, os.ErrDeadlineExceeded) { return err // reset by the peer, the connection closed, or the stream's use is over } a.resetStreamUse(s, gen, http2.ErrCodeInternal) - return fmt.Errorf("h2: stream %d reset: %d of %d bytes of a write still waited for the peer's window at WriteTimeout (%v): %w", - id, len(data), total, a.writeTimeout, err) + a.logWaitReset(id, len(data), &w) + return waitRateError(id, len(data), fmt.Sprintf("a write of %d bytes", total), a.writeTimeout, &w, err) } n, _, _, err := m.StreamWrite(s, gen, data, a.sendData) if err != nil { diff --git a/internal/engine/epoll/engine.go b/internal/engine/epoll/engine.go index d3559c72d..a00f6a261 100644 --- a/internal/engine/epoll/engine.go +++ b/internal/engine/epoll/engine.go @@ -24,6 +24,7 @@ import ( // Engine implements the epoll-based I/O engine. type Engine struct { loops []*Loop + loopWatch *stream.LoopWatch // the loops' stamps, for the pool handlers' wait (celeris#1170) cfg resource.Config handler stream.Handler addr atomic.Pointer[net.Addr] @@ -176,6 +177,8 @@ func (e *Engine) Listen(ctx context.Context) error { e.mu.Lock() e.loops = make([]*Loop, resolved.Workers) + e.loopWatch = stream.NewLoopWatch() + e.loopWatch.SetLogger(e.cfg.Logger) for i := range resolved.Workers { l := newLoop(i, cpus[i], e.handler, resolved, e.cfg, @@ -193,6 +196,8 @@ func (e *Engine) Listen(ctx context.Context) error { l.transplantStranded = &e.metrics.transplantStranded l.transplantAdoptRefused = &e.metrics.transplantAdoptRefused l.sweepCnt = &e.metrics.sweep + l.loopSlot = e.loopWatch.Slot() + l.h2cfg.LoopWatch = e.loopWatch l.pause = &e.pause l.drainBudget = &e.drainBudget l.shutdownBegun = &e.shutdownBegun diff --git a/internal/engine/epoll/loop.go b/internal/engine/epoll/loop.go index 1330b61a3..1c88277ba 100644 --- a/internal/engine/epoll/loop.go +++ b/internal/engine/epoll/loop.go @@ -295,7 +295,8 @@ type Loop struct { dirtyHead *connState // head of intrusive doubly-linked dirty list h2Conns []int // FDs of H2 connections (for write queue polling) h2cfg conn.H2Config - detachQueue []*connState // detached conns with pending writes (goroutine-safe via detachQMu) + loopSlot *stream.LoopSlot // stamped around each batch of events (celeris#1170) + detachQueue []*connState // detached conns with pending writes (goroutine-safe via detachQMu) detachQMu sync.Mutex detachQSpare []*connState // reuse slice to avoid alloc in drainDetachQueue detachQPending atomic.Int32 // 1 when detachQueue has entries; gates the hot-path drain @@ -670,6 +671,7 @@ func (l *Loop) run(ctx context.Context) { } } + l.loopSlot.Leave() // not in a batch while it waits n, err := unix.EpollWait(l.epollFD, l.events, timeoutMs) if err != nil { if err == unix.EINTR { @@ -695,6 +697,7 @@ func (l *Loop) run(ctx context.Context) { // is loop-thread-local, so no synchronization is needed. l.cachedNow = time.Now().UnixNano() now = l.cachedNow + l.loopSlot.Enter(now) // the batch below runs handlers: a stuck one shows (celeris#1170) } for i := range n { ev := &l.events[i] @@ -1000,6 +1003,7 @@ func (l *Loop) run(ctx context.Context) { if len(l.liveConns) == 0 { l.sweepRetract() } + l.loopSlot.Leave() // a parked loop is not in a batch: its last stamp must not age (celeris#1170) l.wakeMu.Lock() if !l.acceptPaused.Load() || l.hasDriverConns.Load() || l.adoptQPending.Load() != 0 || l.detachQPending.Load() != 0 { diff --git a/internal/engine/epoll/park_loopwatch_1170_linux_test.go b/internal/engine/epoll/park_loopwatch_1170_linux_test.go new file mode 100644 index 000000000..235c4e302 --- /dev/null +++ b/internal/engine/epoll/park_loopwatch_1170_linux_test.go @@ -0,0 +1,86 @@ +//go:build linux + +package epoll + +import ( + "context" + "net" + "net/http" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" + "github.com/goceleris/celeris/internal/resource" +) + +// celeris#1170: a loop that parks (an adaptive sub-engine demoted to standby, a +// paused engine with no connection) blocks on a Go channel, not in epoll_wait, and +// must not leave the stamp of its last batch behind: the pool handlers' wait reads +// every loop's stamp, and a stale one reads as a loop stuck for ever, which switches +// the rate rule off for the server (CI saw it on adaptive). After a request has been +// served and the loops have parked no loop reads as stuck, however long it parks. +func TestParkedLoopsAreNotStuck1170(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("pick port: %v", err) + } + addr := ln.Addr().String() + _ = ln.Close() + e, err := New(resource.Config{ + Addr: addr, + Protocol: engine.HTTP1, + Resources: resource.Resources{Workers: 2}, + DisableDeferAccept: true, // no pause linger: the loops park as PauseAccept is called + }, okHandler658{}) + if err != nil { + t.Fatalf("epoll engine unavailable: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- e.Listen(ctx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Error("engine did not stop within 5s") + } + }) + if err := awaitBound(e, done, 8*time.Second); err != nil { + t.Fatalf("epoll engine did not bind: %v", err) + } + // A batch on each loop. + cl := &http.Client{Transport: &http.Transport{DisableKeepAlives: true}} // no connection stays: the loops can park + for i := 0; i < 16; i++ { + resp, err := cl.Get("http://" + addr + "/") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + } + e.mu.Lock() + loops := append([]*Loop(nil), e.loops...) + watch := e.loopWatch + e.mu.Unlock() + if err := e.PauseAccept(); err != nil { + t.Fatalf("pause: %v", err) + } + for dl := time.Now().Add(3 * time.Second); ; time.Sleep(2 * time.Millisecond) { + parked := 0 + for _, l := range loops { + if l.suspended.Load() { + parked++ + } + } + if parked == len(loops) { + break + } + if time.Now().After(dl) { + t.Fatalf("PREMISE: the paused loops did not park") + } + } + time.Sleep(300 * time.Millisecond) + if watch.BlockedFor(time.Now().UnixNano(), int64(100*time.Millisecond)) { + t.Fatal("a parked loop reads as stuck in a batch: its stamp was left behind") + } +} diff --git a/internal/engine/iouring/engine.go b/internal/engine/iouring/engine.go index 75f5f38b3..761a73541 100644 --- a/internal/engine/iouring/engine.go +++ b/internal/engine/iouring/engine.go @@ -26,6 +26,7 @@ import ( // Engine implements the io_uring-based I/O engine. type Engine struct { workers []*Worker + loopWatch *stream.LoopWatch // the workers' stamps, for the pool handlers' wait (celeris#1170) tier TierStrategy profile engine.CapabilityProfile cfg resource.Config @@ -552,6 +553,8 @@ func (e *Engine) Listen(ctx context.Context) error { func (e *Engine) createWorkers(tier TierStrategy, cpus []int, resolved resource.ResolvedResources) ([]*Worker, error) { workers := make([]*Worker, len(cpus)) + e.loopWatch = stream.NewLoopWatch() + e.loopWatch.SetLogger(e.cfg.Logger) for i := range workers { w, err := newWorker(i, cpus[i], tier, e.handler, resolved, e.cfg, @@ -587,6 +590,8 @@ func (e *Engine) createWorkers(tier TierStrategy, cpus []int, w.pause = &e.pause // celeris#662 pause linger w.drainBudget = &e.drainBudget w.shutdownBegun = &e.shutdownBegun + w.loopSlot = e.loopWatch.Slot() + w.h2cfg.LoopWatch = e.loopWatch workers[i] = w } return workers, nil diff --git a/internal/engine/iouring/worker.go b/internal/engine/iouring/worker.go index cc506abad..c4383adfb 100644 --- a/internal/engine/iouring/worker.go +++ b/internal/engine/iouring/worker.go @@ -863,7 +863,8 @@ type Worker struct { wakeFD *wakefd.WakeFD h2PollArmed bool // true when POLL_ADD is active on the wakeup eventfd h2cfg conn.H2Config - emptyIters uint32 // consecutive iterations with zero CQEs (for adaptive timeout) + loopSlot *stream.LoopSlot // stamped around each batch of CQEs (celeris#1170) + emptyIters uint32 // consecutive iterations with zero CQEs (for adaptive timeout) detachQueue []*connState detachQMu sync.Mutex detachQSpare []*connState @@ -1722,6 +1723,7 @@ func (w *Worker) run(ctx context.Context) { } var cqHead, cqTail uint32 + w.loopSlot.Leave() // not in a batch while it waits if w.sqpoll { // SQPOLL path: kernel thread submits SQEs from the shared ring. // We never call Submit() — just clear the pending counter. @@ -1824,6 +1826,7 @@ func (w *Worker) run(ctx context.Context) { // per request. w.cachedNow = time.Now().UnixNano() now := w.cachedNow + w.loopSlot.Enter(now) // the batch below runs handlers: a stuck one shows (celeris#1170) for cqHead != cqTail { entry := w.ring.cqeAt(cqHead) // Inlined CQE dispatch — eliminates processCQE method call @@ -2112,6 +2115,7 @@ func (w *Worker) run(ctx context.Context) { if len(w.liveConns) == 0 { w.sweepRetract() } + w.loopSlot.Leave() // a parked worker is not in a batch: its last stamp must not age (celeris#1170) w.wakeMu.Lock() if !w.acceptPaused.Load() || w.driverActionPending.Load() != 0 || w.detachQPending.Load() != 0 { diff --git a/internal/protocol/h2/stream/manager.go b/internal/protocol/h2/stream/manager.go index 58c0f3529..02f924e77 100644 --- a/internal/protocol/h2/stream/manager.go +++ b/internal/protocol/h2/stream/manager.go @@ -51,9 +51,14 @@ type Manager struct { // holds buffered bytes, in stalls, guarded by stallMu; stallN is how many, for // a load on the paths that have none; connReleased is the bytes of DATA the // connection released while a stream was stalled (ReserveSendWindow). - stallMu sync.Mutex - stalls map[uint32]*stall - stallN atomic.Int32 + stallMu sync.Mutex + stalls map[uint32]*stall + stallN atomic.Int32 + // connHigh is the highest the connection window has been since it was last + // raised by the peer: connHigh less the window now is the bytes in flight on the + // connection, sent and not yet granted back (connInFlight, write_wait.go). Beside + // stallN, in the padding before connReleased: the Manager stays in its size class. + connHigh atomic.Int32 connReleased atomic.Int64 // retired holds the pool-handler streams whose handler has returned and @@ -101,6 +106,7 @@ func NewManager() *Manager { } m.hints = &m.ownHints m.initWin.Store(65535) + m.connHigh.Store(65535) return m } @@ -579,7 +585,7 @@ func (m *Manager) GetLastClientStreamID() uint32 { // UpdateConnectionWindow atomically updates the connection-level flow control window. func (m *Manager) UpdateConnectionWindow(delta int32) { - atomic.AddInt32(&m.connectionWindow, delta) + m.noteConnWindow(atomic.AddInt32(&m.connectionWindow, delta)) } // GetConnectionWindow atomically returns the current connection window size. @@ -612,12 +618,18 @@ func (m *Manager) CountActiveStreams() int { // outboundHeld is the bytes the streams hold buffered for the peer's window // (charged and refunded where the buffer fills, drains, and is dropped with its // stream), sendWindowWaiters the pool handlers waiting for a window in -// AwaitSendWindow. Atomic loads, no scan; safe from any goroutine. +// AwaitSendWindow. With a write deadline (WriteTimeout > 0) neither exempts the +// connection from nothing: a stream whose response waits for the peer's window, +// buffered or held by a pool handler, is bounded by its own minimum rate +// (write_deadline.go, write_wait.go), and the connection is exempt while it +// lives, as for a buffered tail since celeris#889 (A2 retired), and subject to +// the connection's clocks once it is gone (celeris#1170). Without one (WriteTimeout +// <= 0) a response that waits is not exempt, as before. Atomic loads, no scan; safe +// from any goroutine. func (m *Manager) ServerOwes() bool { return m.activeStreams.Load() > 0 && (m.reqTimeout > 0 || m.requestOpen.Load() == 0) && - (m.wrTimeout > 0 || m.outboundHeld.Load() == 0) && - m.sendWindowWaiters.Load() == 0 + (m.wrTimeout > 0 || (m.outboundHeld.Load() == 0 && m.sendWindowWaiters.Load() == 0)) } // updateActiveCount adjusts activeStreams, and requestOpen, atomically when a diff --git a/internal/protocol/h2/stream/outbound.go b/internal/protocol/h2/stream/outbound.go index 2bc12c7f9..276964f2e 100644 --- a/internal/protocol/h2/stream/outbound.go +++ b/internal/protocol/h2/stream/outbound.go @@ -39,19 +39,26 @@ import ( // 4 MiB matches the HTTP/1 per-connection backlog limit (the engines' // maxPendingBytes). // -// A waiting handler is blocked while the peer withholds window, for no -// longer than the server's WriteTimeout from its write: then its stream is -// reset (AwaitSendWindow's deadline, set by the conn layer). The connection's -// read and idle timeouts do not bound it: they measure the peer's silence, and -// a peer that withholds window but sends any frame, a PING, keeps them from -// firing. Whatever waits for a waiting handler waits with it: a request that -// needs a lock it holds across its write, a cache's coalesced fill it leads -// (celeris#913). On std that stalls only those requests. On epoll, io_uring -// and adaptive a sync handler that waits for it holds its event loop, and so -// every connection on that loop, until the deadline. With WriteTimeout -// disabled there is no deadline: the wait lasts until the peer grants window -// or the connection closes, and never ends if the loop that would see either -// is the one waiting. +// A waiting handler is blocked while the peer withholds window, bounded by the +// server's WriteTimeout as a minimum rate, not by a deadline: the peer must let +// at least 64 KiB of what it still has to send (or all of it, if less) out per +// WriteTimeout window, with the carry-over and the grace the rule of every stalled +// stream gives (AwaitSendWindowRate, write_wait.go; celeris#1170), or its stream is +// reset. It was an absolute deadline, WriteTimeout from the start of the wait +// (celeris#906), which cut a peer that read at a healthy rate when the bytes queued +// ahead of its stream took longer. The connection's read and idle timeouts do not +// bound it: they measure the peer's silence, and a peer that withholds window but +// sends any frame, a PING, keeps them from firing. Whatever waits for a waiting +// handler waits with it: a request that needs a lock it holds across its write, a +// cache's coalesced fill it leads (celeris#913). On std that stalls only those +// requests. On epoll, io_uring and adaptive a sync handler that waits for it holds +// its event loop, and so every connection on that loop, and with a rate a reader +// that takes its floor would hold it for the whole download: so while an event loop +// of the server has been in one batch for half a WriteTimeout (at least 250 ms) +// every waiting handler that has waited a WriteTimeout is bounded by WriteTimeout in +// all, as before (LoopWatch). With WriteTimeout disabled there is no bound: the wait +// lasts until the peer grants window or the connection closes, and never ends if the +// loop that would see either is the one waiting. const OutboundBudget = 4 << 20 // OutboundHeld returns how many bytes this connection's streams hold in their @@ -256,7 +263,9 @@ func (s *Stream) TryBufferOutbound(data []byte, endStream bool) bool { // AwaitSendWindow blocks the pool handler of s until both of s's send windows // are open, so some of its DATA can be sent (celeris#893), and then returns -// nil. It returns context.Canceled when s is cancelled: the peer reset it, or +// nil. This is the wait with an absolute deadline; the handlers of the conn layer +// use AwaitSendWindowRate (write_wait.go), whose bound is a rate, and this one is +// kept for the tests and benchmarks of the wake machinery they share. It returns context.Canceled when s is cancelled: the peer reset it, or // its connection closed (Manager.Close cancels every stream), and nothing // more is to be sent. It returns os.ErrDeadlineExceeded once deadline has // passed, whether or not the windows are open (a zero deadline never @@ -464,7 +473,7 @@ func (m *Manager) FlushOutbound(s *Stream, send OutboundSend) (finished bool) { finished = last && (s.OutboundEndStream || s.outboundAbandoned) send(s.ID, end, buf.Next(n)) m.refundOutbound(n) // celeris#893 - m.noteReleased(s.ID, n) + m.noteReleased(s, n) if buf.Len() == 0 { buf.Reset() } @@ -503,13 +512,13 @@ func (m *Manager) StreamWrite(s *Stream, gen uint64, data []byte, send OutboundS } if !s.RunsOnPool() { m.sendOrBufferLocked(s, data, send) - s.respWritten += int64(len(data)) return len(data), id, false, nil } n = int(m.ReserveSendWindow(s, clampWindowRequest(len(data)))) if n > 0 { send(id, false, data[:n]) - s.respWritten += int64(n) + s.noteWritten(n) + m.NoteWaitSent(id, n) // the credit of a wait that is in progress (write_wait.go) } return n, id, true, nil } @@ -531,6 +540,7 @@ func (m *Manager) sendOrBufferLocked(s *Stream, data []byte, send OutboundSend) n = int(m.ReserveSendWindow(s, clampWindowRequest(len(data)))) if n > 0 { send(s.ID, false, data[:n]) + s.noteWritten(n) // before the record's seed is taken: what is sent counts in it } } if n < len(data) { @@ -540,6 +550,7 @@ func (m *Manager) sendOrBufferLocked(s *Stream, data []byte, send OutboundSend) s.OutboundBuffer.Write(data[n:]) m.chargeOutbound(len(data) - n) m.noteBuffered(s, len(data)-n) + s.noteWritten(len(data) - n) } } diff --git a/internal/protocol/h2/stream/processor.go b/internal/protocol/h2/stream/processor.go index 7730a3a97..2c979983f 100644 --- a/internal/protocol/h2/stream/processor.go +++ b/internal/protocol/h2/stream/processor.go @@ -2098,6 +2098,7 @@ func (p *Processor) handleWindowUpdate(f *http2.WindowUpdateFrame) error { } //nolint:gosec // G115: safe conversion, newWin validated above if atomic.CompareAndSwapInt32(&p.manager.connectionWindow, oldWin, int32(newWin)) { + p.manager.noteConnWindow(int32(newWin)) //nolint:gosec // G115: validated above break } } @@ -2586,6 +2587,7 @@ func (p *Processor) HandleRawWindowUpdate(streamID uint32, payload []byte) error } //nolint:gosec // G115: safe conversion, newWin validated above if atomic.CompareAndSwapInt32(&p.manager.connectionWindow, oldWin, int32(newWin)) { + p.manager.noteConnWindow(int32(newWin)) //nolint:gosec // G115: validated above break } } diff --git a/internal/protocol/h2/stream/stream.go b/internal/protocol/h2/stream/stream.go index 4420d418b..a3d5b3504 100644 --- a/internal/protocol/h2/stream/stream.go +++ b/internal/protocol/h2/stream/stream.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "math" "strings" "sync" "sync/atomic" @@ -164,18 +165,25 @@ type Stream struct { h1Mode bool // single-threaded H1 stream; skip mutex in GetHeaders protoMajor uint8 // 0=infer from h1Mode, 1=HTTP/1.1, 2=HTTP/2 flags atomic.Uint32 - ctx atomic.Pointer[streamCtx] // this use's context, made on first ask (celeris#836) - parent context.Context // what this use's context ends with, bound when it is made (BindParentContext, celeris#889); unlocked, see there - phase Phase - CachedCtx any // per-connection cached context (avoids pool Get/Put per request) - OnDetach func() // called by Context.Detach to install write-thread safety - OnWSUpgrade func(func([]byte)) // called by Context to install WS data delivery callback - OnWSRawWrite func() func([]byte) // returns raw write fn (bypasses chunked encoding) - OnWSDetachClose func(func()) // installs callback for engine-side connection close - OnWSSetError func(func(error)) // installs error handler for engine-side I/O failures - OnWSReadPauser func() (func(), func()) // returns (pause, resume) callbacks for inbound backpressure - OnWSSetIdleDeadline func(int64) // sets the absolute idle deadline (Unix nanoseconds) - hdrBuf [16][2]string + // seedSince is how many bytes a StreamWriter has sent since the stream's last + // write-deadline record ended, plus one, saturating at MaxInt32; zero for a stream + // that has had no record (write_deadline.go, celeris#1170 review r2 F2): a later + // record's seed counts no more than this. Four bytes after flags, + // in the padding before the next pointer, where the Stream's size + // class has room. Guarded by mu. + seedSince int32 + ctx atomic.Pointer[streamCtx] // this use's context, made on first ask (celeris#836) + parent context.Context // what this use's context ends with, bound when it is made (BindParentContext, celeris#889); unlocked, see there + phase Phase + CachedCtx any // per-connection cached context (avoids pool Get/Put per request) + OnDetach func() // called by Context.Detach to install write-thread safety + OnWSUpgrade func(func([]byte)) // called by Context to install WS data delivery callback + OnWSRawWrite func() func([]byte) // returns raw write fn (bypasses chunked encoding) + OnWSDetachClose func(func()) // installs callback for engine-side connection close + OnWSSetError func(func(error)) // installs error handler for engine-side I/O failures + OnWSReadPauser func() (func(), func()) // returns (pause, resume) callbacks for inbound backpressure + OnWSSetIdleDeadline func(int64) // sets the absolute idle deadline (Unix nanoseconds) + hdrBuf [16][2]string // Last in the struct, so the fields the hot paths read keep their offsets. // respWritten is how many body bytes a StreamWriter has handed to this // use of the stream, sent or buffered (Manager.StreamWrite). Guarded by mu. @@ -429,13 +437,23 @@ func (s *Stream) useCtx() *streamCtx { // Cancel cancels the stream's context. func (s *Stream) Cancel() { + s.CancelFirst() +} + +// CancelFirst is Cancel, and reports whether this call was the one that +// cancelled the stream: false when it was already cancelled (the peer's +// RST_STREAM, a connection that closed, another reset). A caller that is about +// to send a RST_STREAM sends it only when it was first, so a stream the peer has +// reset gets no second reset after it (RFC 9113 §5.1). +func (s *Stream) CancelFirst() bool { old := s.flags.Or(flagCancelled) if old&flagCancelled != 0 { - return // already cancelled + return false // already cancelled } if c := s.ctx.Load(); c != nil { c.cancel() } + return true } // endCtx ends this use's context: it is cancelled, if it is not yet, and @@ -621,6 +639,7 @@ func (s *Stream) resetLocked() { s.OutboundEndStream = false s.outboundAbandoned = false s.respWritten = 0 + s.seedSince = 0 s.respDeclared.Store(0) s.headersSent.Store(false) s.EndStream = false @@ -853,6 +872,7 @@ func ResetH2StreamInline(s *Stream, id uint32) { s.Trailers = s.Trailers[:0] s.OutboundEndStream = false s.respWritten = 0 + s.seedSince = 0 s.respDeclared.Store(0) s.headersSent.Store(false) s.EndStream = false @@ -1103,3 +1123,18 @@ func (s *Stream) WriteLock() { s.writeMu.Lock() } // WriteUnlock releases the per-stream write lock. func (s *Stream) WriteUnlock() { s.writeMu.Unlock() } + +// noteWritten counts n body bytes a StreamWriter handed to this use of the stream +// (respWritten) and, once the stream has had a write-deadline record, since the last +// one ended (seedSince, which holds that count plus one; zero means no record has +// ended yet). The stream's lock must be held. +func (s *Stream) noteWritten(n int) { + s.respWritten += int64(n) + if s.seedSince > 0 { + s.seedSince = int32(min(int64(s.seedSince)+int64(n), math.MaxInt32)) //nolint:gosec // G115: clamped to MaxInt32 + } +} + +// recordEnded notes that a write-deadline record of the stream ended: its next +// record's seed counts the bytes sent from here. The stream's lock must be held. +func (s *Stream) recordEnded() { s.seedSince = 1 } diff --git a/internal/protocol/h2/stream/window_reader_rate_bench_1170_test.go b/internal/protocol/h2/stream/window_reader_rate_bench_1170_test.go new file mode 100644 index 000000000..43fb2e19a --- /dev/null +++ b/internal/protocol/h2/stream/window_reader_rate_bench_1170_test.go @@ -0,0 +1,88 @@ +//go:build unix + +package stream + +import ( + "context" + "fmt" + "os" + "runtime" + "sync" + "sync/atomic" + "syscall" + "testing" + "time" +) + +// BenchmarkConnWindowReaderRate1170 is BenchmarkConnWindowReader892 for the wait of +// celeris#1170: the same regime (stream windows open, the connection window the +// bottleneck, every waiter ends one wait per connection WINDOW_UPDATE and waits +// again), with the wait bounded as a minimum rate (rate=set: a record made by the +// first block, credited per send, looked up per wait) or not at all (rate=none: no +// WriteTimeout, no record, no timer). 892's deadline=set arm is the absolute +// deadline this replaces. Opt-in (CELERIS_893_WAITERS_BENCH=1). +func BenchmarkConnWindowReaderRate1170(b *testing.B) { + if os.Getenv("CELERIS_893_WAITERS_BENCH") == "" { + b.Skip("opt-in: CELERIS_893_WAITERS_BENCH=1 (celeris#892 item 28)") + } + for _, waiters := range []int{1, 8} { + for _, withRate := range []bool{true, false} { + name := "none" + if withRate { + name = "set" + } + b.Run(fmt.Sprintf("waiters=%d/rate=%s", waiters, name), func(b *testing.B) { + p := NewProcessor(HandlerFunc(func(context.Context, *Stream) error { return nil }), newTestFrameWriter(), newTestResponseWriter()) + m := p.GetManager() + m.connectionWindow = 0 + if withRate { + m.SetWriteTimeout(time.Hour) + } + var wg sync.WaitGroup + for i := 0; i < waiters; i++ { + s := m.CreateStream(uint32(2*i + 1)) + s.SetState(StateOpen) + s.flags.Or(flagAsyncRunning) + s.SetWindowSize(1 << 30) + wg.Add(1) + go func() { + defer wg.Done() + var w SendWait + defer w.End(s, s.Gen()) + for s.AwaitSendWindowRate(&w, 1<<30) == nil { + // What a writer does: reserve a chunk of 64; the connection + // grants 1, so every reservation is partial. + if n := m.ReserveSendWindow(s, 64); n > 0 { + m.NoteWaitSent(s.ID, int(n)) + } + } + }() + } + for until := time.Now().Add(5 * time.Second); int(m.sendWindowWaiters.Load()) < waiters; { + if time.Now().After(until) { + b.Fatalf("%d of %d waiters", m.sendWindowWaiters.Load(), waiters) + } + time.Sleep(time.Millisecond) + } + inc := []byte{0, 0, 0, 1} + var r0, r1 syscall.Rusage + _ = syscall.Getrusage(syscall.RUSAGE_SELF, &r0) + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := p.HandleRawWindowUpdate(0, inc); err != nil { + b.Fatal(err) + } + for atomic.LoadInt32(&m.connectionWindow) > 0 { + runtime.Gosched() + } + } + b.StopTimer() + _ = syscall.Getrusage(syscall.RUSAGE_SELF, &r1) + cpu := (r1.Utime.Nano() + r1.Stime.Nano()) - (r0.Utime.Nano() + r0.Stime.Nano()) + b.ReportMetric(float64(cpu)/float64(b.N), "cpu-ns/op") + m.Close() + wg.Wait() + }) + } + } +} diff --git a/internal/protocol/h2/stream/write_deadline.go b/internal/protocol/h2/stream/write_deadline.go index 58ef90f6a..b67d8eddf 100644 --- a/internal/protocol/h2/stream/write_deadline.go +++ b/internal/protocol/h2/stream/write_deadline.go @@ -84,6 +84,17 @@ const writeBankWindows = 4 // share may be held, on a connection that moves above the floor, before it is reset. const writeIdleWindows = 3 +// writeIdleCapWindows is the most idle windows a stream below its share may be +// held, on a connection that moves, when the connection has a long queue of bytes +// in flight (celeris#1170): the allowance is ceil(in flight / what the connection +// released in the window) windows, at least writeIdleWindows and at most this. A +// stream that waits behind its siblings' bytes in the client's socket is not +// idle for as long as the client takes to read what was sent ahead of it, and +// no longer. The cost is the bound of a stream that is truly paused or abandoned +// while the connection moves: 4 (the seed) + this + 1 windows. The one place the +// bound of the minimum rate is stretched; see the PR for #1170. +const writeIdleCapWindows = 12 + // maxTimeout bounds a deadline length so that now + timeout cannot overflow an // int64 of nanoseconds (a timeout of math.MaxInt64, the common way to write "never", // made every deadline due at once: review r7 F3): a hundred years. @@ -106,7 +117,12 @@ type stall struct { bank int // surplus released in earlier windows (and the seed), carried over idle int // consecutive windows held below its share on a connection above the floor connAt int64 // Manager.connReleased when the window began - connHeld int64 // Manager.outboundHeld when the window began + connHeld int64 // what the connection held (buffered and waiting) when the window began + idleMax int // idle windows allowed in the current idle run: the largest allowance it has seen + prevRel int64 // what the connection's other streams released lately: the largest of the windows before, halved each window + win int // windows judged + lastMove int // the window in which the connection's other streams last released a floor (0: never) + waiter bool // a pool handler blocked for window, not a buffer (write_wait.go): judged by its handler } // SetWriteTimeout sets the write deadline (the server's WriteTimeout): the length @@ -136,13 +152,19 @@ func (m *Manager) noteBuffered(s *Stream, n int) { id := s.ID m.stallMu.Lock() st := m.stalls[id] + if st != nil && st.waiter { + // A stream is a waiter or a buffer, never both (a pool handler that waits + // buffers nothing). If it ever were, the sweep judges it: the waiter's + // judge defers to a record that is not its own (judgeWait). + st.waiter = false + m.lowerWrDL(st.armedAt + int64(m.wrTimeout)) + } if st == nil { now := time.Now().UnixNano() // read only when a window begins, not for every append to a stalled stream if m.stalls == nil { m.stalls = make(map[uint32]*stall) } - seed := min(max(int(m.initWin.Load())-int(s.windowSize.Load()), 0), writeBankWindows*writeMinRelease) - st = &stall{armedAt: now, bank: seed, connAt: m.connReleased.Load(), connHeld: max(m.outboundHeld.Load(), int64(n))} + st = &stall{armedAt: now, bank: m.seedLocked(s), connAt: m.connReleased.Load(), connHeld: m.heldLocked(int64(n))} m.stalls[id] = st m.stallN.Store(int32(len(m.stalls))) m.lowerWrDL(now + int64(m.wrTimeout)) @@ -170,22 +192,68 @@ func (m *Manager) lowerWrDL(at int64) { // noteReleased records that n bytes of the stream's buffer were just sent // (FlushOutbound), with the stream's lock held: the stream's window is credited, // and the record ends with the buffer. -func (m *Manager) noteReleased(id uint32, n int) { +func (m *Manager) noteReleased(s *Stream, n int) { if m.wrTimeout <= 0 || n <= 0 { return } m.stallMu.Lock() - if st := m.stalls[id]; st != nil { + if st := m.stalls[s.ID]; st != nil && !st.waiter { st.released += n st.pending -= n if st.pending <= 0 { - delete(m.stalls, id) - m.stallN.Store(int32(len(m.stalls))) + m.delStallLocked(s.ID) + s.recordEnded() // s.mu is held: the next record's seed counts what is sent from here } } m.stallMu.Unlock() } +// seedLocked is the bank a stream's first window starts with: the bytes it had in +// flight (sent, not yet granted back: the peer's initial window less what is left +// of its own), at most writeBankWindows floors, and no more than the stream has +// sent since its previous record ended (Stream.seedSince). The last bound is for a streaming +// response whose Writes each wait: without it every record was seeded again with +// the same bytes the client never granted back, and a client that granted each +// Write's bytes once in five windows held the stream at a fifth of the floor for +// ever (celeris#1170 review r2 F2). A stream with no previous record has the first +// two bounds only. The stream's lock (for seedSince) must be held. +func (m *Manager) seedLocked(s *Stream) int { + seed := min(max(int(m.initWin.Load())-int(s.windowSize.Load()), 0), writeBankWindows*writeMinRelease) + if s.seedSince > 0 { + seed = min(seed, int(s.seedSince)-1) + } + return seed +} + +// heldLocked is what the connection holds waiting for the peer: the bytes its +// streams buffer and the bytes its pool handlers still have to send, and at +// least own (the stream judged). stallMu must be held. +func (m *Manager) heldLocked(own int64) int64 { + return max(m.outboundHeld.Load()+m.waitHeldLocked(), own) +} + +// waitHeldLocked is the bytes the pool handlers that wait for window still have to +// send: the sum of their records' pending. A scan of the stalled streams (at most +// the connection's concurrent streams), made where a window begins, not on any path +// that sends. stallMu must be held. +func (m *Manager) waitHeldLocked() int64 { + var n int64 + for _, st := range m.stalls { + if st.waiter { + n += int64(st.pending) + } + } + return n +} + +// delStallLocked ends the record of stream id. stallMu must be held. +func (m *Manager) delStallLocked(id uint32) { + if _, ok := m.stalls[id]; ok { + delete(m.stalls, id) + m.stallN.Store(int32(len(m.stalls))) + } +} + // dropStall ends the record of stream id, whose buffer was dropped or whose // stream has left the connection. A load of stallN when no stream is stalled. func (m *Manager) dropStall(id uint32) { @@ -193,10 +261,7 @@ func (m *Manager) dropStall(id uint32) { return } m.stallMu.Lock() - if _, ok := m.stalls[id]; ok { - delete(m.stalls, id) - m.stallN.Store(int32(len(m.stalls))) - } + m.delStallLocked(id) m.stallMu.Unlock() } @@ -245,7 +310,7 @@ func (m *Manager) ExpiredWrites(now int64, out []uint32) []uint32 { ids, sts := idBuf[:0], stBuf[:0] m.stallMu.Lock() for id, st := range m.stalls { - if st.armedAt+wt <= now { + if !st.waiter && st.armedAt+wt <= now { // a waiter is judged by its handler ids, sts = append(ids, id), append(sts, st) } } @@ -283,8 +348,11 @@ func (m *Manager) ExpiredWrites(now int64, out []uint32) []uint32 { if st == nil || st != sts[i] || st.armedAt+wt > now { continue } + if st.waiter { + continue // became a waiter's since it was found due + } if views[i]&viewGone != 0 { - delete(m.stalls, id) + m.delStallLocked(id) continue } if !m.holdsLocked(st, views[i]&viewOwn != 0 && connShut, now) { @@ -297,6 +365,9 @@ func (m *Manager) ExpiredWrites(now int64, out []uint32) []uint32 { } var next int64 for _, st := range m.stalls { + if st.waiter { + continue // the hint is the sweep's: a waiter's window is its handler's timer + } if at := st.armedAt + wt; next == 0 || at < next { next = at } @@ -318,6 +389,23 @@ func (m *Manager) holdsLocked(st *stall, starved bool, now int64) bool { connRel := m.connReleased.Load() - st.connAt connFloor := min(int64(writeMinRelease), st.connHeld) moving := connRel >= connFloor // the connection as a whole released its floor + // A connection that moved lately is still moving through windows in which it + // released nothing, for as many as the bank covers (review r10 F1): a client that + // returns connection window in large steps, as Go's Transport does (about 1 MiB + // every 1.05 s at 1 MB/s), leaves some windows with neither a grant nor a send, and + // a queue of siblings' bytes ahead of a stream can leave the connection without a + // release for a few windows between the first grants and the stream's own; that must + // not end a stream's idle run, or be its verdict. Such a window counts as an idle + // one (the allowance bounds them). What carries is what the OTHER streams + // released: the stream's own release must not shelter it in the windows after (a + // client that took its floor every fourth window would live for ever on it). + st.win++ + others := max(connRel-int64(st.released), 0) + if others >= connFloor { + st.lastMove = st.win + } + movingH := moving || (st.lastMove > 0 && st.win-st.lastMove < writeBankWindows) + rate := max(connRel, st.prevRel) // what the idle allowance divides the bytes in flight by hold := false switch { @@ -342,13 +430,23 @@ func (m *Manager) holdsLocked(st *stall, starved bool, now int64) bool { } st.bank = min(max(total-charge, 0), writeBankWindows*writeMinRelease) st.idle = 0 - case moving: + case movingH: share := int(connFloor) / (2 * max(len(m.stalls), 1)) if st.released > 0 && st.released >= share { hold = true st.idle = 0 - } else if st.idle++; st.idle <= writeIdleWindows { - hold = true + } else { + // The allowance of an idle run is the largest it has seen, within the cap + // (review r10 F3: a run that began in the tail branch had none, and one + // that began in a window the connection released a lot in had a small + // one for good). + if st.idle == 0 { + st.idleMax = 0 + } + st.idleMax = max(st.idleMax, m.idleAllowance(rate)) + if st.idle++; st.idle <= st.idleMax { + hold = true + } } st.bank = 0 case (st.held < writeMinRelease && st.released*2 >= floor) || st.connHeld < writeMinRelease: @@ -369,8 +467,9 @@ func (m *Manager) holdsLocked(st *stall, starved bool, now int64) bool { } } if hold { + st.prevRel = max(others, st.prevRel/2) st.armedAt, st.held, st.released = now, st.pending, 0 - st.connAt, st.connHeld = m.connReleased.Load(), max(m.outboundHeld.Load(), int64(st.pending)) + st.connAt, st.connHeld = m.connReleased.Load(), m.heldLocked(int64(st.pending)) } return hold } diff --git a/internal/protocol/h2/stream/write_deadline_889_test.go b/internal/protocol/h2/stream/write_deadline_889_test.go index 3d5a1bb6e..f891473d3 100644 --- a/internal/protocol/h2/stream/write_deadline_889_test.go +++ b/internal/protocol/h2/stream/write_deadline_889_test.go @@ -52,6 +52,10 @@ func wdSetup889(t testing.TB) (*Processor, *Manager, *rstRecorder889) { m.SetWriteTimeout(wt889) m.UpdateConnectionWindow(1 << 30) m.initWin.Store(0) // no seed: a stream's first window is judged on what it releases (the seed has its own tests) + // No bytes in flight at the connection level: the rule's tables are derived for + // a client that returns the connection window as it reads, with nothing queued + // ahead (the idle allowance of celeris#1170 has its own tests). + m.connHigh.Store(0) return p, m, rec } @@ -434,8 +438,9 @@ func TestExpireWriteWakesAPoolHandlerAsReset889(t *testing.T) { } // With a write deadline a conn whose only business is a stalled stream is -// exempt from the conn clocks (D2 resets the stream); without one, or with a -// pool handler waiting in AwaitSendWindow, it is not. +// exempt from the conn clocks (D2 resets the stream), and so is one whose pool +// handler waits for the window (its wait is judged by the same rule, celeris#1170: +// the connection's clocks apply once the stream is gone); without one, neither is. func TestServerOwesWithAWriteDeadline889(t *testing.T) { _, m, _ := wdSetup889(t) s := stalled889(m, 1, 5000) @@ -444,14 +449,19 @@ func TestServerOwesWithAWriteDeadline889(t *testing.T) { t.Fatal("a stalled stream does not exempt the conn although D2 bounds it") } m.sendWindowWaiters.Add(1) - if m.ServerOwes() { - t.Fatal("a pool handler waiting for the window exempts the conn: its bound is #906's, and the conn clock stays") + if !m.ServerOwes() { + t.Fatal("a pool handler waiting for the window does not exempt the conn although its wait is bounded by the rate (celeris#1170)") } m.sendWindowWaiters.Add(-1) m.SetWriteTimeout(0) if m.ServerOwes() { t.Fatal("without a write deadline a stalled stream exempts the conn (the old rule)") } + m.sendWindowWaiters.Add(1) + if m.ServerOwes() { + t.Fatal("without a write deadline a pool handler waiting for the window exempts the conn (the old rule)") + } + m.sendWindowWaiters.Add(-1) } // The common path pays nothing: reserving window allocates nothing and touches no diff --git a/internal/protocol/h2/stream/write_rate_889_test.go b/internal/protocol/h2/stream/write_rate_889_test.go index 9b26ebbee..cf3a3c92d 100644 --- a/internal/protocol/h2/stream/write_rate_889_test.go +++ b/internal/protocol/h2/stream/write_rate_889_test.go @@ -487,6 +487,7 @@ func TestWriteIdleCountRestarts889(t *testing.T) { got := firstReset889(m, 1, time.Now().UnixNano(), 20, func(w int) { if w == 3 { // the conn window is shut and b's own has room: starved; a sibling takes the grant atomic.StoreInt32(&m.connectionWindow, 0) + m.connHigh.Store(0) // nothing in flight at the connection level (the allowance has its own tests) b.SetWindowSize(1 << 20) a.SetWindowSize(70000) m.UpdateConnectionWindow(70000) @@ -563,8 +564,10 @@ func TestWriteConnFloorIsWhatTheConnHolds889(t *testing.T) { } } -// The bank does not survive a window held on the share: a stream below its floor -// on a connection that does not move is the verdict. +// The bank does not survive a window held on the share. (Until celeris#1170 this was +// shown by the verdict of the next window, a stream below its floor on a connection +// that stopped moving; the connection that moved in the window before now carries +// through one window that releases nothing, review r10 F1, so the books are looked at.) func TestWriteBankIsSpentByAShareWindow889(t *testing.T) { _, m, _ := wdSetup889(t) b := stalled889(m, 1, 64<<20) @@ -586,16 +589,35 @@ func TestWriteBankIsSpentByAShareWindow889(t *testing.T) { if out := windowEnd889(m, start, 2); len(out) != 0 { t.Fatalf("window 2: %v", out) } - // Window 3: b takes 40000 and nothing else moves: 40000 is under the floor and the - // connection is under its floor. The 30000 banked in window 1 do not make 70000. + // The 30000 banked in window 1 are gone: a window held on the share clears the bank. + m.stallMu.Lock() + bank := m.stalls[1].bank + m.stallMu.Unlock() + if bank != 0 { + t.Fatalf("the bank of window 1 survived a window held on the share: %d", bank) + } + // Window 3: b takes 40000 and nothing else moves: the connection moved in window 2, so + // one empty window is held (idle); the 30000 banked in window 1 would have made 70000. b.SetWindowSize(40000) m.FlushOutbound(b, discard889) - reset := false for _, id := range windowEnd889(m, start, 3) { - reset = reset || id == 1 + if id == 1 { + t.Fatal("window 3: stream 1 reset: one window in which the connection released nothing, after one in which it moved, is held") + } + } + // Windows 4 to 6: still nothing moves besides b, 40000 a window: under the floor, the + // connection under its floor for as many windows as the bank covers (it last moved in + // window 2): the verdict. + reset := false + for w := 4; w <= 6 && !reset; w++ { + b.SetWindowSize(40000) + m.FlushOutbound(b, discard889) + for _, id := range windowEnd889(m, start, w) { + reset = reset || id == 1 + } } if !reset { - t.Fatal("window 3: stream 1 held; the bank of window 1 survived a window held on the share") + t.Fatal("stream 1 held for four windows in which the connection released nothing") } } diff --git a/internal/protocol/h2/stream/write_wait.go b/internal/protocol/h2/stream/write_wait.go new file mode 100644 index 000000000..b8d9f8e44 --- /dev/null +++ b/internal/protocol/h2/stream/write_wait.go @@ -0,0 +1,461 @@ +package stream + +import ( + "context" + "log/slog" + "os" + "sync" + "sync/atomic" + "time" +) + +// The pool handler's wait for window as a minimum rate (celeris#1170, stage 3 of +// celeris#889). +// +// A pool handler whose response the connection's outbound budget could not buffer +// sends it itself (conn.h2ResponseAdapter.sendRest, WriteUse) and blocks in +// AwaitSendWindowRate while both windows are shut. It was bounded by an absolute +// deadline, WriteTimeout from the start of the wait (#906), whatever the client +// read, which cut a client that read at a healthy rate when the bytes ahead of its +// stream in the socket took longer than that. Now the wait is a record in the +// connection's stall books (write_deadline.go), judged by the rule of every +// stalled stream (holdsLocked: the floor per window, the bank and its seed, the +// connection's credit, the share and idle allowance, the tail grace), with these +// differences: +// +// - the record is the handler's: made by its first blocking call, credited with the +// bytes the handler sends (NoteWaitSent, or StreamWrite for a StreamWriter), +// ended with the call (SendWait.End) or with the stream (dropStall); +// - the judge is the handler, on a timer of its own armed to the end of the record's +// window, not the sweep, so that the bound holds when the event loop cannot run +// (a sync handler waiting for a lock this handler holds holds its loop, and a wait +// that only the loop's sweep could end would end never): the sweep skips waiter +// records, so one record has one judge and a stream is reset at most once; +// - the pool handler's own bytes (waitHeldLocked) count in what the connection holds, for +// the tail grace. +// +// While an event loop of the server is stuck in one batch for as long as a window +// allows (LoopWatch), a waiter that has waited a whole WriteTimeout is bounded as +// #906 bounded it, whatever the client reads: the wait would otherwise be able to +// hold a lock that loop's handlers wait for for as long as the download takes. + +// SendWait is the state of one wait of a pool handler for window: one response's +// sendRest, or one StreamWriter Write. The zero value is ready; End must be called +// when the call ends. +type SendWait struct { + m *Manager + id uint32 + start int64 // when the wait first blocked (wall clock ns) + made bool // the record was made (and not just re-made) + failed bool // judged and reset by the loop-blocked fallback + // Loops is the server's watch on its event loops, set by the caller (the conn + // layer's adapter has it from the engine): while one of them is stuck, the wait + // is bounded by WriteTimeout in all. Nil: never. + Loops *LoopWatch +} + +// BlockedLoop reports whether the wait ended because an event loop was stuck, +// not because the client was slow. +func (w *SendWait) BlockedLoop() bool { return w.failed } + +// End ends the wait's record, if it has one still, and notes where the stream's +// next record's seed counts from. s must be the stream the wait was on, in the use +// the wait began (gen, from Stream.Gen; the sendRest of a plain handler passes +// s.Gen()). +func (w *SendWait) End(s *Stream, gen uint64) { + m := w.m + if m == nil { + return + } + w.m = nil + s.mu.Lock() // the use cannot end, nor the object be reset, while it is held + if s.gen.Load() == gen && s.manager == m { + m.dropWait(w.id) + s.recordEnded() + } + s.mu.Unlock() +} + +// NoteWaitSent credits the waiting pool handler of stream id with n bytes it has +// sent: they are released in the record's window, and no longer to be sent. A +// no-op if the stream has no waiter record (WriteTimeout off, or no wait). +func (m *Manager) NoteWaitSent(id uint32, n int) { + if n <= 0 || m.stallN.Load() == 0 { + return + } + m.stallMu.Lock() + if st := m.stalls[id]; st != nil && st.waiter { + st.released += n + st.pending -= min(n, st.pending) + if st.pending <= 0 { + m.delStallLocked(id) + } + } + m.stallMu.Unlock() +} + +// noteConnWindow records that the peer raised the connection window to cw: the +// highest it has been is where the bytes in flight are measured from. +func (m *Manager) noteConnWindow(cw int32) { + for { + cur := m.connHigh.Load() + if cw <= cur || m.connHigh.CompareAndSwap(cur, cw) { + return + } + } +} + +// connInFlight is the bytes sent on the connection and not yet granted back at the +// connection level: the highest the connection window has been less what it is. A +// client that returns connection window as it reads (Go's Transport does, as its +// body is read) leaves exactly what is queued ahead of the next byte it reads; +// one that never does leaves the whole of what was sent, which is capped by the +// idle allowance's cap. Atomic loads only. +func (m *Manager) connInFlight() int64 { + return max(int64(m.connHigh.Load())-int64(atomic.LoadInt32(&m.connectionWindow)), 0) +} + +// idleAllowance is how many windows a stream below its share may be held on a +// connection that moves, given that the connection released connRel bytes in the +// window that ends: the windows the client needs to read what is in flight, at the +// rate it reads, between writeIdleWindows and writeIdleCapWindows. Atomic loads +// only (it is called with stallMu held). +func (m *Manager) idleAllowance(connRel int64) int { + if connRel <= 0 { + return writeIdleWindows + } + k := (m.connInFlight() + connRel - 1) / connRel + return int(min(max(k, writeIdleWindows), writeIdleCapWindows)) +} + +// beginWait makes the waiter record of s for a wait with rest bytes still to send, +// and returns the end of its window, or 0 if the call needs none (WriteTimeout off). +// It remakes a record that has gone while the stream lives, without a seed (a record +// that vanished is not evidence of bytes in flight). Called with no lock held. +func (m *Manager) beginWait(s *Stream, id uint32, w *SendWait, rest int, gen uint64) int64 { + wt := int64(m.wrTimeout) + if wt <= 0 { + return 0 + } + // The seed reads the stream's books under its lock, taken before stallMu. + seed := 0 + if !w.made { + if err := s.UseLock(gen); err == nil { + seed = m.seedLocked(s) + s.UseUnlock() + } + } + m.stallMu.Lock() + st := m.stalls[id] + if st == nil { + now := time.Now().UnixNano() // read only when a record is made, not for every wait + if m.stalls == nil { + m.stalls = make(map[uint32]*stall) + } + st = &stall{armedAt: now, bank: seed, waiter: true, held: rest, pending: rest, + connAt: m.connReleased.Load(), connHeld: m.heldLocked(int64(rest))} + m.stalls[id] = st + m.stallN.Store(int32(len(m.stalls))) + } + end := st.armedAt + wt + m.stallMu.Unlock() + if w.start == 0 { + w.start = time.Now().UnixNano() + } + w.m, w.id, w.made = m, id, true + return end +} + +// waitVerdict is what judgeWait decided. +type waitVerdict uint8 + +const ( + waitHeld waitVerdict = iota // go on waiting, the window was renewed or has not ended + waitCancelled // the stream is cancelled (or its record is not this wait's): do not reset it + waitExpired // reset it +) + +// judgeWait is the verdict at the end of the window of the waiter record of s: the +// rule of holdsLocked, or the fallback of a stuck loop. It returns the end of the +// window the wait goes on in when it holds. Called by the waiter's timer; takes +// stallMu only. +func (m *Manager) judgeWait(s *Stream, id uint32, w *SendWait, rest int) (waitVerdict, int64) { + if s.IsCancelled() { + m.dropWait(id) + return waitCancelled, 0 + } + wt := int64(m.wrTimeout) + now := time.Now().UnixNano() + if lw := w.Loops; lw != nil { + if idx, age, stuck := lw.stuck(now, loopStuckFor(wt)); !stuck { + lw.noteClear() + } else { + lw.noteStuck(idx, age, wt) + if now-w.start >= wt { + m.dropWait(id) + w.failed = true + return waitExpired, 0 + } + } + } + starved := s.windowSize.Load() > 0 && atomic.LoadInt32(&m.connectionWindow) <= 0 + m.stallMu.Lock() + defer m.stallMu.Unlock() + st := m.stalls[id] + switch { + case st == nil: + // The record has gone while the stream lives (two goroutines writing one + // stream, a removal racing the timer): remake it, without a seed, and go on: + // a wait nobody judges would hold the connection for ever. + if m.stalls == nil { + m.stalls = make(map[uint32]*stall) + } + st = &stall{armedAt: now, waiter: true, held: rest, pending: rest, + connAt: m.connReleased.Load(), connHeld: m.heldLocked(int64(rest))} + m.stalls[id] = st + m.stallN.Store(int32(len(m.stalls))) + return waitHeld, now + wt + case !st.waiter: + return waitHeld, max(st.armedAt+wt, now+wt) // the sweep's record: it judges (a time to look again, never in the past) + } + if st.armedAt+wt > now { + return waitHeld, st.armedAt + wt // not due: a grant renewed it, or the timer fired early + } + st.pending = rest // the books follow the handler's count + if m.holdsLocked(st, starved, now) { + return waitHeld, st.armedAt + wt + } + m.delStallLocked(id) + return waitExpired, 0 +} + +// dropWait ends the waiter record of id. +func (m *Manager) dropWait(id uint32) { + m.stallMu.Lock() + if st := m.stalls[id]; st != nil && st.waiter { + m.delStallLocked(id) + } + m.stallMu.Unlock() +} + +// AwaitSendWindowRate is AwaitSendWindow for a pool handler that has rest bytes to +// send, bounded by WriteTimeout as a minimum rate instead of an absolute deadline. +// It returns nil when both windows are open, context.Canceled when the stream is +// cancelled, and os.ErrDeadlineExceeded when the wait is judged not to make its +// rate (the caller then resets the stream), as AwaitSendWindow does at its +// deadline. w is the caller's SendWait for this response; call w.End when done. +// With WriteTimeout off there is no bound and no record. +func (s *Stream) AwaitSendWindowRate(w *SendWait, rest int) error { + m := s.manager + if m == nil { + return nil + } + if s.IsCancelled() { + return context.Canceled + } + if m.WindowsOpen(s) { + return nil + } + return s.awaitSendWindowRate(m, s.ID, s.Context().Done(), s.Gen(), w, rest) +} + +// AwaitSendWindowRateUse is AwaitSendWindowRate for a StreamWriter, which may +// outlive its handler (celeris#904): it returns ErrStreamEnded once gen is no +// longer the stream's use. +func (s *Stream) AwaitSendWindowRateUse(gen uint64, w *SendWait, rest int) error { + if err := s.UseLock(gen); err != nil { + return err + } + m := s.manager + if m == nil { + s.UseUnlock() + return ErrStreamEnded + } + if m.WindowsOpen(s) { + s.UseUnlock() + return nil + } + done, id := s.Context().Done(), s.ID + s.UseUnlock() + return s.awaitSendWindowRate(m, id, done, gen, w, rest) +} + +// awaitSendWindowRate is the wait of AwaitSendWindowRate and ...Use: it is +// awaitSendWindow (see its notes on lost wakeups) with the timer armed to the +// record's window end and, when it fires, a verdict instead of an expiry. +func (s *Stream) awaitSendWindowRate(m *Manager, id uint32, done <-chan struct{}, gen uint64, w *SendWait, rest int) error { + open := func() bool { return m.WindowsOpen(s) } + m.sendWindowWaiters.Add(1) + defer m.sendWindowWaiters.Add(-1) + sw := m.registerSendWaiter(s, id) + defer m.unregisterSendWaiter(sw) + + end := m.beginWait(s, id, w, rest, gen) + var expired <-chan time.Time + if end != 0 { + d := time.Until(time.Unix(0, end)) + if sw.timer == nil { + sw.timer = time.NewTimer(d) + } else { + sw.timer.Reset(d) + } + expired = sw.timer.C + defer sw.timer.Stop() + } + fire := func() error { // the window ended + if s.IsCancelled() { + return context.Canceled + } + if open() { + return nil // a grant that came as the window ended is not judged: it is sent, and credited, first + } + switch v, next := m.judgeWait(s, id, w, rest); v { + case waitCancelled: + return context.Canceled + case waitExpired: + return os.ErrDeadlineExceeded + default: + sw.timer.Reset(time.Until(time.Unix(0, next))) + return errStillWaiting + } + } + for { + if s.IsCancelled() { + return context.Canceled + } + select { + case <-expired: + if err := fire(); err != errStillWaiting { + return err + } + continue + default: + } + if open() { + return nil + } + select { + case <-sw.ch: + case <-expired: + if err := fire(); err != errStillWaiting { + return err + } + case <-done: + return context.Canceled + } + } +} + +// errStillWaiting is fire's "held": the wait goes on. +var errStillWaiting = &stillWaiting{} + +type stillWaiting struct{} + +func (*stillWaiting) Error() string { return "still waiting" } + +// A loop's watch ------------------------------------------------------------- + +// LoopWatch is a server's record of whether any of its event loops is stuck in one +// batch. Each loop has a slot it stamps when it starts a batch of events and clears +// before it waits for the next; BlockedFor reads them. Server-wide on purpose: +// the loops that a lock-holding pool handler's wait freezes are the ones whose +// handlers wait for the lock, on any connection, not the waiter's own. +type LoopWatch struct { + mu sync.Mutex + slots atomic.Pointer[[]*LoopSlot] + // log is where an episode of a stuck loop is told (SetLogger); episode is set from + // the first waiter that sees one until the first that does not. + log atomic.Pointer[slog.Logger] + episode atomic.Bool +} + +// SetLogger sets where the watch tells an episode of a stuck loop, once per episode +// (a Warn line): the rule is off for the server while it lasts, and nothing else says +// so. Nil: nothing is told. +func (w *LoopWatch) SetLogger(l *slog.Logger) { w.log.Store(l) } + +// noteStuck tells the operator, once per episode, that the rate rule of waiting pool +// handlers is suspended because loop idx has been in one batch for age ns. +func (w *LoopWatch) noteStuck(idx int, age, wt int64) { + if w.episode.CompareAndSwap(false, true) { + if l := w.log.Load(); l != nil { + l.Warn("h2: the minimum rate of a pool handler's wait for window is suspended: an event loop has been in one batch for half a WriteTimeout, so such waits are bounded by WriteTimeout in all until it ends", + "loop", idx, "stuck_for", time.Duration(age), "write_timeout", time.Duration(wt), "threshold", time.Duration(loopStuckFor(wt))) + } + } +} + +// noteClear ends the episode, if one was told. +func (w *LoopWatch) noteClear() { + if w.episode.Load() { + w.episode.Store(false) + } +} + +// LoopSlot is one loop's stamp. Padded: each loop writes its own twice a batch. +type LoopSlot struct { + since atomic.Int64 + _ [56]byte +} + +// NewLoopWatch returns an empty watch. +func NewLoopWatch() *LoopWatch { return &LoopWatch{} } + +// Slot adds a loop to the watch and returns its slot. +func (w *LoopWatch) Slot() *LoopSlot { + s := &LoopSlot{} + w.mu.Lock() + var cur []*LoopSlot + if p := w.slots.Load(); p != nil { + cur = *p + } + next := append(append(make([]*LoopSlot, 0, len(cur)+1), cur...), s) + w.slots.Store(&next) + w.mu.Unlock() + return s +} + +// Enter stamps the start of a batch at now (wall clock ns). +func (s *LoopSlot) Enter(now int64) { + if s != nil { + s.since.Store(now) + } +} + +// Leave clears the stamp: the loop is going to wait. +func (s *LoopSlot) Leave() { + if s != nil { + s.since.Store(0) + } +} + +// BlockedFor reports whether some loop has been in one batch since at least d +// before now. +func (w *LoopWatch) BlockedFor(now, d int64) bool { + _, _, ok := w.stuck(now, d) + return ok +} + +// stuck is BlockedFor with the first loop found stuck and for how long. +func (w *LoopWatch) stuck(now, d int64) (idx int, age int64, ok bool) { + p := w.slots.Load() + if p == nil { + return 0, 0, false + } + for i, s := range *p { + if t := s.since.Load(); t != 0 && now-t >= d { + return i, now - t, true + } + } + return 0, 0, false +} + +// minLoopStuck is the shortest a loop's batch is taken as stuck: ordinary batches +// of a loaded loop are not. +const minLoopStuck = 250 * time.Millisecond + +// loopStuckFor is how long a loop must have been in one batch to switch the +// fallback on: half a WriteTimeout, and at least minLoopStuck. +func loopStuckFor(wt int64) int64 { + return max(wt/2, int64(minLoopStuck)) +} diff --git a/internal/protocol/h2/stream/write_wait_1170_test.go b/internal/protocol/h2/stream/write_wait_1170_test.go new file mode 100644 index 000000000..ed574f203 --- /dev/null +++ b/internal/protocol/h2/stream/write_wait_1170_test.go @@ -0,0 +1,804 @@ +package stream + +import ( + "context" + "errors" + "log/slog" + "os" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/http2" +) + +// celeris#1170: a pool handler's wait for window is a record in the stall books, +// judged by the rule of every stalled stream, by the handler's own timer. + +// waitRig1170 is a manager with a short write window (real timers) and a pool +// stream whose send window is shut. +func waitRig1170(t *testing.T, wt time.Duration, initWin uint32) (*Manager, *Stream) { + t.Helper() + _, m, _ := wdSetup889(t) + m.SetWriteTimeout(wt) + m.initWin.Store(initWin) + s := openStream893(t, m, 1) + s.flags.Or(flagAsyncRunning) + s.SetWindowSize(0) + return m, s +} + +// run1170 waits for window as a pool handler with rest bytes to send and reports +// the error and how long it took. +func run1170(s *Stream, w *SendWait, rest int) (time.Duration, error) { + t0 := time.Now() + err := s.AwaitSendWindowRate(w, rest) + return time.Since(t0), err +} + +const wait1170 = 40 * time.Millisecond + +// waitHeld1170 is the bytes the waiters of m still have to send. +func waitHeld1170(m *Manager) int64 { + m.stallMu.Lock() + defer m.stallMu.Unlock() + return m.waitHeldLocked() +} + +// Nothing is granted: the wait is judged at the end of its window, like a +// buffered tail. With no bytes in flight that is the first window; with a 1 MiB +// window the seed covers four more (the bounds of TestWriteUnreadStreamBounds889). +func TestWaiterNothingGrantedIsResetAtTheBound1170(t *testing.T) { + for _, tc := range []struct { + initWin uint32 + windows int + }{{0, 1}, {65535, 1}, {1 << 20, 5}} { + m, s := waitRig1170(t, wait1170, tc.initWin) + var w SendWait + took, err := run1170(s, &w, 4<<20) + w.End(s, s.Gen()) + if !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("initWin %d: err %v, want os.ErrDeadlineExceeded", tc.initWin, err) + } + if lo, hi := time.Duration(tc.windows-1)*wait1170+wait1170/2, time.Duration(tc.windows)*wait1170+500*time.Millisecond; took < lo || took > hi { + t.Fatalf("initWin %d: reset after %v, want about %d windows of %v", tc.initWin, took, tc.windows, wait1170) + } + if w.BlockedLoop() { + t.Fatal("reported a blocked loop") + } + if n := m.stalledStreams(); n != 0 || waitHeld1170(m) != 0 { + t.Fatalf("after the verdict: %d records, waitHeld %d", n, waitHeld1170(m)) + } + } +} + +// A grant that comes inside the bank's cover is sent, and the wait goes on. +func TestWaiterGrantInsideTheSeedIsServed1170(t *testing.T) { + m, s := waitRig1170(t, wait1170, 1<<20) + go func() { + time.Sleep(3*wait1170 + wait1170/2) + s.SetWindowSize(100000) + m.notifyStreamWindow(s.ID) + }() + var w SendWait + took, err := run1170(s, &w, 4<<20) + if err != nil || took < 3*wait1170 { + t.Fatalf("err %v after %v, want nil after the grant at 3.5 windows", err, took) + } + if m.stalledStreams() != 1 || waitHeld1170(m) != 4<<20 { + t.Fatalf("while the handler sends: %d records, waitHeld %d, want its record", m.stalledStreams(), waitHeld1170(m)) + } + w.End(s, s.Gen()) + if m.stalledStreams() != 0 || waitHeld1170(m) != 0 { + t.Fatalf("after End: %d records, waitHeld %d", m.stalledStreams(), waitHeld1170(m)) + } +} + +// The connection window the peer raises moves the high mark connInFlight is measured +// from, through the frame the processor applies (not only through +// UpdateConnectionWindow). +func TestConnWindowUpdateFrameMovesTheHighMark1170(t *testing.T) { + p, m, _ := wdSetup889(t) + m.connHigh.Store(65535) + m.connectionWindow = 65535 + m.connectionWindow -= 40000 // 40000 sent + if got := m.connInFlight(); got != 40000 { + t.Fatalf("in flight %d, want 40000", got) + } + upd := func(n uint32) { + if err := p.handleWindowUpdate(&http2.WindowUpdateFrame{FrameHeader: http2.FrameHeader{}, Increment: n}); err != nil { + t.Fatal(err) + } + } + upd(30000) // the client reads and returns 30000 + if got := m.connInFlight(); got != 10000 { + t.Fatalf("after a return of 30000: %d in flight, want 10000", got) + } + upd(1 << 20) // the client opens the window far beyond what was sent + if got := m.connInFlight(); got != 0 || m.connHigh.Load() < 1<<20 { + t.Fatalf("after opening beyond: %d in flight, high %d", got, m.connHigh.Load()) + } +} + +// A grant that arrives as the window ends is not lost to the timer: the wait +// returns to send it (the window is open when the timer fires), it is not judged. +func TestWaiterGrantAtTheWindowEndIsNotJudged1170(t *testing.T) { + m, s := waitRig1170(t, wait1170, 0) + _ = m + // The window opens without a notice just before the timer: only the timer's own + // look at the windows can find it. + go func() { + time.Sleep(wait1170 / 2) + s.SetWindowSize(100000) + }() + var w SendWait + _, err := run1170(s, &w, 4<<20) + w.End(s, s.Gen()) + if err != nil { + t.Fatalf("err %v: a window that opened was judged as a stall", err) + } +} + +// A cancelled stream is never expired (the peer's RST_STREAM came first): the wait +// ends with context.Canceled, whichever the timer or the cancel reached first. +func TestWaiterCancelledStreamIsNotExpired1170(t *testing.T) { + m, s := waitRig1170(t, wait1170, 0) + _ = m + go func() { + time.Sleep(wait1170 / 4) + s.Cancel() + }() + var w SendWait + _, err := run1170(s, &w, 4<<20) + w.End(s, s.Gen()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err %v, want context.Canceled", err) + } + // And at the verdict itself: judgeWait of a cancelled stream does not expire. + m2, s2 := waitRig1170(t, wait1170, 0) + var w2 SendWait + m2.beginWait(s2, s2.ID, &w2, 1000, s2.Gen()) + s2.Cancel() + if v, _ := m2.judgeWait(s2, s2.ID, &w2, 1000); v != waitCancelled { + t.Fatalf("verdict %v for a cancelled stream, want waitCancelled", v) + } + if m2.stalledStreams() != 0 { + t.Fatal("the record of a cancelled stream stays") + } +} + +// The record of a live stream has gone (two goroutines on one stream, a removal that +// raced the timer): the wait is neither judged nor closed by anything else, so the +// judge remakes the record, without a seed, and goes on. +func TestWaiterMissingRecordIsRemade1170(t *testing.T) { + m, s := waitRig1170(t, wait1170, 1<<20) // a seed of four windows, which a remade record must not have + go func() { + time.Sleep(wait1170 / 4) + m.dropStall(s.ID) + }() + var w SendWait + took, err := run1170(s, &w, 4<<20) + w.End(s, s.Gen()) + if !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("err %v, want os.ErrDeadlineExceeded: a wait without a record was never judged", err) + } + if took > 3*wait1170+500*time.Millisecond { + t.Fatalf("reset after %v: the remade record took the seed (4 windows) or was not judged", took) + } + // Direct: no record and a live stream is remade, held, with no seed. + m2, s2 := waitRig1170(t, wait1170, 1<<20) + var w2 SendWait + if v, end := m2.judgeWait(s2, s2.ID, &w2, 5000); v != waitHeld || end == 0 { + t.Fatalf("no record: verdict %v end %d, want waitHeld and a window", v, end) + } + m2.stallMu.Lock() + st := m2.stalls[s2.ID] + ok := st != nil && st.waiter && st.bank == 0 && st.pending == 5000 && m2.waitHeldLocked() == 5000 + m2.stallMu.Unlock() + if !ok { + t.Fatalf("the remade record: %+v, waitHeld %d", st, waitHeld1170(m2)) + } +} + +// With WriteTimeout off there is no record and no timer: only a grant or the end of +// the stream ends the wait. +func TestWaiterWithoutAWriteTimeoutHasNoRecord1170(t *testing.T) { + m, s := waitRig1170(t, 0, 1<<20) + done := make(chan error, 1) + var w SendWait + go func() { done <- s.AwaitSendWindowRate(&w, 1<<20) }() + awaitWaiters889(t, m, 1) + if n := m.stalledStreams(); n != 0 { + t.Fatalf("%d records with WriteTimeout off", n) + } + select { + case err := <-done: + t.Fatalf("returned %v with nothing granted", err) + case <-time.After(150 * time.Millisecond): + } + s.Cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatalf("err %v, want context.Canceled", err) + } + w.End(s, s.Gen()) +} + +// The record's books: made by the first block, credited by NoteWaitSent, ended by +// End or by the bytes running out; waitHeld follows; the sweep does not see it. +func TestWaiterRecordBooks1170(t *testing.T) { + m, s := waitRig1170(t, time.Hour, 0) + other := stalled889(m, 3, 200000) // a buffered stream beside it (not a tail: the grace would hold it) + var w SendWait + if end := m.beginWait(s, s.ID, &w, 10000, s.Gen()); end == 0 { + t.Fatal("no window") + } + if waitHeld1170(m) != 10000 || m.stalledStreams() != 2 { + t.Fatalf("waitHeld %d, %d records", waitHeld1170(m), m.stalledStreams()) + } + // The connection holds the buffer and the waiter's bytes. + m.stallMu.Lock() + h := m.heldLocked(0) + m.stallMu.Unlock() + if h != 210000 { + t.Fatalf("the connection holds %d, want 210000", h) + } + m.NoteWaitSent(s.ID, 4000) + m.stallMu.Lock() + st := m.stalls[s.ID] + got := st.released == 4000 && st.pending == 6000 && m.waitHeldLocked() == 6000 + m.stallMu.Unlock() + if !got { + t.Fatalf("after a credit: %+v waitHeld %d", st, waitHeld1170(m)) + } + m.NoteWaitSent(other.ID, 100) // a buffered tail is credited by the flush, not by this + m.stallMu.Lock() + rel := m.stalls[other.ID].released + m.stallMu.Unlock() + if rel != 0 { + t.Fatalf("NoteWaitSent credited a buffered record: %d", rel) + } + // The sweep skips it, in the verdicts and in the hint. + if out := m.ExpiredWrites(time.Now().UnixNano()+int64(2*time.Hour), make([]uint32, 0, 8)); len(out) != 1 || out[0] != other.ID { + t.Fatalf("the sweep expired %v, want only the buffered stream %d", out, other.ID) + } + m.dropStall(other.ID) + if m.hints.WrDL.Load() != 0 { + _ = m.ExpiredWrites(time.Now().UnixNano(), make([]uint32, 0, 8)) + } + if h := m.NextWriteDeadline(); h != 0 { + t.Fatalf("hint %d with only a waiter: the sweep would spin", h) + } + m.NoteWaitSent(s.ID, 6000) // the rest goes out: the record ends with it + if m.stalledStreams() != 0 || waitHeld1170(m) != 0 { + t.Fatalf("after the last bytes: %d records, waitHeld %d", m.stalledStreams(), waitHeld1170(m)) + } + // End after the record is gone is a no-op, and a second End too. + w.End(s, s.Gen()) + w.End(s, s.Gen()) + // dropAllStalls zeroes waitHeld. + var w2 SendWait + m.beginWait(s, s.ID, &w2, 777, s.Gen()) + m.dropAllStalls() + if waitHeld1170(m) != 0 || m.stalledStreams() != 0 { + t.Fatalf("dropAllStalls: waitHeld %d, %d records", waitHeld1170(m), m.stalledStreams()) + } +} + +// The record follows the handler's own count of what is left, whatever the credits +// said: a send that was not credited does not leave the books ahead of the handler. +func TestWaiterBooksFollowTheHandlersCount1170(t *testing.T) { + m, s := waitRig1170(t, time.Millisecond, 1<<20) // a seed: the verdict holds + var w SendWait + m.beginWait(s, s.ID, &w, 10000, s.Gen()) + time.Sleep(5 * time.Millisecond) // the window ended: due + // The handler sent 3000 and the credit was not made; it tells its rest to the judge. + m.judgeWait(s, s.ID, &w, 7000) + m.stallMu.Lock() + st := m.stalls[s.ID] + got := st != nil && st.pending == 7000 && m.waitHeldLocked() == 7000 + m.stallMu.Unlock() + if !got { + t.Fatalf("the record after the judge: %+v waitHeld %d, want pending 7000", st, waitHeld1170(m)) + } +} + +// A stream is a waiter or a buffer, never both; if it ever were, the sweep judges +// the record and the waiter's judge defers to it. +func TestWaiterAndBufferNeverHaveTwoJudges1170(t *testing.T) { + m, s := waitRig1170(t, wait1170, 0) + var w SendWait + m.beginWait(s, s.ID, &w, 300000, s.Gen()) + s.BufferOutbound(make([]byte, 200000), true) // noteBuffered on a waiter's stream + m.stallMu.Lock() + st := m.stalls[s.ID] + conv := st != nil && !st.waiter && m.waitHeldLocked() == 0 && st.pending == 500000 + m.stallMu.Unlock() + if !conv { + t.Fatalf("the record after a buffer joined: %+v waitHeld %d", st, waitHeld1170(m)) + } + if v, _ := m.judgeWait(s, s.ID, &w, 300000); v != waitHeld { + t.Fatalf("the waiter's judge decided %v on the sweep's record", v) + } + if out := m.ExpiredWrites(time.Now().UnixNano()+int64(wait1170)*2, make([]uint32, 0, 4)); len(out) != 1 { + t.Fatalf("the sweep expired %v, want the stream (it judges now)", out) + } +} + +// The seed counts no more than what was sent since the stream's last record ended +// (review r2 F2): a dribbler that grants each Write's bytes late got a fresh seed of +// everything in flight for every Write. +func TestWriteSeedCountsOnlyWhatWasSentSinceTheLastRecord1170(t *testing.T) { + _, m, _ := wdSetup889(t) + m.initWin.Store(1 << 20) // 256 KiB of seed for a first record + s := m.CreateStream(1) + s.SetState(StateOpen) + s.SetWindowSize(0) + seed := func() int { + s.mu.Lock() + defer s.mu.Unlock() + return m.seedLocked(s) + } + if got := seed(); got != 256<<10 { + t.Fatalf("first record: seed %d, want the cap (256 KiB)", got) + } + s.mu.Lock() + s.recordEnded() + s.mu.Unlock() + if got := seed(); got != 0 { + t.Fatalf("nothing sent since the last record: seed %d, want 0", got) + } + s.mu.Lock() + s.noteWritten(5000) + s.mu.Unlock() + if got := seed(); got != 5000 { + t.Fatalf("5000 sent since: seed %d, want 5000", got) + } + s.mu.Lock() + s.noteWritten(1 << 30) + s.mu.Unlock() + if got := seed(); got != 256<<10 { + t.Fatalf("a lot sent since: seed %d, want the cap", got) + } +} + +// The detached streaming path of D2 (inline StreamWriter): each Write that buffers +// ends a record when the buffer drains, and the next one's seed counts only what +// the Write sent before it buffered. +func TestInlineStreamWritesDoNotReSeed1170(t *testing.T) { + _, m, _ := wdSetup889(t) + m.initWin.Store(1 << 20) + s := m.CreateStream(1) + s.SetState(StateOpen) + s.SetWindowSize(0) + gen := s.Gen() + bank := func() int { + m.stallMu.Lock() + defer m.stallMu.Unlock() + if st := m.stalls[1]; st != nil { + return st.bank + } + return -1 + } + if _, _, _, err := m.StreamWrite(s, gen, make([]byte, 100000), discard889); err != nil { + t.Fatal(err) + } + if got := bank(); got != 256<<10 { + t.Fatalf("first Write: seed %d, want 256 KiB", got) + } + s.SetWindowSize(100000) // the client grants this Write's bytes: the buffer drains, the record ends + m.FlushOutbound(s, discard889) + if got := bank(); got != -1 { + t.Fatalf("the record stays after the buffer drained: bank %d", got) + } + s.SetWindowSize(0) + if _, _, _, err := m.StreamWrite(s, gen, make([]byte, 100000), discard889); err != nil { + t.Fatal(err) + } + if got := bank(); got != 0 { + t.Fatalf("second Write, which sent nothing before it buffered: seed %d, want 0", got) + } +} + +// The idle allowance (option B of celeris#1170): ceil(in flight / what the +// connection released in the window), between writeIdleWindows and the cap. +func TestIdleAllowance1170(t *testing.T) { + _, m, _ := wdSetup889(t) + m.connHigh.Store(int32(1 << 30)) + atomicStoreCW := func(cw int32) { m.connectionWindow = cw } + for _, tc := range []struct { + inflight int64 + rel int64 + want int + }{ + {0, 100000, writeIdleWindows}, + {300000, 100000, writeIdleWindows}, + {300001, 100000, 4}, + {1000000, 100000, 10}, + {1200000, 100000, 12}, + {1200001, 100000, writeIdleCapWindows}, + {1600000, 100000, writeIdleCapWindows}, + {1 << 28, 1000, writeIdleCapWindows}, + {1 << 20, 0, writeIdleWindows}, // a connection that moves released something; 0 is no basis for a quotient + } { + atomicStoreCW(int32(1<<30 - tc.inflight)) + if got := m.idleAllowance(tc.rel); got != tc.want { + t.Errorf("in flight %d, released %d: allowance %d, want %d", tc.inflight, tc.rel, got, tc.want) + } + } + if writeIdleCapWindows != 12 { + t.Errorf("the cap is %d, decided 12", writeIdleCapWindows) + } +} + +// connInFlight follows the highest the window has been, and does not go negative. +func TestConnInFlight1170(t *testing.T) { + m := NewManager() + if got := m.connInFlight(); got != 0 { + t.Fatalf("new: %d", got) + } + m.UpdateConnectionWindow(1 << 20) // the peer opens the window + atomicSub := func(n int32) { m.connectionWindow -= n } + atomicSub(300000) + if got := m.connInFlight(); got != 300000 { + t.Fatalf("after sending 300000: %d", got) + } + m.UpdateConnectionWindow(100000) // the peer returns some (the high mark follows only above it) + if got := m.connInFlight(); got != 200000 { + t.Fatalf("after a grant of 100000: %d", got) + } + m.UpdateConnectionWindow(1 << 20) // opens more than it consumed: the new high + if got := m.connInFlight(); got != 0 { + t.Fatalf("after opening beyond the high mark: %d", got) + } +} + +// The bound of an unread stream on a connection that moves, with the allowance at +// its cap (a long queue ahead in the socket): the seed spends 1+k windows, the +// allowance holds it 16 more, the next is the verdict. The documented 21 for a +// window of 256 KiB or more, 17 for the default one. Without a queue ahead the +// bounds are those of TestWriteUnreadStreamBounds889. +func TestUnreadStreamBoundWithAQueueAhead1170(t *testing.T) { + for _, tc := range []struct { + initWin uint32 + inflight int64 + want int + }{ + {65535, 100 << 20, 1 + writeIdleCapWindows}, + {1 << 20, 100 << 20, 4 + writeIdleCapWindows + 1}, + {1 << 20, 0, 8}, + {65535, 0, 4}, + {1 << 20, 700000, 4 + 7 + 1}, // ceil(700000/ 100000-ish): see rate below + } { + _, m, _ := wdSetup889(t) + m.initWin.Store(tc.initWin) + m.connHigh.Store(int32(1<<30 + 65535)) // wdSetup's window, as the peer opened it + m.connectionWindow = int32(1<<30+65535) - int32(tc.inflight) + stalled889(m, 1, 8<<20) + a := m.CreateStream(3) + a.SetState(StateOpen) + a.SetWindowSize(0) + a.BufferOutbound(make([]byte, 64<<20), true) + w := firstReset889(m, 1, time.Now().UnixNano(), 60, func(int) { + a.SetWindowSize(100000) // the sibling takes 100000 a window: the connection moves + m.FlushOutbound(a, discard889) + m.connectionWindow = int32(1<<30+65535) - int32(tc.inflight) // the client returns what it reads + }) + if w != tc.want { + t.Errorf("window %d, in flight %d: reset in window %d, want %d", tc.initWin, tc.inflight, w, tc.want) + } + } +} + +// The allowance is fixed when the idle run begins: a queue that drains while the +// stream is idle does not shorten the run. +func TestIdleAllowanceIsFixedWhenTheRunBegins1170(t *testing.T) { + _, m, _ := wdSetup889(t) + m.initWin.Store(65535) + m.connHigh.Store(int32(1<<30 + 65535)) + m.connectionWindow = int32(1<<30+65535) - 100<<20 + stalled889(m, 1, 8<<20) + a := m.CreateStream(3) + a.SetState(StateOpen) + a.SetWindowSize(0) + a.BufferOutbound(make([]byte, 64<<20), true) + w := firstReset889(m, 1, time.Now().UnixNano(), 60, func(win int) { + a.SetWindowSize(100000) + m.FlushOutbound(a, discard889) + if win > 1 { + m.connHigh.Store(0) // the queue is gone after the first window: a re-derived allowance would be 3 + } + }) + if w != 1+writeIdleCapWindows { + t.Fatalf("reset in window %d, want %d (the allowance of the first idle window, 16)", w, 1+writeIdleCapWindows) + } +} + +// While an event loop is stuck in one batch for half a WriteTimeout (and at least +// 250 ms) a waiter that has waited a whole WriteTimeout is reset at its next +// judgement, whatever its seed and credit say; just under the threshold it is not. +func TestWaiterLoopBlockedFallback1170(t *testing.T) { + for _, tc := range []struct { + name string + stuck time.Duration // how long the loop has been in its batch when the waiter first blocks + blocks bool + }{ + {"just under the threshold", 200 * time.Millisecond, false}, + {"just over the threshold", 400 * time.Millisecond, true}, + {"no loop stuck", 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + _, s := waitRig1170(t, wait1170, 1<<20) // the seed would hold it for 5 windows + lw := NewLoopWatch() + slot := lw.Slot() + if tc.stuck > 0 { + // A loop that is always this far into a batch (it re-enters as time passes). + quit := make(chan struct{}) + defer close(quit) + stamp := func() { slot.Enter(time.Now().Add(-tc.stuck).UnixNano()) } + stamp() + go func() { + for { + select { + case <-quit: + return + case <-time.After(2 * time.Millisecond): + stamp() + } + } + }() + } + w := SendWait{Loops: lw} + took, err := run1170(s, &w, 4<<20) + w.End(s, s.Gen()) + if !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("err %v", err) + } + if tc.blocks { + if !w.BlockedLoop() || took > 3*wait1170+500*time.Millisecond { + t.Fatalf("blocked=%v after %v: want the fallback at the first window", w.BlockedLoop(), took) + } + } else if w.BlockedLoop() || took < 4*wait1170 { + t.Fatalf("blocked=%v after %v: the rule's bound (5 windows) applies", w.BlockedLoop(), took) + } + }) + } +} + +// An idle standby loop is not a stuck one: it stamps nothing (it waits), and a loop +// that left its batch clears its stamp. +func TestLoopWatch1170(t *testing.T) { + lw := NewLoopWatch() + a, b := lw.Slot(), lw.Slot() + now := time.Now().UnixNano() + if lw.BlockedFor(now, 1) { + t.Fatal("idle loops read as stuck") + } + a.Enter(now - int64(time.Second)) + if !lw.BlockedFor(now, int64(500*time.Millisecond)) || lw.BlockedFor(now, int64(2*time.Second)) { + t.Fatal("a batch of one second: stuck for 500 ms, not for 2 s") + } + a.Leave() + b.Enter(now) + if lw.BlockedFor(now, int64(time.Millisecond)) { + t.Fatal("a batch that has just begun, or one that ended, reads as stuck") + } + var nilSlot *LoopSlot + nilSlot.Enter(1) + nilSlot.Leave() // a loop without a slot (a test) is not a crash + if loopStuckFor(int64(time.Second)) != int64(500*time.Millisecond) || loopStuckFor(int64(100*time.Millisecond)) != int64(250*time.Millisecond) { + t.Fatal("the threshold is half a WriteTimeout, at least 250 ms") + } +} + +// An idle run that begins in the tail branch has an allowance too (review r10 F3): a +// tail window, then a window in which a sibling's last bytes move the connection, was +// reset at once (the allowance was only set by the moving branch), where the rule before +// celeris#1170 held it. +func TestIdleRunThatBeginsInATailKeepsAnAllowance1170(t *testing.T) { + m := NewManager() + m.SetWriteTimeout(time.Second) + a := &stall{held: 10, pending: 10, connHeld: 20} + b := &stall{held: 10, pending: 10, connHeld: 20} + m.stalls = map[uint32]*stall{1: a, 3: b} + m.stallN.Store(2) + now := time.Now().UnixNano() + m.stallMu.Lock() + defer m.stallMu.Unlock() + h1 := m.holdsLocked(a, false, now) + _ = m.holdsLocked(b, false, now) + m.connReleased.Add(10) // b's bytes go out in window 2 + b.released, b.pending = 10, 0 + h2 := m.holdsLocked(a, false, now+int64(time.Second)) + if !h1 || !h2 { + t.Fatalf("window 1 hold=%v, window 2 hold=%v (idle %d, allowance %d): a tail that waited one window and then saw the connection move was reset", h1, h2, a.idle, a.idleMax) + } +} + +// The allowance of an idle run is the largest the run has seen, within the cap +// (review r10 F3): a fast first window (the connection released a lot at once) must not +// fix a small allowance for the whole run. +func TestIdleAllowanceIsTheLargestOfTheRun1170(t *testing.T) { + run := func(r1, r2 int64) int { + m := NewManager() + m.SetWriteTimeout(time.Second) + m.connHigh.Store(16 << 20) + m.connectionWindow = 0 // 16 MiB in flight + st := &stall{held: 1 << 20, pending: 1 << 20, connHeld: 8 << 20} + other := &stall{held: 1 << 20, pending: 1 << 20, connHeld: 8 << 20} + m.stalls = map[uint32]*stall{1: st, 3: other} + m.stallN.Store(2) + now := time.Now().UnixNano() + m.stallMu.Lock() + defer m.stallMu.Unlock() + for w := 1; w <= 60; w++ { + r := r2 + if w == 1 { + r = r1 + } + m.connReleased.Add(r) + now += int64(time.Second) + if !m.holdsLocked(st, false, now) { + return w + } + } + return 0 + } + steady := run(1<<20, 1<<20) + fast := run(4<<20, 1<<20) // the run begins in a window in which 4 MiB were released + if fast != steady { + t.Fatalf("reset in window %d after a fast first window, %d when steady: the first window fixed the allowance", fast, steady) + } +} + +// A connection that moved lately is still moving through the windows the bank covers +// (four) in which it released nothing (review r10 F1: Go's Transport returns connection +// window in about 1 MiB steps, so some 1 s windows at 1 MB/s have neither a grant nor a +// send), and the verdict is the first window after them. What carries is what the OTHER +// streams released: a stream's own release does not shelter it in the next window. +func TestEmptyWindowsOfAMovingConnectionAreHeldUpToTheHorizon1170(t *testing.T) { + // Stream 1 takes 20000 a window (over its share, under its floor) beside stream 3, + // which moves the connection in the windows moves(w) says. + run := func(moves func(w int) bool) int { + _, m, _ := wdSetup889(t) + m.initWin.Store(0) + a := stalled889(m, 1, 64<<20) + b := stalled889(m, 3, 64<<20) + start := time.Now().UnixNano() + return firstReset889(m, 1, start, 40, func(w int) { + a.SetWindowSize(20000) + m.FlushOutbound(a, discard889) + if moves(w) { + b.SetWindowSize(70000) + m.FlushOutbound(b, discard889) + } + }) + } + // The sibling moves in windows 1 to 5, then stops: stream 1, which takes its share, + // is held through the windows the bank covers (6, 7 and 8: the horizon) and reset in + // the fourth. + if got := run(func(w int) bool { return w <= 5 }); got != 5+writeBankWindows { + t.Errorf("the sibling stops after window 5: reset in window %d, want %d (%d empty windows are held)", got, 5+writeBankWindows, writeBankWindows-1) + } + // Empty windows in the middle of a moving connection, up to the horizon, end nothing. + if got := run(func(w int) bool { return w < 4 || w > 6 }); got != 0 { + t.Errorf("three empty windows in the middle: reset in window %d, want never", got) + } + // One more does. + if got := run(func(w int) bool { return w < 4 || w > 7 }); got != 7 { + t.Errorf("four empty windows in a row: reset in window %d, want 7", got) + } +} + +// logCount1170 is a slog handler that counts the records it gets. +type logCount1170 struct { + n atomic.Int32 + last atomic.Value +} + +func (h *logCount1170) Enabled(context.Context, slog.Level) bool { return true } +func (h *logCount1170) Handle(_ context.Context, r slog.Record) error { + h.n.Add(1) + h.last.Store(r.Message) + return nil +} +func (h *logCount1170) WithAttrs([]slog.Attr) slog.Handler { return h } +func (h *logCount1170) WithGroup(string) slog.Handler { return h } + +// The fallback tells the operator, once per episode: the rate rule is off for the +// server while a loop is stuck, and nothing else says so (review r10 M1). +func TestLoopWatchTellsAnEpisodeOnce1170(t *testing.T) { + var h logCount1170 + lw := NewLoopWatch() + lw.SetLogger(slog.New(&h)) + slot := lw.Slot() + wt := int64(time.Second) + now := time.Now().UnixNano() + for i := 0; i < 5; i++ { // five judgements that see the loop stuck: one line + slot.Enter(now - int64(time.Second)) + if idx, _, ok := lw.stuck(now, loopStuckFor(wt)); ok { + lw.noteStuck(idx, 0, wt) + } + } + if got := h.n.Load(); got != 1 { + t.Fatalf("%d lines for one episode, want 1", got) + } + slot.Leave() // it ends: a judgement that finds no loop stuck ends the episode + lw.noteClear() + slot.Enter(now - int64(time.Second)) + lw.noteStuck(0, 0, wt) + if got := h.n.Load(); got != 2 { + t.Fatalf("%d lines after a second episode, want 2", got) + } + none := NewLoopWatch() // no logger: nothing, no crash + none.noteStuck(0, 0, wt) +} + +// The windows a connection is held "moving" by the horizon alone count as idle windows of +// the stream's run (review r11 N1): the worst case of an abandoned stream next to +// siblings that keep the connection moving is the seed plus the allowance plus one, 17 +// windows (13 without a seed), and not more however the siblings pace themselves. Without +// the count, a decoy that releases a floor in one window out of four held it for 49 windows +// (53 with the seed). In flight: 16 MiB (the allowance is the cap), stream 1 never +// granted, stream 3 the decoy. +func TestHorizonWindowsAreIdleWindows1170(t *testing.T) { + run := func(seed bool, moves func(w int) bool) int { + _, m, _ := wdSetup889(t) + if seed { + m.initWin.Store(1 << 20) // the client's initial stream window: the seed covers four floors + } + m.connHigh.Store(1 << 30) + stalled889(m, 1, 64<<20) + b := stalled889(m, 3, 64<<20) + m.connectionWindow = 1<<30 - 16<<20 + return firstReset889(m, 1, time.Now().UnixNano(), 80, func(w int) { + if moves(w) { + b.SetWindowSize(70000) + m.FlushOutbound(b, discard889) + } + m.connectionWindow = 1<<30 - 16<<20 + }) + } + every := func(int) bool { return true } + fourth := func(w int) bool { return w%4 == 1 } + for _, tc := range []struct { + name string + moves func(w int) bool + want int // without the seed + }{ + {"a decoy in every window", every, 1 + writeIdleCapWindows}, + {"a decoy in one window out of four", fourth, 1 + writeIdleCapWindows}, + {"a decoy that stops after three windows", func(w int) bool { return w <= 3 }, 3 + writeBankWindows}, + } { + if got := run(false, tc.moves); got != tc.want { + t.Errorf("%s: reset in window %d, want %d", tc.name, got, tc.want) + } + } + if got := run(true, fourth); got != writeBankWindows+1+writeIdleCapWindows { + t.Errorf("with the seed, a decoy in one window out of four: reset in window %d, want %d", got, writeBankWindows+1+writeIdleCapWindows) + } +} + +// Each idle run has its own allowance (review r11 NIT): the running maximum starts again +// when the stream takes its share and a new run begins, so a long queue that has drained +// does not leave the next run with the allowance of the first. +func TestIdleAllowanceStartsAgainWithEachRun1170(t *testing.T) { + _, m, _ := wdSetup889(t) + m.connHigh.Store(1 << 30) + a := stalled889(m, 1, 64<<20) + b := stalled889(m, 3, 64<<20) + m.connectionWindow = 1<<30 - 16<<20 // 16 MiB in flight: the first run has the cap + got := firstReset889(m, 1, time.Now().UnixNano(), 80, func(w int) { + b.SetWindowSize(70000) // the connection moves in every window + m.FlushOutbound(b, discard889) + switch w { + case 3: // stream 1 takes its share: the run ends + a.SetWindowSize(20000) + m.FlushOutbound(a, discard889) + case 4: // the queue has drained: no bytes in flight for the second run + m.connectionWindow = 1 << 30 + } + }) + if want := 3 + writeIdleWindows + 1; got != want { + t.Fatalf("reset in window %d, want %d: the second run (window 4 on, nothing in flight) must have the minimum allowance, not the first run's", got, want) + } +} diff --git a/test/server/h2_pool_wait_cells_1170_linux_test.go b/test/server/h2_pool_wait_cells_1170_linux_test.go new file mode 100644 index 000000000..ec9a2a557 --- /dev/null +++ b/test/server/h2_pool_wait_cells_1170_linux_test.go @@ -0,0 +1,562 @@ +//go:build linux + +package celeris_test + +import ( + "bufio" + "context" + "fmt" + "io" + "log/slog" + "net" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/http2" + + "github.com/goceleris/celeris" +) + +// celeris#1170, the raw-client cells: what a pool handler's wait for window does +// with a client that reads nothing, a bit, in big steps, or never again. + +// pool1170 is the routes of these cells. /pool is an async Blob of n bytes (a body +// the connection's 4 MiB output budget cannot buffer waits in sendRest); /stream is +// an async StreamWriter that writes chunk bytes at a time up to n, and when a Write +// fails notes it in *streamErr and sleeps sleep ms before it returns; /hold takes mu +// across its Blob and /locked takes mu (the wedge of #906's review). +type pool1170 struct { + streamErr atomic.Value // the error of the last failed Write + mu sync.Mutex + logs sink1170 // what the server logged (Debug and up): why a wait was reset, the episodes of a stuck loop +} + +// sink1170 is a slog handler that keeps the records of a server: the CI logs do not say +// which rule cut a stream, and a cell that fails must (review r10 item 4). +type sink1170 struct { + mu sync.Mutex + recs []string // "message reason=..." per record +} + +func (s *sink1170) Enabled(context.Context, slog.Level) bool { return true } +func (s *sink1170) Handle(_ context.Context, r slog.Record) error { + line := r.Message + r.Attrs(func(a slog.Attr) bool { + if a.Key == "reason" || a.Key == "loop" { + line += " " + a.Key + "=" + a.Value.String() + } + return true + }) + s.mu.Lock() + s.recs = append(s.recs, line) + s.mu.Unlock() + return nil +} +func (s *sink1170) WithAttrs([]slog.Attr) slog.Handler { return s } +func (s *sink1170) WithGroup(string) slog.Handler { return s } + +// resets counts the resets of a pool handler's wait by reason ("rate", "loop-blocked"). +func (s *sink1170) resets() map[string]int { + out := map[string]int{} + s.mu.Lock() + defer s.mu.Unlock() + for _, l := range s.recs { + if strings.HasPrefix(l, "h2: pool handler wait reset") { + if i := strings.Index(l, "reason="); i >= 0 { + out[l[i+len("reason="):]]++ + } + } + } + return out +} + +// episodes counts the times the server said the rate rule was suspended. +func (s *sink1170) episodes() int { + n := 0 + s.mu.Lock() + defer s.mu.Unlock() + for _, l := range s.recs { + if strings.HasPrefix(l, "h2: the minimum rate of a pool handler's wait") { + n++ + } + } + return n +} + +// summary is for a failing cell's message. +func (s *sink1170) summary() string { + return fmt.Sprintf("server log: resets by reason %v, %d episode(s) of a stuck loop", s.resets(), s.episodes()) +} + +func (p *pool1170) server(t *testing.T, eng celeris.EngineType, read, idle, write time.Duration) string { + t.Helper() + cfg := celeris.Config{Engine: eng, ReadTimeout: read, IdleTimeout: idle, WriteTimeout: write, Logger: slog.New(&p.logs)} + return startServerConfig761(t, cfg, func(s *celeris.Server) { + s.GET("/pool", func(c *celeris.Context) error { + n, _ := strconv.Atoi(c.Query("n")) + return c.Blob(200, "application/octet-stream", make([]byte, n)) + }).Async() + s.GET("/stream", func(c *celeris.Context) error { + n, _ := strconv.Atoi(c.Query("n")) + chunk, _ := strconv.Atoi(c.Query("chunk")) + sleep, _ := strconv.Atoi(c.Query("sleep")) + sw := c.StreamWriter() + if err := sw.WriteHeader(200, [][2]string{{"content-type", "application/octet-stream"}}); err != nil { + return err + } + buf := make([]byte, chunk) + for sent := 0; sent < n; sent += chunk { + if _, err := sw.Write(buf); err != nil { + p.streamErr.Store(err.Error()) + time.Sleep(time.Duration(sleep) * time.Millisecond) + return nil + } + } + return sw.Close() + }).Async() + s.GET("/fill", func(c *celeris.Context) error { + return c.Blob(200, "application/octet-stream", make([]byte, 1536<<10)) + }) + s.GET("/hold", func(c *celeris.Context) error { + n, _ := strconv.Atoi(c.Query("n")) + p.mu.Lock() + defer p.mu.Unlock() + return c.Blob(200, "application/octet-stream", make([]byte, n)) + }) + s.GET("/locked", func(c *celeris.Context) error { + p.mu.Lock() + p.mu.Unlock() // waiting for the lock is the point + return c.String(200, "ok") + }) + s.GET("/ping", func(c *celeris.Context) error { return c.String(200, "pong") }) + s.GET("/sleep", func(c *celeris.Context) error { + ms, _ := strconv.Atoi(c.Query("ms")) + time.Sleep(time.Duration(ms) * time.Millisecond) // a blocking call on the loop + return c.String(200, "slept") + }) + }) +} + +// TestH2PoolWaiterThatGrantsNothingIsReset1170: a client that reads nothing and +// PINGs. The pool handler's wait is reset by the same bound as a buffered tail: window +// 1 for the default window, window 5 for a 1 MiB one (the bytes in flight are the seed). +// The connection stays open: the bound resets the stream. +func TestH2PoolWaiterThatGrantsNothingIsReset1170(t *testing.T) { + for _, e := range nativeEngines889() { + for _, tc := range []struct { + iw uint32 + lo, hi time.Duration + }{{0, 900 * time.Millisecond, 2500 * time.Millisecond}, {1 << 20, 4500 * time.Millisecond, 8 * time.Second}} { + t.Run(fmt.Sprintf("%s/iw%d", e.name, tc.iw>>10), func(t *testing.T) { + t.Parallel() + var p pool1170 + c := dialH2raw889(t, p.server(t, e.eng, -1, -1, time.Second), h2dial889{initialWindow: tc.iw}) + c.window889(0, 1<<30-65535) // the connection window is not the limit: the stream's is + c.request(1, "GET", "/pool?n=8388608", true, true) + defer c.pingEvery889(200 * time.Millisecond)() + ev := c.await889(1, 12*time.Second) + t.Logf("WT1170 grants nothing %s/iw%d: %s", e.name, tc.iw>>10, ev) + wantWriteReset889(t, ev, tc.lo, tc.hi) + }) + } + } +} + +// TestH2PoolWaiterDribblerIsReset1170: WINDOW_UPDATE(1) every 100 ms moves nothing. +// Window 2 for the default window, window 5 for a 1 MiB one. +func TestH2PoolWaiterDribblerIsReset1170(t *testing.T) { + for _, e := range nativeEngines889() { + for _, tc := range []struct { + iw uint32 + lo, hi time.Duration + }{{0, 1800 * time.Millisecond, 3500 * time.Millisecond}, {1 << 20, 4500 * time.Millisecond, 8 * time.Second}} { + t.Run(fmt.Sprintf("%s/iw%d", e.name, tc.iw>>10), func(t *testing.T) { + t.Parallel() + var p pool1170 + c := dialH2raw889(t, p.server(t, e.eng, -1, -1, time.Second), h2dial889{initialWindow: tc.iw}) + c.window889(0, 1<<30-65535) // the connection window is not the limit: the stream's is + c.request(1, "GET", "/pool?n=8388608", true, true) + defer every889(100*time.Millisecond, func() { c.window889(1, 1); c.window889(0, 1) })() + ev := c.await889(1, 12*time.Second) + t.Logf("WT1170 dribbler %s/iw%d: %s", e.name, tc.iw>>10, ev) + wantWriteReset889(t, ev, tc.lo, tc.hi) + }) + } + } +} + +// TestH2PoolStreamWriterDribblerIsReset1170 (review r2 F2): a pool StreamWriter +// writes 64 KiB at a time, and the client grants each Write's bytes once in five +// windows. A fresh seed of everything in flight for every Write held that for ever, +// at a fifth of the floor. The seed counts what was sent since the last record. +func TestH2PoolStreamWriterDribblerIsReset1170(t *testing.T) { + for _, e := range nativeEngines889() { + t.Run(e.name, func(t *testing.T) { + t.Parallel() + var p pool1170 + c := dialH2raw889(t, p.server(t, e.eng, -1, -1, time.Second), h2dial889{initialWindow: 1 << 20}) + c.window889(0, 1<<30-65535) + c.request(1, "GET", "/stream?n=67108864&chunk=65536", true, true) + defer c.pingEvery889(300 * time.Millisecond)() + // The client takes the first megabyte at once (its window), then grants 64 KiB + // every 5 s: a fifth of the 64 KiB per second the floor asks. + defer every889(5*time.Second, func() { c.window889(1, 65536); c.window889(0, 65536) })() + ev := c.await889(1, 40*time.Second) + t.Logf("WT1170 stream dribbler %s: %s", e.name, ev) + if ev.timeout || !ev.reset || ev.rst != http2.ErrCodeInternal { + t.Fatalf("want the stream reset with INTERNAL_ERROR (a fifth of the floor is not the floor), got %s", ev) + } + }) + } +} + +// TestH2PoolWaiterBurstyReaderIsNotClosedByReadTimeout1170 (review r2 decision 2): a +// client that takes a whole 1 MiB window at once and grants it back every 2 s +// (512 KB/s on average, eight times the floor of 64 KiB per 5 s window; a half-window +// updater like curl or nghttp2 with a think time), so its frames are 2 s apart, +// longer than ReadTimeout (1 s), and nothing is queued for it in between. The wait is +// bounded by its rate, so the connection is exempt from the clocks while it lives. +// Measured on the previous head: this client is NOT closed there either (also with +// gaps of 4 s): a conn with a pool handler in flight is shielded by the engines' pool +// gate, whatever ServerOwes says. So the cell is a guard (it is green on main), not +// the red the review expected; the ServerOwes term is pinned by the unit test. +func TestH2PoolWaiterBurstyReaderIsNotClosedByReadTimeout1170(t *testing.T) { + for _, e := range nativeEngines889() { + t.Run(e.name, func(t *testing.T) { + t.Parallel() + var p pool1170 + addr := p.server(t, e.eng, time.Second, time.Second, 5*time.Second) + c := dialH2raw889(t, addr, h2dial889{initialWindow: 1 << 20}) + c.window889(0, 1<<30-65535) + const total = 6 << 20 + c.request(1, "GET", "/pool?n="+strconv.Itoa(total), true, true) + var got, acked int + _ = c.conn.SetReadDeadline(time.Now().Add(60 * time.Second)) + t0 := time.Now() + tick := time.Now().Add(2 * time.Second) + end := "" + for end == "" { + // Read as fast as the server sends, until the next grant is due. + _ = c.conn.SetReadDeadline(tick) + f, err := c.fr.ReadFrame() + if err != nil { + if !isTimeout889(err) { + end = "conn error: " + err.Error() + break + } + if got > acked { + c.window889(1, uint32(got-acked)) + c.window889(0, uint32(got-acked)) + acked = got + } + tick = time.Now().Add(2 * time.Second) + if time.Since(t0) > 60*time.Second { + end = "still open at the limit" + } + continue + } + switch f := f.(type) { + case *http2.DataFrame: + if f.StreamID == 1 { + got += len(f.Data()) + if f.StreamEnded() { + end = "complete" + } + } + case *http2.RSTStreamFrame: + end = "RST " + f.ErrCode.String() + case *http2.GoAwayFrame: + end = "GOAWAY " + f.ErrCode.String() + } + } + t.Logf("RT1170 %s: %s, %d bytes at %v", e.name, end, got, time.Since(t0).Round(time.Millisecond)) + if end != "complete" || got != total { + t.Fatalf("a client granting 1 MiB every 2 s ended %q after %d of %d bytes at %v", end, got, total, time.Since(t0).Round(time.Millisecond)) + } + }) + } +} + +// TestH2PoolWaiterOfASilentClientEndsTheConnection1170: the other side of the same +// decision. A silent client (nothing granted, no frame but a PING-less silence) with +// a waiting pool handler: the handler's wait is reset in window 1+k (5 with a 1 MiB +// window), the stream leaves the connection at the reset, and the connection's +// clocks (all 1 s) close it. Bound: 5 WriteTimeouts + ReadTimeout + slack. Also with +// a handler that goes on sleeping 3 s after the error: a pool handler in flight +// shields its connection from the clocks (the engines' pool gate), as on main, so the +// connection closes when the handler returns (measured: red on main only for that +// reason, it is the same after). +func TestH2PoolWaiterOfASilentClientEndsTheConnection1170(t *testing.T) { + for _, e := range nativeEngines889() { + for _, pc := range []struct { + path string + sleep time.Duration // what the handler goes on doing after its error + }{{"/pool?n=8388608", 0}, {"/stream?n=8388608&chunk=1048576&sleep=3000", 3 * time.Second}} { + path := pc.path + t.Run(e.name+path[:5], func(t *testing.T) { + t.Parallel() + var p pool1170 + addr := p.server(t, e.eng, time.Second, time.Second, time.Second) + c := dialH2raw889(t, addr, h2dial889{initialWindow: 1 << 20}) + c.window889(0, 1<<30-65535) + c.request(1, "GET", path, true, true) + ev := c.await889(1, 20*time.Second) + closedAt, closed := c.closedAfter889(20 * time.Second) + t.Logf("SC1170 %s %s: stream %s; connection closed=%v at %v", e.name, path[:5], ev, closed, closedAt.Round(time.Millisecond)) + if !ev.reset || ev.rst != http2.ErrCodeInternal { + t.Fatalf("the stream was not reset with INTERNAL_ERROR: %s", ev) + } + if lim := 10*time.Second + pc.sleep; !closed || closedAt > lim { + t.Fatalf("the connection was closed=%v at %v, want within 5 WriteTimeouts + ReadTimeout + the handler's sleep + slack (%v)", closed, closedAt, lim) + } + }) + } + } +} + +// TestH2PoolWaiterWithoutAWriteTimeoutKeepsTheOldClocks1170: WriteTimeout -1 is no +// bound on the wait and no exemption: a silent client's connection is closed by +// ReadTimeout, as before. +func TestH2PoolWaiterWithoutAWriteTimeoutKeepsTheOldClocks1170(t *testing.T) { + for _, e := range nativeEngines889() { + t.Run(e.name, func(t *testing.T) { + t.Parallel() + var p pool1170 + c := dialH2raw889(t, p.server(t, e.eng, time.Second, time.Second, -1), h2dial889{initialWindow: 1 << 20}) + c.window889(0, 1<<30-65535) + c.request(1, "GET", "/pool?n=8388608", true, true) + _ = c.await889(1, 20*time.Second) + closedAt, closed := c.closedAfter889(20 * time.Second) + t.Logf("NW1170 %s: connection closed=%v at %v", e.name, closed, closedAt.Round(time.Millisecond)) + if !closed || closedAt > 6*time.Second { + t.Fatalf("closed=%v at %v, want closed by ReadTimeout (1 s) within a few seconds", closed, closedAt) + } + }) + } +} + +// TestH2PoolWaiterHoldingALockDoesNotWedgeTheServer1170: the wedge of #906's review, +// with a client that reads at a compliant rate. The connection is put over the +// outbound budget (four sync GETs of a 1.5 MiB body nobody credits, as #906's own +// cell does), then /hold writes 8 MiB while it holds a mutex that /locked (sync, on +// an event loop) waits for. The client reads /hold at 128 KB/s and grants it as it +// reads, twice the floor at WriteTimeout 1 s, so the rule alone would hold /hold for +// the whole download (64 s) and the loop of the locker with it. If the locker's loop is +// the waiter's own, no WINDOW_UPDATE is read and the rule resets the wait by itself; +// if it is another loop (the case the fallback is for) only the fallback to WriteTimeout +// in all can: the trial is repeated on fresh connections, one locker each, so that a +// server without the fallback fails it with the probability of a coin in every trial. +// Every locker is answered within 1.5 WriteTimeouts plus slack. +func TestH2PoolWaiterHoldingALockDoesNotWedgeTheServer1170(t *testing.T) { + const timeout = time.Second + for _, e := range nativeEngines889() { + t.Run(e.name, func(t *testing.T) { + t.Parallel() + var p pool1170 + addr := p.server(t, e.eng, -1, -1, timeout) + trial := func(i int) { + c := dialH2raw889(t, addr, h2dial889{}) + c.window889(0, 1<<30-65535) + for k := 0; k < 4; k++ { + c.request(uint32(2*k+1), "GET", "/fill", true, true) // never credited + } + time.Sleep(300 * time.Millisecond) + c.request(9, "GET", "/hold?n=8388608", true, true) + stop := make(chan struct{}) + var recv atomic.Int64 + go func() { // reads, answers PINGs + for { + f, err := c.fr.ReadFrame() + if err != nil { + return + } + switch f := f.(type) { + case *http2.DataFrame: + if f.StreamID == 9 { + recv.Add(int64(len(f.Data()))) + } + case *http2.PingFrame: + if !f.IsAck() { + c.wmu.Lock() + _ = c.fr.WritePing(true, f.Data) + c.wmu.Unlock() + } + } + } + }() + go func() { // grants stream 9 as it "consumes" at 128 KB/s + t0, acked := time.Now(), 0 + tk := time.NewTicker(50 * time.Millisecond) + defer tk.Stop() + for { + select { + case <-stop: + return + case <-tk.C: + consumed := min(int(128e3*time.Since(t0).Seconds()), int(recv.Load())) + if consumed > acked { + c.window889(9, uint32(consumed-acked)) + c.window889(0, uint32(consumed-acked)) + acked = consumed + } + } + } + }() + time.Sleep(700 * time.Millisecond) // /hold waits for window, with the lock + t0 := time.Now() + lockers := get893(1, "http://"+addr+"/locked", timeout+6*time.Second) + took := time.Since(t0) + n, sl := answered893(lockers) + t.Logf("WG1170 %s trial %d: /locked %s in %v; /hold had received %d bytes", e.name, i, sl, took.Round(time.Millisecond), recv.Load()) + close(stop) + _ = c.conn.Close() // ends /hold's wait and releases its lock + time.Sleep(200 * time.Millisecond) + if n != 1 { + t.Errorf("trial %d: /locked not answered: the loop waited for the lock a compliant reader's download held", i) + } + if took > 3*timeout { + t.Errorf("trial %d: the locker waited %v, want about 1.5 x WriteTimeout (%v) plus slack", i, took, timeout) + } + } + for i := 0; i < 8; i++ { + trial(i) + } + }) + } +} + +// TestH2PoolWaiterIsNotCutWhileALoopIsBlockedBelowTheThreshold1170 and ...Above...: +// the fallback is server-wide and has a threshold (half a WriteTimeout, at least 250 +// ms). A route that blocks one loop for 0.1 s at a time (under it) leaves Go's +// Transport 4 x 8 MiB at WriteTimeout 1 s as green as without it; one that blocks 0.8 +// s (over) switches the fallback on, and the downloads are bounded by WriteTimeout in +// all, as before celeris#1170: some are reset, as documented. +func TestH2PoolWaiterLoopBlockedThreshold1170(t *testing.T) { + // "under" and "over" one after the other, the engines of one side by side: four + // servers with their sleepers and downloads at once under -race or coverage on a + // CI runner starve each other's loops, and a loop that the CPU leaves stuck for + // half a WriteTimeout is the thing "under" says must not happen. + for _, tc := range []struct { + name string + block int // ms a batch blocks + cut bool + }{{"under", 100, false}, {"over", 800, true}} { + t.Run(tc.name, func(t *testing.T) { + for _, e := range nativeEngines889() { + if e.name == "io_uring" { + continue // its own send clock at WriteTimeout 1 s + } + t.Run(e.name, func(t *testing.T) { + t.Parallel() + var p pool1170 + addr := p.server(t, e.eng, -1, -1, time.Second) + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { // one connection at a time: one loop is blocked, then not + defer wg.Done() + for { + select { + case <-stop: + return + case <-time.After(300 * time.Millisecond): + } + _ = get893(1, "http://"+addr+"/sleep?ms="+strconv.Itoa(tc.block), 5*time.Second) + } + }() + ends, at := downloads1170(t, addr, "/pool", 4, 8<<20, 1e6, 60*time.Second) + close(stop) + wg.Wait() + cut := 0 + for _, r := range ends { + if r != "complete" { + cut++ + } + } + t.Logf("LB1170 %s/%s: %d cut: %q at %v; %s", e.name, tc.name, cut, ends, at, p.logs.summary()) + if tc.cut && (cut == 0 || p.logs.resets()["loop-blocked"] == 0 || p.logs.episodes() == 0) { + t.Fatalf("%d cut although a loop was blocked for 0.8 s at a time: the fallback to WriteTimeout in all must cut some, say so (%s)", cut, p.logs.summary()) + } + if !tc.cut && (cut != 0 || p.logs.resets()["loop-blocked"] != 0) { + t.Fatalf("%d downloads were cut although the loops were blocked for only 0.1 s at a time (under the threshold of 0.5 s): %s", cut, p.logs.summary()) + } + }) + } + }) + } +} + +// TestH2PoolWaiterAfterTheLoopsParked1170 (review r10 F2): PauseAccept, then the +// last connections close: a loop with no connection parks on a channel, not in +// epoll_wait or the ring wait, right after the batch that closed its last one. If it +// kept the stamp of that batch it read as stuck for ever, and every waiter on the +// engine fell back to WriteTimeout in all until ResumeAccept, silently. Go's Transport +// holds an h2 connection through the pause and downloads 4 x 8 MiB after it; nothing +// may be cut, and nothing may be said about a stuck loop. io_uring is run at 5 s (its +// own send clock cuts at 1 s), epoll and adaptive at 1 s. (Adaptive's standby side +// runs the same park code.) +func TestH2PoolWaiterAfterTheLoopsParked1170(t *testing.T) { + for _, e := range nativeEngines889() { + if e.name == "adaptive" { + continue // it has no PauseAccept of its own; its standby side runs the park code of these two + } + wt := time.Second + if e.name == "io_uring" { + wt = 5 * time.Second + } + t.Run(e.name, func(t *testing.T) { + var p pool1170 + cfg := celeris.Config{Engine: e.eng, ReadTimeout: -1, IdleTimeout: -1, WriteTimeout: wt, Logger: slog.New(&p.logs)} + srv, addr := startPickedPing1104(t, cfg, func(s *celeris.Server) { + ping1104(s) + s.GET("/pool", func(c *celeris.Context) error { + n, _ := strconv.Atoi(c.Query("n")) + return c.Blob(200, "application/octet-stream", make([]byte, n)) + }).Async() + }) + cl := goTransport1170(1e6) + if resp, err := cl.Get("http://" + addr + "/pool?n=1"); err == nil { // the h2 connection, kept by the Transport + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } else { + t.Fatalf("warm-up: %v", err) + } + // h1 connections that each did a request: every loop has handled a batch. + var hs []net.Conn + for i := 0; i < 16; i++ { + c, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + t.Fatalf("dial: %v", err) + } + _ = c.SetDeadline(time.Now().Add(5 * time.Second)) + _, _ = c.Write([]byte("GET /ping HTTP/1.1\r\nHost: x\r\n\r\n")) + if line, err := bufio.NewReader(c).ReadString('\n'); err != nil || !strings.Contains(line, "200") { + t.Fatalf("h1 ping: %q %v", line, err) + } + hs = append(hs, c) + } + if err := srv.PauseAccept(); err != nil { + t.Fatalf("PauseAccept: %v", err) + } + for _, c := range hs { + _ = c.Close() + } + time.Sleep(1500 * time.Millisecond) // the loops with no connection are parked, and a stale stamp has aged + ends, at := downloadsOn1170(t, cl, addr, "/pool", 4, 8<<20, 60*time.Second) + _ = srv.ResumeAccept() + t.Logf("PK1170 %s WT%v: %q at %v; %s", e.name, wt, ends, at, p.logs.summary()) + for i, r := range ends { + if r != "complete" { + t.Fatalf("download %d: %s at %v: %s", i, r, at[i].Round(time.Millisecond), p.logs.summary()) + } + } + if p.logs.episodes() != 0 { + t.Fatalf("the server said the rate rule was suspended (a parked loop read as stuck): %s", p.logs.summary()) + } + }) + } +} diff --git a/test/server/h2_pool_wait_rate_1170_linux_test.go b/test/server/h2_pool_wait_rate_1170_linux_test.go new file mode 100644 index 000000000..be1b0bc1f --- /dev/null +++ b/test/server/h2_pool_wait_rate_1170_linux_test.go @@ -0,0 +1,199 @@ +//go:build linux + +package celeris_test + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "strconv" + "sync" + "testing" + "time" +) + +// celeris#1170 (stage 3 of #889). A pool handler whose response the connection's +// outbound budget could not buffer waits for the peer's window in sendRest, and was +// bounded by an absolute WriteTimeout from the start of the wait (#906), whatever +// the client read. These cells use Go's own x/net/http2.Transport over a link of +// bps bytes/s, the client the issue was found with. + +// slowConn1170 reads at most bps bytes per second in total (the link), 4 KiB a read. +type slowConn1170 struct { + net.Conn + th *throttled889 +} + +func (c *slowConn1170) Read(p []byte) (int, error) { return c.th.Read(p) } + +// goTransport1170 is a Go HTTP/2 client over h2c (net/http's Transport with +// unencrypted HTTP/2, the bundled implementation of x/net/http2's transport: 4 MiB +// stream window, 1 GiB connection window) whose connection is read at bps. +func goTransport1170(bps float64) *http.Client { + var pr http.Protocols + pr.SetUnencryptedHTTP2(true) + tr := &http.Transport{ + Protocols: &pr, + DialContext: func(ctx context.Context, network, a string) (net.Conn, error) { + c, err := (&net.Dialer{}).DialContext(ctx, network, a) + if err != nil { + return nil, err + } + sc := &slowConn1170{Conn: c} + sc.th = &throttled889{r: c, bps: bps, t0: time.Now()} + return sc, nil + }, + } + return &http.Client{Transport: tr} +} + +// downloads1170 runs n parallel GETs of path (size bytes each) on one connection and +// reports, per request, how it ended and when. +func downloads1170(t *testing.T, addr, path string, n, size int, bps float64, limit time.Duration) (ends []string, at []time.Duration) { + t.Helper() + cl := goTransport1170(bps) + // One tiny request first: the downloads share its connection (a Transport that + // is asked for several at once may dial several). + if resp, err := cl.Get("http://" + addr + path + "?n=1"); err == nil { + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + } + return downloadsOn1170(t, cl, addr, path, n, size, limit) +} + +// downloadsOn1170 is downloads1170 on a client that already holds its connection. +func downloadsOn1170(t *testing.T, cl *http.Client, addr, path string, n, size int, limit time.Duration) (ends []string, at []time.Duration) { + t.Helper() + ends, at = make([]string, n), make([]time.Duration, n) + t0 := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), limit) + defer cancel() + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + req, _ := http.NewRequestWithContext(ctx, "GET", "http://"+addr+path+"?n="+strconv.Itoa(size), nil) + resp, err := cl.Do(req) + if err != nil { + ends[i], at[i] = "do: "+err.Error(), time.Since(t0) + return + } + got, err := io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + at[i] = time.Since(t0) + switch { + case err != nil: + ends[i] = fmt.Sprintf("body after %d of %d: %v", got, size, err) + case int(got) != size: + ends[i] = fmt.Sprintf("short: %d of %d", got, size) + default: + ends[i] = "complete" + } + }() + } + wg.Wait() + return ends, at +} + +// TestH2GoTransportParallelDownloadsOnASlowLinkAreNotCut1170: four 8 MiB +// downloads by Go's Transport on a 1 MB/s link (each about 250 KB/s, four times the +// 64 KiB-per-window floor). On the previous head three of the four were reset with +// INTERNAL_ERROR (the client sees it at about 17 s, behind the bytes queued ahead of +// it): the first grant of the second, third and fourth stream comes 5, 9 and 14 s +// after their handlers began to wait (a 4 MiB stream window each is queued in the +// socket ahead of the next), and #906's deadline was 1 WriteTimeout. +// +// io_uring at WriteTimeout 1 s is not in the table: its own send clock (a send +// that does not complete for WriteTimeout, "io_uring is unchanged" in +// Config.WriteTimeout) closes the connection first, identically on main (measured: +// every stream ends in an unexpected EOF at 17 to 24 s). It is not this rule. +func TestH2GoTransportParallelDownloadsOnASlowLinkAreNotCut1170(t *testing.T) { + // One WriteTimeout at a time, and at 1 s one engine at a time: a few of these cells + // at once under -race or a coverage profile starve each other's event loops (CI saw + // it), and a loop that is stuck in one batch for half a WriteTimeout switches the + // rule off for the server (the fallback), which at 1 s is a cell's flake, not a + // finding. At 5 s the threshold is 2.5 s and the engines run side by side. + for _, wt := range []time.Duration{time.Second, 5 * time.Second} { + t.Run(fmt.Sprintf("WT%v", wt), func(t *testing.T) { + for _, e := range nativeEngines889() { + if e.name == "io_uring" && wt < 5*time.Second { + continue + } + t.Run(e.name, func(t *testing.T) { + if wt >= 5*time.Second { + t.Parallel() + } + var p pool1170 + addr := p.server(t, e.eng, -1, -1, wt) + // Every loop has handled a batch of events before the download starts + // and is idle after it: a loop that kept the stamp of its last batch + // would read as stuck for ever, and switch the rule off. + _ = get893(8, "http://"+addr+"/ping", 5*time.Second) + time.Sleep(700 * time.Millisecond) + ends, at := downloads1170(t, addr, "/pool", 4, 8<<20, 1e6, 60*time.Second) + t.Logf("GT1170 %s/WT%v: %q at %v; %s", e.name, wt, ends, at, p.logs.summary()) + for i, r := range ends { + if r != "complete" { + t.Fatalf("download %d: %s at %v (all four must complete); %s", i, r, at[i].Round(time.Millisecond), p.logs.summary()) + } + } + }) + } + }) + } +} + +// TestH2GoTransportManyParallelDownloadsAreNotCut1170: eight 8 MiB downloads on a +// 2 MB/s link, WriteTimeout 1 s: the last stream's first grant is 7 x 4 MiB / 2 MB/s +// = 14 s after its wait began, 14 windows (the idle allowance covers it, not the 3 of +// the rule before celeris#1170). +func TestH2GoTransportManyParallelDownloadsAreNotCut1170(t *testing.T) { + for _, e := range nativeEngines889() { + if e.name == "io_uring" { + continue // its own send clock at WriteTimeout 1 s, see above + } + t.Run(e.name, func(t *testing.T) { + t.Parallel() + var p pool1170 + addr := p.server(t, e.eng, -1, -1, time.Second) + ends, at := downloads1170(t, addr, "/pool", 8, 8<<20, 2e6, 90*time.Second) + t.Logf("GT1170 many %s: %q at %v; %s", e.name, ends, at, p.logs.summary()) + for i, r := range ends { + if r != "complete" { + t.Fatalf("download %d: %s at %v (all eight must complete); %s", i, r, at[i].Round(time.Millisecond), p.logs.summary()) + } + } + }) + } +} + +// TestH2GoTransportQueueBeyondTheCapIsCut1170: the cap exists. Four 8 MiB +// downloads on 1 MB/s at WriteTimeout 0.5 s: the last waiter's first grant is about +// 27 windows in, past the allowance (12 + the seed's 4), so it is reset, as +// documented ("N x W / L under about 20 x WriteTimeout"). +func TestH2GoTransportQueueBeyondTheCapIsCut1170(t *testing.T) { + for _, e := range nativeEngines889() { + if e.name == "io_uring" { + continue // its own send clock, see above + } + t.Run(e.name, func(t *testing.T) { + t.Parallel() + var p pool1170 + addr := p.server(t, e.eng, -1, -1, 500*time.Millisecond) + ends, at := downloads1170(t, addr, "/pool", 4, 8<<20, 1e6, 60*time.Second) + t.Logf("GT1170 beyond the cap %s: %q at %v; %s", e.name, ends, at, p.logs.summary()) + cut := 0 + for _, r := range ends { + if r != "complete" { + cut++ + } + } + if cut == 0 { + t.Fatalf("all four completed: the queue (about 27 windows) is past the cap of 12 + 4 and a waiter must have been reset") + } + }) + } +} diff --git a/test/server/h2_write_rate_889_linux_test.go b/test/server/h2_write_rate_889_linux_test.go index f7dbda50a..975828f3d 100644 --- a/test/server/h2_write_rate_889_linux_test.go +++ b/test/server/h2_write_rate_889_linux_test.go @@ -258,9 +258,16 @@ func TestH2ParallelDownloadsSharingASlowLinkAreNotCut889(t *testing.T) { // TestH2PausedAndDribbledStreamsOnAHealthyConnAreBounded889: on a connection that // moves well above the floor a stream the client does not read at all (no // WINDOW_UPDATE), or feeds a byte every 100 ms, is held for a few windows (it may be -// waiting behind its siblings in the same socket) and reset after that: three -// windows with nothing of its own, so the fourth judgement, 4 s in. Its sibling, -// read at 250 KB/s, goes on. On the previous head it was reset at 1.0 s. +// waiting behind its siblings in the same socket) and reset after that. Its sibling, +// read at 250 KB/s, goes on. On the head before celeris#889 D2 it was reset at 1.0 s; +// with D2 three windows with nothing of its own, the fourth judgement, 4 s in; since +// celeris#1170 (option B: the idle allowance follows the bytes in flight, at most 12 +// windows, and the connection counts as moving for four windows after its siblings +// stop) the allowance is the queue the sibling's 2 MiB leaves in flight over its rate, +// and the stream is reset when that is read and the siblings have been quiet for the +// horizon: measured 12.2 to 13.8 s here (the client sees the RST behind the data it +// is still reading). The bound is 4 + 12 + 1 windows in all (the windows the horizon +// holds count inside the allowance), so [3.5 s, 18 s]. func TestH2PausedAndDribbledStreamsOnAHealthyConnAreBounded889(t *testing.T) { for _, e := range nativeEngines889() { for _, mode := range []string{"paused", "dribbled"} { @@ -273,8 +280,8 @@ func TestH2PausedAndDribbledStreamsOnAHealthyConnAreBounded889(t *testing.T) { } res, got, at := slowLink889(t, addr, 250e3, []uint32{1, 3}, 2<<20, skip, dribble) t.Logf("WR889 paused %s/%s: %v %v at %v", e.name, mode, res, got, at) - if res[3] != "RST INTERNAL_ERROR" || at[3] < 3500*time.Millisecond || at[3] > 7*time.Second { - t.Fatalf("stream 3 (%s): %q at %v, want RST INTERNAL_ERROR in [3.5 s, 7 s] (three windows of grace, then the verdict)", mode, res[3], at[3].Round(time.Millisecond)) + if res[3] != "RST INTERNAL_ERROR" || at[3] < 3500*time.Millisecond || at[3] > 18*time.Second { + t.Fatalf("stream 3 (%s): %q at %v, want RST INTERNAL_ERROR in [3.5 s, 18 s] (the allowance of option B, then the verdict)", mode, res[3], at[3].Round(time.Millisecond)) } if r := res[1]; r != "" && r != "complete" { t.Fatalf("the healthy sibling, stream 1, ended with %q", r)