From ebf1bf470c918453890bd8fc8749e6c4c70b76b9 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 2 Nov 2026 23:56:19 +0100 Subject: [PATCH 1/9] test(iouring): failing tests for engines and seams that outlive a failed test (Refs #1229) --- .../drain_clock_step_1176_linux_test.go | 6 +- .../iouring/engine_cleanup_1229_linux_test.go | 153 ++++++++++++++++++ ...n_release_stop_accepting_896_linux_test.go | 8 +- 3 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 internal/engine/iouring/engine_cleanup_1229_linux_test.go diff --git a/internal/engine/iouring/drain_clock_step_1176_linux_test.go b/internal/engine/iouring/drain_clock_step_1176_linux_test.go index 5e6cd868b..451e0798c 100644 --- a/internal/engine/iouring/drain_clock_step_1176_linux_test.go +++ b/internal/engine/iouring/drain_clock_step_1176_linux_test.go @@ -6,7 +6,6 @@ import ( "context" "io" "net" - "sync" "sync/atomic" "testing" "time" @@ -356,10 +355,7 @@ func TestPoolWaitGateWaitsItsWindowWhateverTheWallDoes1176(t *testing.T) { // releaseOnce1176 is h's release, closed at most once, so a test that fails // before the handler is released still lets it go. -func releaseOnce1176(h *holdHandler896) func() { - var once sync.Once - return func() { once.Do(func() { close(h.release) }) } -} +func releaseOnce1176(h *holdHandler896) func() { return h.unblock } func isTimeout1176(err error) bool { ne, ok := err.(net.Error) diff --git a/internal/engine/iouring/engine_cleanup_1229_linux_test.go b/internal/engine/iouring/engine_cleanup_1229_linux_test.go new file mode 100644 index 000000000..0940c3d95 --- /dev/null +++ b/internal/engine/iouring/engine_cleanup_1229_linux_test.go @@ -0,0 +1,153 @@ +//go:build linux + +package iouring + +import ( + "testing" + "time" +) + +// celeris#1229: a test that fails after it started an engine must stop the +// engine. The helpers below start engines for the 896 and 1173 tests and +// used to leave the stopping to the test body, which a failed assertion +// (t.Fatalf: runtime.Goexit) never reaches. On the Coverage job's 8 MiB +// RLIMIT_MEMLOCK four 896 tests failed on a dial timeout, left four engines +// running, and every later test that installs a package seam (ringWaitSeam, +// cancelAcceptSeam) had those workers read it: answered EBUSY for waits it +// never asked about, and raced with the seam's restore. + +// engineAfterItsTestEnded1229 runs start in a subtest that ends at once, the +// way a failed assertion ends one, and returns the run the helper started and +// whether Listen had returned by the time the subtest's cleanups had run. A +// run that is still going is cancelled here, so that this test does not leak +// what it looks for. +func engineAfterItsTestEnded1229(t *testing.T, start func(t *testing.T) *engineRun896) (run *engineRun896, stopped bool) { + t.Helper() + t.Run("ended", func(t *testing.T) { + run = start(t) + t.SkipNow() + }) + if run == nil { + t.Skip("the engine did not start here") + } + select { + case err := <-run.done: + if err != nil { + t.Errorf("Listen returned %v", err) + } + return run, true + default: + } + run.cancel() + select { + case <-run.done: + case <-time.After(10 * time.Second): + t.Error("the leaked engine did not stop when cancelled") + } + return run, false +} + +func TestEngineStartHelpersStopTheirEngineWhenTheTestEnds1229(t *testing.T) { + const leaked = "the engine was still running after the test that started it had ended: " + + "a failed assertion would leave its workers reading the package's test seams" + t.Run("start896", func(t *testing.T) { + if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 { + return start896(t, freeAddr639(t), 2) + }); !stopped { + t.Error(leaked) + } + }) + t.Run("startHold896", func(t *testing.T) { + if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 { + return startHold896(t, freeAddr639(t), 2, 0, newHoldHandler896(false)) + }); !stopped { + t.Error(leaked) + } + }) + // A /hold handler still running at the end is what a failure in the middle + // of an iteration leaves: Listen waits for it, so the engine's cleanup has + // to release it before it waits. + t.Run("startHold896/handler running", func(t *testing.T) { + var h *holdHandler896 + if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 { + h = newHoldHandler896(false) + addr := freeAddr639(t) + run := startHold896(t, addr, 2, 0, h) + c := h2Request896(t, addr, "/hold") + t.Cleanup(func() { _ = c.Close() }) + select { + case <-h.started: + case <-time.After(5 * time.Second): + t.Fatal("the /hold handler never ran") + } + return run + }); !stopped { + h.unblock() + t.Error(leaked) + } + }) + t.Run("startGate1173", func(t *testing.T) { + var g *gate1173 + if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 { + g = &gate1173{started: make(chan struct{}, 1), gate: make(chan struct{})} + run, _ := startGate1173(t, g) + return run + }); !stopped { + t.Error(leaked) + } + }) +} + +// seamRestoredUnderALateReader1229 installs a seam with install in a subtest, +// whose cleanup restores it, while a goroutine that is not ordered with that +// restore goes on reading the seam through read: a worker that an earlier test +// leaked. The restore is a write the race detector must not find unordered +// with the read, so a seam read and written as a plain variable fails this +// test under -race ("race detected during execution of test"). +func seamRestoredUnderALateReader1229(t *testing.T, install func(t *testing.T), read func()) { + t.Helper() + stop := make(chan struct{}) + finished := make(chan struct{}) + t.Run("seam", func(t *testing.T) { + install(t) + go func() { + defer close(finished) + for { + select { + case <-stop: + return + default: + read() + } + } + }() + }) + close(stop) + <-finished +} + +func TestRingWaitSeamRestoreIsSafeAgainstALateReader1229(t *testing.T) { + var w Worker + seamRestoredUnderALateReader1229(t, func(t *testing.T) { + setRingWaitSeam884(t, func(*Worker, ringWaitMode, error) error { return nil }) + }, func() { _ = w.ringWaitContinues(waitModeTimed, nil) }) +} + +func TestCancelAcceptSeamRestoreIsSafeAgainstALateReader1229(t *testing.T) { + r := newAcceptRig896(t, &baseTier{}, false) + seamRestoredUnderALateReader1229(t, func(t *testing.T) { + setCancelAcceptSeam1229(t, func(*Worker, int) {}) + }, func() { + // stopAccepting reads the seam first; the listen socket is closed by the + // first call, and a later one finds none and returns. + r.w.stopAccepting(t.Context()) + }) +} + +// setCancelAcceptSeam1229 installs cancelAcceptSeam for the test and restores +// it nil when the test ends. +func setCancelAcceptSeam1229(t *testing.T, fn func(*Worker, int)) { + t.Helper() + cancelAcceptSeam = fn + t.Cleanup(func() { cancelAcceptSeam = nil }) +} diff --git a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go index 90761e984..c38ecb38c 100644 --- a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go +++ b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go @@ -12,6 +12,7 @@ import ( "net" "os" "strconv" + "sync" "sync/atomic" "testing" "time" @@ -48,6 +49,7 @@ type holdHandler896 struct { release chan struct{} big []byte // the /big response; nil unless the test serves it running atomic.Int64 // /hold handlers that have started (started drops past its 4) + once sync.Once // unblock's } func newHoldHandler896(bigBody bool) *holdHandler896 { @@ -58,6 +60,10 @@ func newHoldHandler896(bigBody bool) *holdHandler896 { return h } +// unblock releases the /hold handlers, once however often it is called: a test +// calls it, and so does the cleanup of the engine that serves them. +func (h *holdHandler896) unblock() { h.once.Do(func() { close(h.release) }) } + func (h *holdHandler896) HandleStream(_ context.Context, s *stream.Stream) error { if s.ResponseWriter == nil { return nil @@ -279,7 +285,7 @@ func h2Iteration896(t *testing.T, calls *atomic.Int64) (string, time.Time) { before := calls.Load() run.cancel() waitCalls896(t, calls, before) - close(h.release) + h.unblock() returned, err := run.stopped() if err != nil { t.Fatalf("Listen returned %v after a cancel", err) From eb90733084fd56af19f1f090856f6474010e6ca8 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Mon, 2 Nov 2026 23:58:28 +0100 Subject: [PATCH 2/9] fix(iouring): tests stop the engines they start and the ring-wait and cancel-accept seams are atomic (Refs #1229) --- .../drain_clock_step_1176_linux_test.go | 3 +- .../iouring/engine_cleanup_1229_linux_test.go | 11 +++-- .../engine/iouring/listen_cancel_seam_896.go | 7 ++- .../iouring/listen_release_896_linux_test.go | 49 ++++++++++++++++--- ...n_release_stop_accepting_896_linux_test.go | 17 ++++--- internal/engine/iouring/ring_wait_seam_884.go | 9 +++- .../iouring/run_ebusy_884_linux_test.go | 13 +++-- internal/engine/iouring/worker.go | 16 +++--- .../worker_exit_pause_lifecycle_linux_test.go | 5 +- ...write_clock_stale_stamp_1173_linux_test.go | 8 +-- ...e_clock_stale_stamp_det_1173_linux_test.go | 6 +-- 11 files changed, 100 insertions(+), 44 deletions(-) diff --git a/internal/engine/iouring/drain_clock_step_1176_linux_test.go b/internal/engine/iouring/drain_clock_step_1176_linux_test.go index 451e0798c..be04200f5 100644 --- a/internal/engine/iouring/drain_clock_step_1176_linux_test.go +++ b/internal/engine/iouring/drain_clock_step_1176_linux_test.go @@ -433,8 +433,7 @@ func TestCancelAcceptWaitIsBoundedByTheMonotonicClock1176(t *testing.T) { return nil } t.Cleanup(func() { shutdownRingWait = orig }) - cancelAcceptSeam = func(w *Worker, _ int) { ring.CompareAndSwap(nil, w.ring) } - t.Cleanup(func() { cancelAcceptSeam = nil }) + setCancelAcceptSeam1229(t, func(w *Worker, _ int) { ring.CompareAndSwap(nil, w.ring) }) restore := stairClock1176(t, step) h := newHoldHandler896(false) diff --git a/internal/engine/iouring/engine_cleanup_1229_linux_test.go b/internal/engine/iouring/engine_cleanup_1229_linux_test.go index 0940c3d95..a351697f6 100644 --- a/internal/engine/iouring/engine_cleanup_1229_linux_test.go +++ b/internal/engine/iouring/engine_cleanup_1229_linux_test.go @@ -16,8 +16,10 @@ import ( // cancelAcceptSeam) had those workers read it: answered EBUSY for waits it // never asked about, and raced with the seam's restore. -// engineAfterItsTestEnded1229 runs start in a subtest that ends at once, the -// way a failed assertion ends one, and returns the run the helper started and +// engineAfterItsTestEnded1229 runs start in a subtest that returns at once +// without stopping what it started, which is all a failed assertion does to +// the cleanups of its test (runtime.Goexit runs them, and nothing else of the +// test's body), and returns the run the helper started and // whether Listen had returned by the time the subtest's cleanups had run. A // run that is still going is cancelled here, so that this test does not leak // what it looks for. @@ -25,7 +27,6 @@ func engineAfterItsTestEnded1229(t *testing.T, start func(t *testing.T) *engineR t.Helper() t.Run("ended", func(t *testing.T) { run = start(t) - t.SkipNow() }) if run == nil { t.Skip("the engine did not start here") @@ -148,6 +149,6 @@ func TestCancelAcceptSeamRestoreIsSafeAgainstALateReader1229(t *testing.T) { // it nil when the test ends. func setCancelAcceptSeam1229(t *testing.T, fn func(*Worker, int)) { t.Helper() - cancelAcceptSeam = fn - t.Cleanup(func() { cancelAcceptSeam = nil }) + cancelAcceptSeam.Store(&fn) + t.Cleanup(func() { cancelAcceptSeam.Store(nil) }) } diff --git a/internal/engine/iouring/listen_cancel_seam_896.go b/internal/engine/iouring/listen_cancel_seam_896.go index c3004d3e0..934021836 100644 --- a/internal/engine/iouring/listen_cancel_seam_896.go +++ b/internal/engine/iouring/listen_cancel_seam_896.go @@ -2,8 +2,13 @@ package iouring +import "sync/atomic" + // cancelAcceptSeam is a test seam (celeris#896): when set, cancelAccept calls it // first, on the worker's thread, with the descriptor it is about to cancel. A // test counts the calls to prove a shutdown reached stopAccepting or the pause // close, and uses it to make the SQ ring full at that moment. Nil in production. -var cancelAcceptSeam func(w *Worker, lfd int) +// An atomic.Pointer, as h2WriteDrainSeam is, because a worker an earlier test +// left running may still read it when a later test sets or restores it +// (celeris#1229). +var cancelAcceptSeam atomic.Pointer[func(w *Worker, lfd int)] diff --git a/internal/engine/iouring/listen_release_896_linux_test.go b/internal/engine/iouring/listen_release_896_linux_test.go index aae46e656..95b042c8d 100644 --- a/internal/engine/iouring/listen_release_896_linux_test.go +++ b/internal/engine/iouring/listen_release_896_linux_test.go @@ -36,9 +36,46 @@ const iterations896 = 200 // engineRun896 is one started engine and the way to stop it. type engineRun896 struct { - e *Engine - cancel context.CancelFunc - done chan error + e *Engine + cancel context.CancelFunc + done chan error // receives Listen's result + returned chan struct{} // closed once done has it (listenAsync896) +} + +// listenAsync896 runs e.Listen(ctx) on a goroutine of its own. done receives +// its result; returned is closed after that, so a cleanup that waits on it +// can still read done. +func listenAsync896(e *Engine, ctx context.Context) (done chan error, returned chan struct{}) { + done = make(chan error, 1) + returned = make(chan struct{}) + go func() { + done <- e.Listen(ctx) + close(returned) + }() + return done, returned +} + +// reapAtCleanup stops the engine when the test ends, whether or not the test +// stopped it: a test that fails after it started the engine ends in +// runtime.Goexit, which skips the stop in its body, and the workers of an +// engine left running read the package's test seams (ringWaitSeam, +// cancelAcceptSeam) in every test after it (celeris#1229). release, if not +// nil, frees whatever Listen waits for first (a handler held by the test). The +// cleanup is registered when the engine has started, so it runs before the +// restores of the seams the test installed earlier. +func (r *engineRun896) reapAtCleanup(t *testing.T, release func()) { + t.Helper() + t.Cleanup(func() { + if release != nil { + release() + } + r.cancel() + select { + case <-r.returned: + case <-time.After(10 * time.Second): + t.Errorf("Listen had not returned 10 s after the test ended: a worker is still running") + } + }) } // stop cancels Listen's context and returns the instant Listen has returned, @@ -78,11 +115,10 @@ func start896Config(t *testing.T, addr string, workers int, disableDefer bool) * skipOrFail656(t, "New: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- e.Listen(ctx) }() + done, returned := listenAsync896(e, ctx) for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); { if e.Addr() != nil { - run = &engineRun896{e: e, cancel: cancel, done: done} + run = &engineRun896{e: e, cancel: cancel, done: done, returned: returned} return nil } select { @@ -103,6 +139,7 @@ func start896Config(t *testing.T, addr string, workers int, disableDefer bool) * } t.Fatalf("engine start on %s: %v", addr, err) } + run.reapAtCleanup(t, nil) return run } diff --git a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go index c38ecb38c..6abec149c 100644 --- a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go +++ b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go @@ -123,11 +123,10 @@ func startHold896(t *testing.T, addr string, workers int, writeTimeout time.Dura skipOrFail656(t, "New: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- e.Listen(ctx) }() + done, returned := listenAsync896(e, ctx) for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); { if e.Addr() != nil { - run = &engineRun896{e: e, cancel: cancel, done: done} + run = &engineRun896{e: e, cancel: cancel, done: done, returned: returned} return nil } select { @@ -148,6 +147,13 @@ func startHold896(t *testing.T, addr string, workers int, writeTimeout time.Dura } t.Fatalf("engine start on %s: %v", addr, err) } + // A handler of h that the test holds is released before Listen is waited + // for: it waits for the HTTP/2 stream handlers on the worker pool. + var release func() + if hh, ok := h.(*holdHandler896); ok { + release = hh.unblock + } + run.reapAtCleanup(t, release) return run } @@ -185,7 +191,7 @@ func h2Request896(t *testing.T, addr, path string) net.Conn { func seam896(t *testing.T, full bool) *atomic.Int64 { t.Helper() var calls atomic.Int64 - cancelAcceptSeam = func(w *Worker, _ int) { + setCancelAcceptSeam1229(t, func(w *Worker, _ int) { if full { for { sqe := w.ring.GetSQE() @@ -196,8 +202,7 @@ func seam896(t *testing.T, full bool) *atomic.Int64 { } } calls.Add(1) - } - t.Cleanup(func() { cancelAcceptSeam = nil }) + }) return &calls } diff --git a/internal/engine/iouring/ring_wait_seam_884.go b/internal/engine/iouring/ring_wait_seam_884.go index 8038f17c6..9347e4ec8 100644 --- a/internal/engine/iouring/ring_wait_seam_884.go +++ b/internal/engine/iouring/ring_wait_seam_884.go @@ -2,6 +2,8 @@ package iouring +import "sync/atomic" + // ringWaitMode names which of Worker.run's ring waits ended: the argument the // seam below receives, so a test can say which wait of the loop it answered. type ringWaitMode uint8 @@ -39,5 +41,8 @@ func (m ringWaitMode) String() string { // 5.19 to 6.1 kernel does for a CQ ring it cannot flush (a kernel the CI and // the laptop do not run), or counts the waits to prove an injection reached // the loop. Several workers call it, so a test keeps its state atomic. Nil in -// production, where each wait costs one load of this variable. -var ringWaitSeam func(w *Worker, mode ringWaitMode, err error) error +// production, where each wait costs one load of this variable. An +// atomic.Pointer, as h2WriteDrainSeam is, because a worker an earlier test left +// running may still read it when a later test sets or restores it +// (celeris#1229). +var ringWaitSeam atomic.Pointer[func(w *Worker, mode ringWaitMode, err error) error] diff --git a/internal/engine/iouring/run_ebusy_884_linux_test.go b/internal/engine/iouring/run_ebusy_884_linux_test.go index 801c2d573..dede3685f 100644 --- a/internal/engine/iouring/run_ebusy_884_linux_test.go +++ b/internal/engine/iouring/run_ebusy_884_linux_test.go @@ -123,12 +123,15 @@ func ebusyErr884() error { // setRingWaitSeam884 installs the seam before the engine starts and removes it // after the engine has stopped: the cleanup is registered first, so it runs -// last, and the workers read the variable without a lock. +// last. The workers load the variable atomically, so a worker an earlier test +// left running cannot race with the restore (it would still be answered by the +// seam, which is why the engine helpers stop their engines when a test fails, +// celeris#1229). func setRingWaitSeam884(t *testing.T, fn func(*Worker, ringWaitMode, error) error) { t.Helper() - old := ringWaitSeam - ringWaitSeam = fn - t.Cleanup(func() { ringWaitSeam = old }) + old := ringWaitSeam.Load() + ringWaitSeam.Store(&fn) + t.Cleanup(func() { ringWaitSeam.Store(old) }) } // warmLoad884 dials n keep-alive connections, each answered once, so every @@ -358,7 +361,7 @@ func TestRingWaitScanSeesAnUnroutedWait884(t *testing.T) { const src = `package iouring func (w *Worker) run() { err := w.ring.SubmitAndWait() - if (err != nil || ringWaitSeam != nil) && !w.ringWaitContinues(1, err) { return } + if (err != nil || ringWaitSeam.Load() != nil) && !w.ringWaitContinues(1, err) { return } err = w.ring.SubmitAndWaitTimeout(1) if err != nil { w.shutdown(); return } _ = w.ring.WaitCQETimeout(1) diff --git a/internal/engine/iouring/worker.go b/internal/engine/iouring/worker.go index eb03825ec..be64d3533 100644 --- a/internal/engine/iouring/worker.go +++ b/internal/engine/iouring/worker.go @@ -1749,7 +1749,7 @@ func (w *Worker) run(ctx context.Context) { if cqHead == cqTail { // No CQEs — wait with adaptive timeout. err := w.ring.WaitCQETimeout(w.adaptiveTimeout()) - if (err != nil || ringWaitSeam != nil) && !w.ringWaitContinues(waitModeSQPoll, err) { + if (err != nil || ringWaitSeam.Load() != nil) && !w.ringWaitContinues(waitModeSQPoll, err) { w.shutdown() return } @@ -1782,7 +1782,7 @@ func (w *Worker) run(ctx context.Context) { // submission + CQE retrieval into one syscall (saves ~300ns vs // separate Submit + next-iteration wait). err := w.ring.SubmitAndWait() - if (err != nil || ringWaitSeam != nil) && !w.ringWaitContinues(waitModeSubmitReady, err) { + if (err != nil || ringWaitSeam.Load() != nil) && !w.ringWaitContinues(waitModeSubmitReady, err) { w.shutdown() return } @@ -1800,7 +1800,7 @@ func (w *Worker) run(ctx context.Context) { // worker's standing sweep timer (armSweepTick, celeris#884 // item 74) is what ends this wait, in about sweepCeiling. err := w.ring.SubmitAndWait() - if (err != nil || ringWaitSeam != nil) && !w.ringWaitContinues(waitModeSubmitSends, err) { + if (err != nil || ringWaitSeam.Load() != nil) && !w.ringWaitContinues(waitModeSubmitSends, err) { w.shutdown() return } @@ -1809,7 +1809,7 @@ func (w *Worker) run(ctx context.Context) { // Mode 3b: no guaranteed CQEs — wait with adaptive timeout for // shutdown checks and CQE_SKIP_SUCCESS operations. err := w.ring.SubmitAndWaitTimeout(w.adaptiveTimeout()) - if (err != nil || ringWaitSeam != nil) && !w.ringWaitContinues(waitModeTimed, err) { + if (err != nil || ringWaitSeam.Load() != nil) && !w.ringWaitContinues(waitModeTimed, err) { w.shutdown() return } @@ -2934,8 +2934,8 @@ func (w *Worker) closeListenerAfterDrain(ctx context.Context) { // completion is handled as the loop would (processCQE), so a connection the // accept delivered meanwhile is served. func (w *Worker) cancelAccept(ctx context.Context, lfd int) { - if h := cancelAcceptSeam; h != nil { - h(w, lfd) + if h := cancelAcceptSeam.Load(); h != nil { + (*h)(w, lfd) } sqe := w.getCancelSQE() if sqe == nil { @@ -8503,8 +8503,8 @@ func (w *Worker) stopDetachedProducers() { // idle one (a park, a slow callback in the gate pass, a long wait end it). Any other error is the ring // failing: the worker stops, and closes what it owns. func (w *Worker) ringWaitContinues(mode ringWaitMode, err error) bool { - if h := ringWaitSeam; h != nil { - err = h(w, mode, err) + if h := ringWaitSeam.Load(); h != nil { + err = (*h)(w, mode, err) } if err == nil { return true diff --git a/internal/engine/iouring/worker_exit_pause_lifecycle_linux_test.go b/internal/engine/iouring/worker_exit_pause_lifecycle_linux_test.go index b69b24072..023014626 100644 --- a/internal/engine/iouring/worker_exit_pause_lifecycle_linux_test.go +++ b/internal/engine/iouring/worker_exit_pause_lifecycle_linux_test.go @@ -249,12 +249,11 @@ func TestGoexitInThePauseCloseStillLeavesTheGroup848(t *testing.T) { e, _, cancel, done := start848(t, "127.0.0.1:0", true) addr := e.Addr().String() var fired atomic.Bool - cancelAcceptSeam = func(*Worker, int) { + setCancelAcceptSeam1229(t, func(*Worker, int) { if fired.CompareAndSwap(false, true) { runtime.Goexit() } - } - t.Cleanup(func() { cancelAcceptSeam = nil }) + }) t0 := time.Now() if err := e.PauseAccept(); err != nil { diff --git a/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go b/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go index 73d19dd24..3d4b888e2 100644 --- a/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go +++ b/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go @@ -83,11 +83,10 @@ func startGate1173(t *testing.T, g *gate1173) (*engineRun896, string) { skipOrFail656(t, "New: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- e.Listen(ctx) }() + done, returned := listenAsync896(e, ctx) for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); { if e.Addr() != nil { - run = &engineRun896{e: e, cancel: cancel, done: done} + run = &engineRun896{e: e, cancel: cancel, done: done, returned: returned} return nil } select { @@ -108,6 +107,9 @@ func startGate1173(t *testing.T, g *gate1173) (*engineRun896, string) { } t.Fatalf("engine start on %s: %v", addr, err) } + // Closing the gate frees a handler a failed test left waiting on it, which + // Listen would otherwise wait for; the tests only send on it. + run.reapAtCleanup(t, func() { close(g.gate) }) return run, addr } diff --git a/internal/engine/iouring/write_clock_stale_stamp_det_1173_linux_test.go b/internal/engine/iouring/write_clock_stale_stamp_det_1173_linux_test.go index 193386684..0a92de605 100644 --- a/internal/engine/iouring/write_clock_stale_stamp_det_1173_linux_test.go +++ b/internal/engine/iouring/write_clock_stale_stamp_det_1173_linux_test.go @@ -37,8 +37,7 @@ func TestWriteClockStampAfterAnEmptyWaitIsFreshDeterministic1173(t *testing.T) { visible atomic.Int64 resultCh = make(chan struct{}, 1) ) - oldWait := ringWaitSeam - ringWaitSeam = func(w *Worker, _ ringWaitMode, err error) error { + waitSeam := func(w *Worker, _ ringWaitMode, err error) error { if err != nil || !armed.Load() || len(w.h2Conns) == 0 { return err } @@ -71,8 +70,9 @@ func TestWriteClockStampAfterAnEmptyWaitIsFreshDeterministic1173(t *testing.T) { } } } + setRingWaitSeam884(t, waitSeam) h2WriteDrainSeam.Store(&seam) - t.Cleanup(func() { ringWaitSeam = oldWait; h2WriteDrainSeam.Store(nil) }) + t.Cleanup(func() { h2WriteDrainSeam.Store(nil) }) g := &gate1173{started: make(chan struct{}, 1), gate: make(chan struct{})} run, addr := startGate1173(t, g) From 1b731a3370b20ee289dcd0244cf81f90a97cf0ba Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 3 Nov 2026 00:18:09 +0100 Subject: [PATCH 3/9] test(iouring): context first in listenAsync896 (Refs #1229) --- internal/engine/iouring/listen_release_896_linux_test.go | 4 ++-- .../iouring/listen_release_stop_accepting_896_linux_test.go | 2 +- .../engine/iouring/write_clock_stale_stamp_1173_linux_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/engine/iouring/listen_release_896_linux_test.go b/internal/engine/iouring/listen_release_896_linux_test.go index 95b042c8d..3525a2778 100644 --- a/internal/engine/iouring/listen_release_896_linux_test.go +++ b/internal/engine/iouring/listen_release_896_linux_test.go @@ -45,7 +45,7 @@ type engineRun896 struct { // listenAsync896 runs e.Listen(ctx) on a goroutine of its own. done receives // its result; returned is closed after that, so a cleanup that waits on it // can still read done. -func listenAsync896(e *Engine, ctx context.Context) (done chan error, returned chan struct{}) { +func listenAsync896(ctx context.Context, e *Engine) (done chan error, returned chan struct{}) { done = make(chan error, 1) returned = make(chan struct{}) go func() { @@ -115,7 +115,7 @@ func start896Config(t *testing.T, addr string, workers int, disableDefer bool) * skipOrFail656(t, "New: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - done, returned := listenAsync896(e, ctx) + done, returned := listenAsync896(ctx, e) for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); { if e.Addr() != nil { run = &engineRun896{e: e, cancel: cancel, done: done, returned: returned} diff --git a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go index 6abec149c..de68c6488 100644 --- a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go +++ b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go @@ -123,7 +123,7 @@ func startHold896(t *testing.T, addr string, workers int, writeTimeout time.Dura skipOrFail656(t, "New: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - done, returned := listenAsync896(e, ctx) + done, returned := listenAsync896(ctx, e) for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); { if e.Addr() != nil { run = &engineRun896{e: e, cancel: cancel, done: done, returned: returned} diff --git a/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go b/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go index 3d4b888e2..705727678 100644 --- a/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go +++ b/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go @@ -83,7 +83,7 @@ func startGate1173(t *testing.T, g *gate1173) (*engineRun896, string) { skipOrFail656(t, "New: %v", err) } ctx, cancel := context.WithCancel(context.Background()) - done, returned := listenAsync896(e, ctx) + done, returned := listenAsync896(ctx, e) for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); { if e.Addr() != nil { run = &engineRun896{e: e, cancel: cancel, done: done, returned: returned} From 0539eb78e9675ebec6b543df8394924c2c3ff326 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 3 Nov 2026 00:53:10 +0100 Subject: [PATCH 4/9] test(iouring): a dial that times out in the 896 tests names the tier and the listen socket (Refs #1229) --- .../iouring/engine_cleanup_1229_linux_test.go | 19 ++++++ ...n_release_stop_accepting_896_linux_test.go | 58 ++++++++++++++++++- 2 files changed, 75 insertions(+), 2 deletions(-) diff --git a/internal/engine/iouring/engine_cleanup_1229_linux_test.go b/internal/engine/iouring/engine_cleanup_1229_linux_test.go index a351697f6..b6171d763 100644 --- a/internal/engine/iouring/engine_cleanup_1229_linux_test.go +++ b/internal/engine/iouring/engine_cleanup_1229_linux_test.go @@ -3,6 +3,8 @@ package iouring import ( + "net" + "strings" "testing" "time" ) @@ -152,3 +154,20 @@ func setCancelAcceptSeam1229(t *testing.T, fn func(*Worker, int)) { cancelAcceptSeam.Store(&fn) t.Cleanup(func() { cancelAcceptSeam.Store(nil) }) } + +// TestDialDiagNamesTheListenSocketOfAPort1229: the failure message of a dial +// that timed out says whether a LISTEN socket was there for it. +func TestDialDiagNamesTheListenSocketOfAPort1229(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + if got := dialDiag896(addr); !strings.Contains(got, "queues(tx:rx)=") { + t.Errorf("dialDiag896 of a live listener = %q, want its /proc/net/tcp row", got) + } + _ = ln.Close() + if got := dialDiag896(addr); !strings.Contains(got, "no LISTEN socket") { + t.Errorf("dialDiag896 of a closed port = %q, want it to say no LISTEN socket", got) + } +} diff --git a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go index de68c6488..48bfcab98 100644 --- a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go +++ b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go @@ -11,7 +11,9 @@ import ( "log/slog" "net" "os" + "path/filepath" "strconv" + "strings" "sync" "sync/atomic" "testing" @@ -154,16 +156,68 @@ func startHold896(t *testing.T, addr string, workers int, writeTimeout time.Dura release = hh.unblock } run.reapAtCleanup(t, release) + // Shown only if the test fails: a dial that times out on a Coverage run + // (celeris#1229) names no tier, and the engine's own log is discarded. + t.Logf("celeris1229 engine on %s: %s", addr, describeEngine896(run.e)) return run } +// describeEngine896 names the tier the engine's workers run on and the +// kernel's view of its listen sockets (dialDiag896), for the failure message +// of a dial that timed out. +func describeEngine896(e *Engine) string { + e.mu.Lock() + ws := e.workers + e.mu.Unlock() + tier := "no workers" + if len(ws) > 0 { + tier = ws[0].tier.Tier().String() + } + return fmt.Sprintf("tier=%s workers=%d", tier, len(ws)) +} + +// dialDiag896 returns the LISTEN rows /proc/net/tcp and tcp6 hold for addr's +// port: the receive queue of a listen socket is its accept queue, the transmit +// queue its backlog. A dial that timed out against a port with no row had no +// listener, one against a row whose queue is full had a listener nobody +// accepts from. +func dialDiag896(addr string) string { + _, ps, err := net.SplitHostPort(addr) + if err != nil { + return "no port in " + addr + } + port, _ := strconv.Atoi(ps) + var rows []string + for _, f := range []string{"/proc/net/tcp", "/proc/net/tcp6"} { + b, err := os.ReadFile(f) + if err != nil { + continue + } + for _, ln := range strings.Split(string(b), "\n")[1:] { + fl := strings.Fields(ln) + // sl local_address rem_address st tx_queue:rx_queue ... + if len(fl) < 5 || fl[3] != "0A" { + continue + } + if i := strings.LastIndexByte(fl[1], ':'); i < 0 || fl[1][i+1:] != fmt.Sprintf("%04X", port) { + continue + } + rows = append(rows, fmt.Sprintf("%s %s queues(tx:rx)=%s", filepath.Base(f), fl[1], fl[4])) + } + } + if len(rows) == 0 { + return fmt.Sprintf("no LISTEN socket on port %d in /proc/net/tcp{,6}", port) + } + return strings.Join(rows, "; ") +} + // h2Request896 opens an HTTP/2 connection to addr (prior knowledge) and sends // GET path on stream 1, without reading any response. func h2Request896(t *testing.T, addr, path string) net.Conn { t.Helper() c, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { - t.Fatalf("dial %s: %v", addr, err) + t.Fatalf("dial %s: %v (%s)", addr, err, dialDiag896(addr)) } _ = c.SetWriteDeadline(time.Now().Add(5 * time.Second)) if _, err := c.Write([]byte(http2.ClientPreface)); err != nil { @@ -317,7 +371,7 @@ func sendDrainIteration896(t *testing.T, calls *atomic.Int64) (string, time.Time run := startHold896(t, addr, 2, -1, h) c, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { - t.Fatalf("dial %s: %v", addr, err) + t.Fatalf("dial %s: %v (%s)", addr, err, dialDiag896(addr)) } defer func() { _ = c.Close() }() if _, err := c.Write([]byte("GET /big HTTP/1.1\r\nHost: x\r\n\r\n")); err != nil { From c0ba6fa42392b66b6461e84a14762876a36f22fe Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 3 Nov 2026 01:57:00 +0100 Subject: [PATCH 5/9] test(iouring): reap per iteration in the 896 loops, free a second gate handler, no hang when -run filters the race subtest (Refs #1229) --- .../iouring/engine_cleanup_1229_linux_test.go | 9 ++++- .../iouring/listen_release_896_linux_test.go | 36 ++++++++++++++++--- ...n_release_stop_accepting_896_linux_test.go | 2 ++ ...write_clock_stale_stamp_1173_linux_test.go | 7 +++- 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/internal/engine/iouring/engine_cleanup_1229_linux_test.go b/internal/engine/iouring/engine_cleanup_1229_linux_test.go index b6171d763..36d909ae3 100644 --- a/internal/engine/iouring/engine_cleanup_1229_linux_test.go +++ b/internal/engine/iouring/engine_cleanup_1229_linux_test.go @@ -106,13 +106,17 @@ func TestEngineStartHelpersStopTheirEngineWhenTheTestEnds1229(t *testing.T) { // restore goes on reading the seam through read: a worker that an earlier test // leaked. The restore is a write the race detector must not find unordered // with the read, so a seam read and written as a plain variable fails this -// test under -race ("race detected during execution of test"). +// test under -race ("race detected during execution of test"), in most runs +// and not in every one: the detector reports the races it observes, and the +// interleaving is not forced. func seamRestoredUnderALateReader1229(t *testing.T, install func(t *testing.T), read func()) { t.Helper() stop := make(chan struct{}) finished := make(chan struct{}) + started := false t.Run("seam", func(t *testing.T) { install(t) + started = true go func() { defer close(finished) for { @@ -125,6 +129,9 @@ func seamRestoredUnderALateReader1229(t *testing.T, install func(t *testing.T), } }() }) + if !started { // -run filtered the subtest out: no reader to wait for + return + } close(stop) <-finished } diff --git a/internal/engine/iouring/listen_release_896_linux_test.go b/internal/engine/iouring/listen_release_896_linux_test.go index 3525a2778..2056878b6 100644 --- a/internal/engine/iouring/listen_release_896_linux_test.go +++ b/internal/engine/iouring/listen_release_896_linux_test.go @@ -40,6 +40,7 @@ type engineRun896 struct { cancel context.CancelFunc done chan error // receives Listen's result returned chan struct{} // closed once done has it (listenAsync896) + reaper *reaper896 // set by reapAtCleanup } // listenAsync896 runs e.Listen(ctx) on a goroutine of its own. done receives @@ -65,16 +66,41 @@ func listenAsync896(ctx context.Context, e *Engine) (done chan error, returned c // restores of the seams the test installed earlier. func (r *engineRun896) reapAtCleanup(t *testing.T, release func()) { t.Helper() - t.Cleanup(func() { - if release != nil { - release() + r.reaper = &reaper896{cancel: r.cancel, returned: r.returned, release: release} + rp := r.reaper + t.Cleanup(func() { rp.reap(t) }) +} + +// reap is reapAtCleanup's stop, at once. A loop that starts an engine per +// iteration defers it, so that the engine, and the handler it holds (a 32 MiB +// response in the send-drain loops), are not kept until the test ends. +func (r *engineRun896) reap(t *testing.T) { + t.Helper() + r.reaper.reap(t) +} + +// reaper896 is what the cleanup holds of a run: not the engine, so that a +// finished iteration's engine is garbage. +type reaper896 struct { + cancel context.CancelFunc + returned chan struct{} + release func() + once sync.Once +} + +func (rp *reaper896) reap(t *testing.T) { + t.Helper() + rp.once.Do(func() { + if rp.release != nil { + rp.release() } - r.cancel() + rp.cancel() select { - case <-r.returned: + case <-rp.returned: case <-time.After(10 * time.Second): t.Errorf("Listen had not returned 10 s after the test ended: a worker is still running") } + rp.release = nil }) } diff --git a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go index 48bfcab98..b28d414c0 100644 --- a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go +++ b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go @@ -334,6 +334,7 @@ func h2Iteration896(t *testing.T, calls *atomic.Int64) (string, time.Time) { addr := freeAddr639(t) h := newHoldHandler896(false) run := startHold896(t, addr, 2, 0, h) + defer run.reap(t) c := h2Request896(t, addr, "/hold") defer func() { _ = c.Close() }() select { @@ -369,6 +370,7 @@ func sendDrainIteration896(t *testing.T, calls *atomic.Int64) (string, time.Time addr := freeAddr639(t) h := newHoldHandler896(true) run := startHold896(t, addr, 2, -1, h) + defer run.reap(t) c, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { t.Fatalf("dial %s: %v (%s)", addr, err, dialDiag896(addr)) diff --git a/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go b/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go index 705727678..57d27fc81 100644 --- a/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go +++ b/internal/engine/iouring/write_clock_stale_stamp_1173_linux_test.go @@ -48,7 +48,12 @@ func (g *gate1173) HandleStream(_ context.Context, s *stream.Stream) error { } for _, hh := range s.Headers { if hh[0] == ":path" && hh[1] == "/gate" { - g.started <- struct{}{} + // Never blocks: started holds one signal, and a handler after the + // first must still reach the gate, which the engine's cleanup closes. + select { + case g.started <- struct{}{}: + default: + } <-g.gate } } From bd9a3f3e02e5c2daf517885f445edcef39202af9 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 3 Nov 2026 01:58:43 +0100 Subject: [PATCH 6/9] test(iouring): a 896 dial timeout lists every socket of the port, the drop counters and the largest scheduling gap (Refs #1229) --- .../iouring/engine_cleanup_1229_linux_test.go | 34 +++- ...n_release_stop_accepting_896_linux_test.go | 173 +++++++++++++++--- 2 files changed, 178 insertions(+), 29 deletions(-) diff --git a/internal/engine/iouring/engine_cleanup_1229_linux_test.go b/internal/engine/iouring/engine_cleanup_1229_linux_test.go index 36d909ae3..db410292a 100644 --- a/internal/engine/iouring/engine_cleanup_1229_linux_test.go +++ b/internal/engine/iouring/engine_cleanup_1229_linux_test.go @@ -162,19 +162,39 @@ func setCancelAcceptSeam1229(t *testing.T, fn func(*Worker, int)) { t.Cleanup(func() { cancelAcceptSeam.Store(nil) }) } -// TestDialDiagNamesTheListenSocketOfAPort1229: the failure message of a dial -// that timed out says whether a LISTEN socket was there for it. -func TestDialDiagNamesTheListenSocketOfAPort1229(t *testing.T) { +// TestDialWatchDescribesTheSocketsOfAPort1229: the failure message of a dial +// that timed out lists every socket of the port, the client's included, with +// its state named, and the counters and the tick gap; it works on a dial that +// went through and on one that was refused. +func TestDialWatchDescribesTheSocketsOfAPort1229(t *testing.T) { ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } addr := ln.Addr().String() - if got := dialDiag896(addr); !strings.Contains(got, "queues(tx:rx)=") { - t.Errorf("dialDiag896 of a live listener = %q, want its /proc/net/tcp row", got) + w := watchDial896(addr) + c, err := net.DialTimeout("tcp", addr, 2*time.Second) + if err != nil { + t.Fatal(err) + } + time.Sleep(30 * time.Millisecond) // a tick or two + got := w.report() + t.Logf("report: %s", got) + for _, want := range []string{"LISTEN", "ESTABLISHED", "largest gap between 10 ms ticks", "ListenDrops+0", "softnet_drop+"} { + if !strings.Contains(got, want) { + t.Errorf("report of a dial that went through lacks %q: %s", want, got) + } } + _ = c.Close() _ = ln.Close() - if got := dialDiag896(addr); !strings.Contains(got, "no LISTEN socket") { - t.Errorf("dialDiag896 of a closed port = %q, want it to say no LISTEN socket", got) + + w = watchDial896(addr) + if c, err := net.DialTimeout("tcp", addr, 2*time.Second); err == nil { + _ = c.Close() + t.Skip("something listens on the port just closed") + } + if got := w.report(); !strings.Contains(got, "[none]") && strings.Contains(got, "LISTEN") { + t.Errorf("report of a dial to a closed port lists a LISTEN socket: %s", got) } + w.end() // a second end after report is harmless } diff --git a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go index b28d414c0..126fa5501 100644 --- a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go +++ b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go @@ -162,9 +162,9 @@ func startHold896(t *testing.T, addr string, workers int, writeTimeout time.Dura return run } -// describeEngine896 names the tier the engine's workers run on and the -// kernel's view of its listen sockets (dialDiag896), for the failure message -// of a dial that timed out. +// describeEngine896 names the tier the engine's workers run on, for the +// failure message of a dial that timed out (dialWatch896 gives the kernel's +// side). func describeEngine896(e *Engine) string { e.mu.Lock() ws := e.workers @@ -176,49 +176,176 @@ func describeEngine896(e *Engine) string { return fmt.Sprintf("tier=%s workers=%d", tier, len(ws)) } -// dialDiag896 returns the LISTEN rows /proc/net/tcp and tcp6 hold for addr's -// port: the receive queue of a listen socket is its accept queue, the transmit -// queue its backlog. A dial that timed out against a port with no row had no -// listener, one against a row whose queue is full had a listener nobody -// accepts from. -func dialDiag896(addr string) string { - _, ps, err := net.SplitHostPort(addr) - if err != nil { - return "no port in " + addr +// dialWatch896 watches what the kernel and the test process do while a test +// dials addr, for the failure message of a dial that times out (celeris#1229: +// 2 s with no answer from an engine that is listening). It reads /proc only, +// ignores every error, and never blocks: report is the whole of its cost. +// +// A dial that fails is one of two things. Either the handshake did not +// complete (the SYN or the SYN-ACK was dropped or not processed: the client +// has a SYN_SENT row, or none) or it completed and the test did not see it for +// 2 s (the client has an ESTABLISHED row, and the ticker below shows the +// process or the VM not running). The rows, the counters and the ticker tell +// them apart. +type dialWatch896 struct { + port int + netstat map[string]int64 // TcpExt counters when the watch began + softnet int64 // dropped packets, summed over the CPUs, when it began + stop chan struct{} + finished chan struct{} + maxGap atomic.Int64 // the largest gap between two ticks of 10 ms, in ns +} + +// dialCounters896 are the TcpExt counters of /proc/net/netstat that count a +// SYN or a connection a listen socket refused. +var dialCounters896 = []string{"ListenDrops", "ListenOverflows", "TCPReqQFullDrop", "TCPReqQFullDoCookies", "TCPDeferAcceptDrop"} + +func watchDial896(addr string) *dialWatch896 { + w := &dialWatch896{stop: make(chan struct{}), finished: make(chan struct{})} + if _, ps, err := net.SplitHostPort(addr); err == nil { + w.port, _ = strconv.Atoi(ps) } - port, _ := strconv.Atoi(ps) + w.netstat = tcpExtCounters896() + w.softnet = softnetDrops896() + started := make(chan struct{}) + go func() { + defer close(w.finished) + tk := time.NewTicker(10 * time.Millisecond) + defer tk.Stop() + last := time.Now() + close(started) + for { + select { + case <-w.stop: + return + case now := <-tk.C: + if gap := int64(now.Sub(last)); gap > w.maxGap.Load() { + w.maxGap.Store(gap) + } + last = now + } + } + }() + <-started + return w +} + +// end stops the ticker; a watch whose dial succeeded calls only this. +func (w *dialWatch896) end() { + select { + case <-w.stop: + default: + close(w.stop) + } + <-w.finished +} + +// report ends the watch and describes the dial that just failed. +func (w *dialWatch896) report() string { + w.end() + var d []string + now := tcpExtCounters896() + for _, k := range dialCounters896 { + if v, ok := now[k]; ok { + d = append(d, fmt.Sprintf("%s+%d", k, v-w.netstat[k])) + } + } + d = append(d, fmt.Sprintf("softnet_drop+%d", softnetDrops896()-w.softnet)) + return fmt.Sprintf("sockets on port %d: [%s]; counters: %s; largest gap between 10 ms ticks: %v", + w.port, strings.Join(tcpRows896(w.port), "; "), strings.Join(d, " "), + time.Duration(w.maxGap.Load()).Round(time.Millisecond)) +} + +// tcpStates896 names the st column of /proc/net/tcp. +var tcpStates896 = map[string]string{ + "01": "ESTABLISHED", "02": "SYN_SENT", "03": "SYN_RECV", "04": "FIN_WAIT1", "05": "FIN_WAIT2", + "06": "TIME_WAIT", "07": "CLOSE", "08": "CLOSE_WAIT", "09": "LAST_ACK", "0A": "LISTEN", "0B": "CLOSING", + "0C": "NEW_SYN_RECV", +} + +// tcpRows896 returns every row of /proc/net/tcp and tcp6 whose local or +// remote port is port, in any state, with its state named. For a LISTEN +// socket tx_queue is always 0 and rx_queue is its accept queue alone: a +// connection TCP_DEFER_ACCEPT holds back is in the request queue and is not +// counted, so an empty queue does not say the handshake failed. +func tcpRows896(port int) []string { var rows []string + want := fmt.Sprintf(":%04X", port) for _, f := range []string{"/proc/net/tcp", "/proc/net/tcp6"} { b, err := os.ReadFile(f) if err != nil { continue } - for _, ln := range strings.Split(string(b), "\n")[1:] { + for _, ln := range strings.Split(string(b), "\n") { fl := strings.Fields(ln) // sl local_address rem_address st tx_queue:rx_queue ... - if len(fl) < 5 || fl[3] != "0A" { + if len(fl) < 5 || fl[0] == "sl" || (!strings.HasSuffix(fl[1], want) && !strings.HasSuffix(fl[2], want)) { continue } - if i := strings.LastIndexByte(fl[1], ':'); i < 0 || fl[1][i+1:] != fmt.Sprintf("%04X", port) { - continue + st := tcpStates896[fl[3]] + if st == "" { + st = fl[3] } - rows = append(rows, fmt.Sprintf("%s %s queues(tx:rx)=%s", filepath.Base(f), fl[1], fl[4])) + rows = append(rows, fmt.Sprintf("%s %s>%s %s q(tx:rx)=%s", filepath.Base(f), fl[1], fl[2], st, fl[4])) } } if len(rows) == 0 { - return fmt.Sprintf("no LISTEN socket on port %d in /proc/net/tcp{,6}", port) + return []string{"none"} + } + return rows +} + +// tcpExtCounters896 parses the TcpExt pair of lines of /proc/net/netstat. +func tcpExtCounters896() map[string]int64 { + out := map[string]int64{} + b, err := os.ReadFile("/proc/net/netstat") + if err != nil { + return out + } + lines := strings.Split(string(b), "\n") + for i := 0; i+1 < len(lines); i++ { + if !strings.HasPrefix(lines[i], "TcpExt:") || !strings.HasPrefix(lines[i+1], "TcpExt:") { + continue + } + names, vals := strings.Fields(lines[i])[1:], strings.Fields(lines[i+1])[1:] + for j := 0; j < len(names) && j < len(vals); j++ { + if v, err := strconv.ParseInt(vals[j], 10, 64); err == nil { + out[names[j]] = v + } + } + break + } + return out +} + +// softnetDrops896 sums the second column (packets dropped for want of room in +// a CPU's backlog) of /proc/net/softnet_stat, whose values are hex. +func softnetDrops896() int64 { + b, err := os.ReadFile("/proc/net/softnet_stat") + if err != nil { + return 0 + } + var n int64 + for _, ln := range strings.Split(string(b), "\n") { + if fl := strings.Fields(ln); len(fl) > 1 { + if v, err := strconv.ParseInt(fl[1], 16, 64); err == nil { + n += v + } + } } - return strings.Join(rows, "; ") + return n } // h2Request896 opens an HTTP/2 connection to addr (prior knowledge) and sends // GET path on stream 1, without reading any response. func h2Request896(t *testing.T, addr, path string) net.Conn { t.Helper() + watch := watchDial896(addr) c, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { - t.Fatalf("dial %s: %v (%s)", addr, err, dialDiag896(addr)) + t.Fatalf("dial %s: %v (%s)", addr, err, watch.report()) } + watch.end() _ = c.SetWriteDeadline(time.Now().Add(5 * time.Second)) if _, err := c.Write([]byte(http2.ClientPreface)); err != nil { t.Fatalf("write preface: %v", err) @@ -371,10 +498,12 @@ func sendDrainIteration896(t *testing.T, calls *atomic.Int64) (string, time.Time h := newHoldHandler896(true) run := startHold896(t, addr, 2, -1, h) defer run.reap(t) + watch := watchDial896(addr) c, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { - t.Fatalf("dial %s: %v (%s)", addr, err, dialDiag896(addr)) + t.Fatalf("dial %s: %v (%s)", addr, err, watch.report()) } + watch.end() defer func() { _ = c.Close() }() if _, err := c.Write([]byte("GET /big HTTP/1.1\r\nHost: x\r\n\r\n")); err != nil { t.Fatalf("write: %v", err) From 6fc315cb6b0cf68e097e849f4f091107798733e0 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 3 Nov 2026 03:59:29 +0100 Subject: [PATCH 7/9] test(iouring): the dial watch keeps a near-deadline snapshot, measures tick gaps with the clock, and is tested on fixed /proc text (Refs #1229) --- .../iouring/dial_watch_1229_linux_test.go | 476 ++++++++++++++++++ .../iouring/engine_cleanup_1229_linux_test.go | 39 -- ...n_release_stop_accepting_896_linux_test.go | 170 +------ 3 files changed, 480 insertions(+), 205 deletions(-) create mode 100644 internal/engine/iouring/dial_watch_1229_linux_test.go diff --git a/internal/engine/iouring/dial_watch_1229_linux_test.go b/internal/engine/iouring/dial_watch_1229_linux_test.go new file mode 100644 index 000000000..5450bea0d --- /dev/null +++ b/internal/engine/iouring/dial_watch_1229_linux_test.go @@ -0,0 +1,476 @@ +//go:build linux + +package iouring + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" +) + +// dialTimeout896 is the timeout of the dials of the 896 tests that +// dialWatch896 watches. +const dialTimeout896 = 2 * time.Second + +// dialWatch896 watches what the kernel and the test process do while a test +// dials addr, for the failure message of a dial that times out (celeris#1229: +// 2 s with no answer from an engine that is listening). It reads /proc only, +// ignores every error, and never blocks: report is the whole of its cost. +// +// A dial that fails is one of two things. Either the handshake did not +// complete (the SYN or the SYN-ACK was dropped or not processed) or it +// completed and the test did not see it for 2 s. Go closes the socket of a +// failed dial before DialTimeout returns, so the client's row is gone when +// report reads /proc and the watch takes its own snapshots while the dial is +// still going (near): a client in SYN_SENT there, with the retransmit count +// of its row, is a SYN nobody answered. After the timeout (after) the rows say +// what the close left: no row but LISTEN, with nothing in SYN_RECV, is a +// listener that never saw the SYN; a client in FIN_WAIT or TIME_WAIT, a +// server-side ESTABLISHED, CLOSE_WAIT or SYN_RECV row, or a LISTEN row with +// rx_queue above 0, is a handshake that got through. For a LISTEN socket +// tx_queue is always 0 and rx_queue is its accept queue alone: a connection +// TCP_DEFER_ACCEPT holds back is in the request queue and is not counted. The +// counters say where a refused SYN was counted, and the largest gap between +// the ticks of a 10 ms ticker says whether the test process (or the VM) was +// not running for that long. +type dialWatch896 struct { + port int + now func() time.Time // time.Now; a test replaces it + timeout time.Duration + before procCounters896 + stop chan struct{} + finished chan struct{} + gaps gapTracker896 + + // written by the watch goroutine, read after finished is closed + near []string + nearAt time.Duration + backlogN int64 // softnet backlog length summed over the CPUs, at near +} + +// gapTracker896 records the largest gap between two ticks. It takes the time +// as an argument: the value a Go ticker delivers is the time the tick was +// scheduled for, so after a stall the first late tick carries a time 10 ms +// after the last and the gap would show as 10 ms. +type gapTracker896 struct { + last time.Time + max atomic.Int64 +} + +func (g *gapTracker896) tick(now time.Time) { + if gap := int64(now.Sub(g.last)); gap > g.max.Load() { + g.max.Store(gap) + } + g.last = now +} + +// procCounters896 are the counters a failed dial is compared on, read from +// /proc at a moment. +type procCounters896 struct { + ext map[string]int64 // TcpExt and IpExt of /proc/net/netstat, by name + tcp map[string]int64 // Tcp of /proc/net/snmp + dropped int64 // softnet_stat column 1, summed over the CPUs + squeeze int64 // softnet_stat column 2, summed over the CPUs +} + +// dialCounters896 name the counters whose deltas are reported, in the order +// they are printed: the TcpExt and IpExt ones of /proc/net/netstat first, the +// Tcp ones of /proc/net/snmp after. +var ( + dialExtCounters896 = []string{"ListenDrops", "ListenOverflows", "TCPReqQFullDrop", "TCPReqQFullDoCookies", + "TCPDeferAcceptDrop", "TCPBacklogDrop", "TCPMemoryPressures", "PFMemallocDrop", "InNoRoutes"} + dialTCPCounters896 = []string{"AttemptFails", "EstabResets", "RetransSegs", "InSegs", "InErrs"} +) + +func readProcCounters896() procCounters896 { + c := procCounters896{ext: map[string]int64{}, tcp: map[string]int64{}} + if b, err := os.ReadFile("/proc/net/netstat"); err == nil { + c.ext = parseProcPairs896(string(b), "TcpExt:", "IpExt:") + } + if b, err := os.ReadFile("/proc/net/snmp"); err == nil { + c.tcp = parseProcPairs896(string(b), "Tcp:") + } + if b, err := os.ReadFile("/proc/net/softnet_stat"); err == nil { + c.dropped = softnetSum896(string(b), softnetDroppedCol896) + c.squeeze = softnetSum896(string(b), softnetSqueezeCol896) + } + return c +} + +// deltas896 prints now - before for the named counters that exist, as +// "Name+n". +func deltas896(names []string, before, now map[string]int64) []string { + var out []string + for _, k := range names { + if v, ok := now[k]; ok { + out = append(out, fmt.Sprintf("%s+%d", k, v-before[k])) + } + } + return out +} + +func watchDial896(addr string, timeout time.Duration) *dialWatch896 { + return watchDialClock896(addr, timeout, time.Now) +} + +func watchDialClock896(addr string, timeout time.Duration, now func() time.Time) *dialWatch896 { + w := &dialWatch896{now: now, timeout: timeout, stop: make(chan struct{}), finished: make(chan struct{})} + if _, ps, err := net.SplitHostPort(addr); err == nil { + w.port, _ = strconv.Atoi(ps) + } + w.before = readProcCounters896() + started := make(chan struct{}) + go func() { + defer close(w.finished) + tk := time.NewTicker(10 * time.Millisecond) + defer tk.Stop() + snap := time.NewTicker(250 * time.Millisecond) + defer snap.Stop() + t0 := now() + w.gaps.last = t0 + close(started) + for { + select { + case <-w.stop: + // The stall that ends the dial is the gap since the last tick. + w.gaps.tick(now()) + return + case <-tk.C: + w.gaps.tick(now()) + case <-snap.C: + // From half the timeout on, keep the last snapshot with a row: + // the client's socket is closed before the dial returns. + if at := now().Sub(t0); at >= timeout/2 { + if rows := tcpRows896(w.port); len(rows) > 0 { + w.near, w.nearAt = rows, at + w.backlogN = softnetBacklog896() + } + } + } + } + }() + <-started + return w +} + +// end stops the watch; a watch whose dial succeeded calls only this. +func (w *dialWatch896) end() { + select { + case <-w.stop: + default: + close(w.stop) + } + <-w.finished +} + +// maxRows896 bounds the rows one snapshot prints. +const maxRows896 = 8 + +func boundRows896(rows []string) string { + if len(rows) == 0 { + return "none" + } + if len(rows) > maxRows896 { + return strings.Join(rows[:maxRows896], "; ") + fmt.Sprintf("; (+%d more)", len(rows)-maxRows896) + } + return strings.Join(rows, "; ") +} + +// report ends the watch and describes the dial that just failed. +func (w *dialWatch896) report() string { + w.end() + now := readProcCounters896() + d := deltas896(dialExtCounters896, w.before.ext, now.ext) + d = append(d, deltas896(dialTCPCounters896, w.before.tcp, now.tcp)...) + d = append(d, fmt.Sprintf("softnet_drop+%d softnet_squeeze+%d", now.dropped-w.before.dropped, now.squeeze-w.before.squeeze)) + near := "no snapshot" + if w.near != nil { + near = fmt.Sprintf("at %v: [%s] syn_recv=%d softnet_backlog_len=%d", w.nearAt.Round(10*time.Millisecond), + boundRows896(w.near), countState896(w.near, "SYN_RECV"), w.backlogN) + } + return fmt.Sprintf("sockets on port %d: near the deadline %s; after the timeout [%s]; counters: %s; "+ + "largest gap between 10 ms ticks: %v", w.port, near, boundRows896(tcpRows896(w.port)), strings.Join(d, " "), + time.Duration(w.gaps.max.Load()).Round(time.Millisecond)) +} + +// countState896 counts the rows of tcpRows896 in the named state. +func countState896(rows []string, state string) int { + n := 0 + for _, r := range rows { + if strings.Contains(r, " "+state+" ") { + n++ + } + } + return n +} + +// tcpStates896 names the st column of /proc/net/tcp (the kernel prints a +// request socket as 03, SYN_RECV). +var tcpStates896 = map[string]string{ + "01": "ESTABLISHED", "02": "SYN_SENT", "03": "SYN_RECV", "04": "FIN_WAIT1", "05": "FIN_WAIT2", + "06": "TIME_WAIT", "07": "CLOSE", "08": "CLOSE_WAIT", "09": "LAST_ACK", "0A": "LISTEN", "0B": "CLOSING", + "0C": "NEW_SYN_RECV", +} + +// tcpRows896 returns every row of /proc/net/tcp and tcp6 whose local or +// remote port is port, in any state, with its state named. +func tcpRows896(port int) []string { + var rows []string + for _, f := range []string{"/proc/net/tcp", "/proc/net/tcp6"} { + if b, err := os.ReadFile(f); err == nil { + rows = append(rows, tcpRowsOf896(filepath.Base(f), string(b), port)...) + } + } + return rows +} + +// tcpRowsOf896 is tcpRows896 on the text of one file, tag its name. +func tcpRowsOf896(tag, text string, port int) []string { + var rows []string + want := fmt.Sprintf(":%04X", port) + for _, ln := range strings.Split(text, "\n") { + fl := strings.Fields(ln) + // sl local_address rem_address st tx_queue:rx_queue tr:tm->when retrnsmt ... + if len(fl) < 5 || fl[0] == "sl" || (!strings.HasSuffix(fl[1], want) && !strings.HasSuffix(fl[2], want)) { + continue + } + st := tcpStates896[fl[3]] + if st == "" { + st = fl[3] + } + row := fmt.Sprintf("%s %s>%s %s q(tx:rx)=%s", tag, fl[1], fl[2], st, fl[4]) + if len(fl) > 6 { // retrnsmt, hex: the SYN (or SYN-ACK) was sent again + if n, err := strconv.ParseInt(fl[6], 16, 64); err == nil && n != 0 { + row += fmt.Sprintf(" retrnsmt=%d", n) + } + } + rows = append(rows, row) + } + return rows +} + +// parseProcPairs896 parses the pairs of lines of /proc/net/netstat and +// /proc/net/snmp: "Prefix: Name Name ..." followed by "Prefix: n n ...". The +// first pair of each prefix is read; a name present under two prefixes is +// taken from the later one asked for. +func parseProcPairs896(text string, prefixes ...string) map[string]int64 { + out := map[string]int64{} + lines := strings.Split(text, "\n") + for _, prefix := range prefixes { + for i := 0; i+1 < len(lines); i++ { + h, v := strings.Fields(lines[i]), strings.Fields(lines[i+1]) + if len(h) == 0 || len(v) == 0 || h[0] != prefix || v[0] != prefix { + continue + } + for j := 1; j < len(h) && j < len(v); j++ { + if n, err := strconv.ParseInt(v[j], 10, 64); err == nil { + out[h[j]] = n + } + } + break + } + } + return out +} + +// the columns of /proc/net/softnet_stat the report reads (0-based). +const ( + softnetDroppedCol896 = 1 // packets dropped for want of room in the CPU's backlog + softnetSqueezeCol896 = 2 // softirq runs that ran out of budget + softnetBacklogCol896 = 14 // length of the CPU's backlog queue, kernels from 5.10 +) + +// softnetSum896 sums column col (0-based; the values are hex) of +// /proc/net/softnet_stat over the CPUs: 1 is the packets dropped for want of +// room in the CPU's backlog, 2 the softirq runs that ran out of budget. +func softnetSum896(text string, col int) int64 { + var n int64 + for _, ln := range strings.Split(text, "\n") { + if fl := strings.Fields(ln); len(fl) > col { + if v, err := strconv.ParseInt(fl[col], 16, 64); err == nil { + n += v + } + } + } + return n +} + +// softnetBacklog896 sums the backlog column of /proc/net/softnet_stat (the length of +// the CPU's backlog queue, kernels from 5.10) over the CPUs: a SYN that sits +// unprocessed shows here. +func softnetBacklog896() int64 { + b, err := os.ReadFile("/proc/net/softnet_stat") + if err != nil { + return 0 + } + return softnetSum896(string(b), softnetBacklogCol896) +} + +// fixtures of the /proc files, with values that tell the columns apart. +const ( + netstatFixture896 = "TcpExt: SyncookiesSent ListenOverflows ListenDrops TCPReqQFullDrop\n" + + "TcpExt: 1 22 333 4444\n" + + "IpExt: InNoRoutes InOctets\n" + + "IpExt: 5 66\n" + + "MPTcpExt: ListenDrops\nMPTcpExt: 9999\n" + snmpFixture896 = "Ip: Forwarding\nIp: 1\n" + + "Tcp: RtoAlgorithm AttemptFails EstabResets RetransSegs\n" + + "Tcp: 1 7 8 9\n" + // two CPUs: processed dropped time_squeeze ... (15 columns, hex) + softnetFixture896 = "0000aaaa 00000010 00000003 0 0 0 0 0 0 0 0 0 0 0 00000002 00000000\n" + + "0000bbbb 00000020 00000005 0 0 0 0 0 0 0 0 0 0 0 00000004 00000001\n" +) + +// tcpRowFixture896 is a /proc/net/tcp row whose local port is 0x1F90 (8080), +// in the state st, with the retransmit count rt. +func tcpRowFixture896(st string, rt int) string { + return fmt.Sprintf(" 0: 0100007F:1F90 0100007F:C350 %s 00000000:00000001 00:00000000 %08X 0 0 12345 1 0000000000000000\n", st, rt) +} + +// TestDialWatchReadsTheProcFiles1229 pins what the report says with fixed +// text, so that a column or a name read off by one, a state name swapped or a +// delta that is always 0 is a failure here and not a wrong report in a +// failed CI run. +func TestDialWatchReadsTheProcFiles1229(t *testing.T) { + const hdr = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" + for st, name := range map[string]string{"01": "ESTABLISHED", "02": "SYN_SENT", "03": "SYN_RECV", "04": "FIN_WAIT1", + "05": "FIN_WAIT2", "06": "TIME_WAIT", "07": "CLOSE", "08": "CLOSE_WAIT", "09": "LAST_ACK", "0A": "LISTEN", "0B": "CLOSING"} { + rows := tcpRowsOf896("tcp", hdr+tcpRowFixture896(st, 0), 8080) + want := "tcp 0100007F:1F90>0100007F:C350 " + name + " q(tx:rx)=00000000:00000001" + if len(rows) != 1 || !strings.HasPrefix(rows[0], want) { + t.Errorf("state %s: rows = %q, want one starting %q", st, rows, want) + } + } + if rows := tcpRowsOf896("tcp", hdr+tcpRowFixture896("02", 3), 8080); len(rows) != 1 || !strings.HasSuffix(rows[0], " retrnsmt=3") { + t.Errorf("a retransmitted SYN_SENT row = %q, want retrnsmt=3", rows) + } + // The remote port counts too (the client's row), a port that is neither does not. + if rows := tcpRowsOf896("tcp", hdr+tcpRowFixture896("01", 0), 0xC350); len(rows) != 1 { + t.Errorf("rows of the remote port = %q, want the one row", rows) + } + if rows := tcpRowsOf896("tcp", hdr+tcpRowFixture896("01", 0), 8081); len(rows) != 0 { + t.Errorf("rows of another port = %q, want none", rows) + } + if got := countState896([]string{"tcp a>b SYN_RECV q(tx:rx)=0:0", "tcp c>d LISTEN q(tx:rx)=0:0"}, "SYN_RECV"); got != 1 { + t.Errorf("countState896 = %d, want 1", got) + } + + ext := parseProcPairs896(netstatFixture896, "TcpExt:", "IpExt:") + for k, want := range map[string]int64{"SyncookiesSent": 1, "ListenOverflows": 22, "ListenDrops": 333, "TCPReqQFullDrop": 4444, "InNoRoutes": 5, "InOctets": 66} { + if ext[k] != want { + t.Errorf("netstat %s = %d, want %d", k, ext[k], want) + } + } + tcp := parseProcPairs896(snmpFixture896, "Tcp:") + for k, want := range map[string]int64{"AttemptFails": 7, "EstabResets": 8, "RetransSegs": 9} { + if tcp[k] != want { + t.Errorf("snmp Tcp %s = %d, want %d", k, tcp[k], want) + } + } + if got := softnetSum896(softnetFixture896, softnetDroppedCol896); got != 0x30 { + t.Errorf("softnet dropped = %#x, want 0x30", got) + } + if got := softnetSum896(softnetFixture896, softnetSqueezeCol896); got != 8 { + t.Errorf("softnet squeeze = %d, want 8", got) + } + if got := softnetSum896(softnetFixture896, softnetBacklogCol896); got != 6 { + t.Errorf("softnet backlog length = %d, want 6", got) + } + + before := map[string]int64{"ListenDrops": 300, "ListenOverflows": 22} + got := strings.Join(deltas896([]string{"ListenDrops", "ListenOverflows", "Absent"}, before, ext), " ") + if want := "ListenDrops+33 ListenOverflows+0"; got != want { + t.Errorf("deltas = %q, want %q", got, want) + } +} + +// TestDialWatchGapSeesAStallThatSpansTheDeadline1229: the gap is measured +// with the clock and not with the value the ticker delivers (the time the tick +// was scheduled for, 10 ms after the last), and the stop counts the gap since +// the last tick: a stall that began before the dial's deadline ends with the +// deadline, and the next tick comes after the report. +func TestDialWatchGapSeesAStallThatSpansTheDeadline1229(t *testing.T) { + var g gapTracker896 + t0 := time.Unix(1000, 0) + g.last = t0 + g.tick(t0.Add(10 * time.Millisecond)) + g.tick(t0.Add(20 * time.Millisecond)) + g.tick(t0.Add(1520 * time.Millisecond)) + g.tick(t0.Add(1530 * time.Millisecond)) + if got := time.Duration(g.max.Load()); got != 1500*time.Millisecond { + t.Errorf("largest gap = %v, want 1.5s", got) + } + + // The watch itself: its clock jumps while the (real) ticker runs, and then + // the watch is stopped; both the tick and the stop take the clock's time. + for _, onStop := range []bool{false, true} { + var offset atomic.Int64 + clock := func() time.Time { return time.Now().Add(time.Duration(offset.Load())) } + w := watchDialClock896("127.0.0.1:1", time.Second, clock) + offset.Store(int64(1500 * time.Millisecond)) + if !onStop { + time.Sleep(60 * time.Millisecond) // a few ticks after the jump + if got := time.Duration(w.gaps.max.Load()); got < 1500*time.Millisecond { + t.Errorf("stall seen on a tick: largest gap before the stop = %v, want at least 1.5s", got) + } + } + w.end() + if got := time.Duration(w.gaps.max.Load()); got < 1500*time.Millisecond { + t.Errorf("stall seen on a tick (onStop=%v): largest gap = %v, want at least 1.5s", onStop, got) + } + } +} + +// TestDialWatchDescribesTheSocketsOfAPort1229: the failure message of a dial +// that timed out lists the sockets of the port with their states, the +// counters and the tick gap, whether the dial went through or not, and the +// snapshot taken during a dial that nobody answers holds the client in +// SYN_SENT, which the report after the timeout cannot (Go closed the socket). +func TestDialWatchDescribesTheSocketsOfAPort1229(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + w := watchDial896(addr, dialTimeout896) + c, err := net.DialTimeout("tcp", addr, dialTimeout896) + if err != nil { + t.Fatal(err) + } + time.Sleep(30 * time.Millisecond) // a tick or two + got := w.report() + t.Logf("report: %s", got) + // The counters are namespace-wide, so only that they are there is asserted. + for _, want := range []string{"LISTEN", "ESTABLISHED", "no snapshot", "largest gap between 10 ms ticks", "ListenDrops+", "softnet_drop+", "RetransSegs+"} { + if !strings.Contains(got, want) { + t.Errorf("report of a dial that went through lacks %q: %s", want, got) + } + } + _ = c.Close() + _ = ln.Close() + + // A dial nobody answers: a reserved address, whose SYN goes unanswered. + const unanswered = "10.255.255.1:41234" + w = watchDial896(unanswered, 600*time.Millisecond) + t0 := time.Now() + c, err = net.DialTimeout("tcp", unanswered, 600*time.Millisecond) + if err == nil { + _ = c.Close() + t.Skip("something answers 10.255.255.1") + } + if time.Since(t0) < 500*time.Millisecond { + t.Skipf("the dial to %s failed at once, not on its timeout: %v", unanswered, err) + } + got = w.report() + t.Logf("report: %s", got) + if !strings.Contains(got, "SYN_SENT") || strings.Contains(got, "no snapshot") { + t.Errorf("report of an unanswered dial lacks the SYN_SENT row of the snapshot: %s", got) + } + w.end() // a second end after report is harmless +} diff --git a/internal/engine/iouring/engine_cleanup_1229_linux_test.go b/internal/engine/iouring/engine_cleanup_1229_linux_test.go index db410292a..6800e60e8 100644 --- a/internal/engine/iouring/engine_cleanup_1229_linux_test.go +++ b/internal/engine/iouring/engine_cleanup_1229_linux_test.go @@ -3,8 +3,6 @@ package iouring import ( - "net" - "strings" "testing" "time" ) @@ -161,40 +159,3 @@ func setCancelAcceptSeam1229(t *testing.T, fn func(*Worker, int)) { cancelAcceptSeam.Store(&fn) t.Cleanup(func() { cancelAcceptSeam.Store(nil) }) } - -// TestDialWatchDescribesTheSocketsOfAPort1229: the failure message of a dial -// that timed out lists every socket of the port, the client's included, with -// its state named, and the counters and the tick gap; it works on a dial that -// went through and on one that was refused. -func TestDialWatchDescribesTheSocketsOfAPort1229(t *testing.T) { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - addr := ln.Addr().String() - w := watchDial896(addr) - c, err := net.DialTimeout("tcp", addr, 2*time.Second) - if err != nil { - t.Fatal(err) - } - time.Sleep(30 * time.Millisecond) // a tick or two - got := w.report() - t.Logf("report: %s", got) - for _, want := range []string{"LISTEN", "ESTABLISHED", "largest gap between 10 ms ticks", "ListenDrops+0", "softnet_drop+"} { - if !strings.Contains(got, want) { - t.Errorf("report of a dial that went through lacks %q: %s", want, got) - } - } - _ = c.Close() - _ = ln.Close() - - w = watchDial896(addr) - if c, err := net.DialTimeout("tcp", addr, 2*time.Second); err == nil { - _ = c.Close() - t.Skip("something listens on the port just closed") - } - if got := w.report(); !strings.Contains(got, "[none]") && strings.Contains(got, "LISTEN") { - t.Errorf("report of a dial to a closed port lists a LISTEN socket: %s", got) - } - w.end() // a second end after report is harmless -} diff --git a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go index 126fa5501..f3168a56b 100644 --- a/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go +++ b/internal/engine/iouring/listen_release_stop_accepting_896_linux_test.go @@ -11,9 +11,7 @@ import ( "log/slog" "net" "os" - "path/filepath" "strconv" - "strings" "sync" "sync/atomic" "testing" @@ -176,172 +174,12 @@ func describeEngine896(e *Engine) string { return fmt.Sprintf("tier=%s workers=%d", tier, len(ws)) } -// dialWatch896 watches what the kernel and the test process do while a test -// dials addr, for the failure message of a dial that times out (celeris#1229: -// 2 s with no answer from an engine that is listening). It reads /proc only, -// ignores every error, and never blocks: report is the whole of its cost. -// -// A dial that fails is one of two things. Either the handshake did not -// complete (the SYN or the SYN-ACK was dropped or not processed: the client -// has a SYN_SENT row, or none) or it completed and the test did not see it for -// 2 s (the client has an ESTABLISHED row, and the ticker below shows the -// process or the VM not running). The rows, the counters and the ticker tell -// them apart. -type dialWatch896 struct { - port int - netstat map[string]int64 // TcpExt counters when the watch began - softnet int64 // dropped packets, summed over the CPUs, when it began - stop chan struct{} - finished chan struct{} - maxGap atomic.Int64 // the largest gap between two ticks of 10 ms, in ns -} - -// dialCounters896 are the TcpExt counters of /proc/net/netstat that count a -// SYN or a connection a listen socket refused. -var dialCounters896 = []string{"ListenDrops", "ListenOverflows", "TCPReqQFullDrop", "TCPReqQFullDoCookies", "TCPDeferAcceptDrop"} - -func watchDial896(addr string) *dialWatch896 { - w := &dialWatch896{stop: make(chan struct{}), finished: make(chan struct{})} - if _, ps, err := net.SplitHostPort(addr); err == nil { - w.port, _ = strconv.Atoi(ps) - } - w.netstat = tcpExtCounters896() - w.softnet = softnetDrops896() - started := make(chan struct{}) - go func() { - defer close(w.finished) - tk := time.NewTicker(10 * time.Millisecond) - defer tk.Stop() - last := time.Now() - close(started) - for { - select { - case <-w.stop: - return - case now := <-tk.C: - if gap := int64(now.Sub(last)); gap > w.maxGap.Load() { - w.maxGap.Store(gap) - } - last = now - } - } - }() - <-started - return w -} - -// end stops the ticker; a watch whose dial succeeded calls only this. -func (w *dialWatch896) end() { - select { - case <-w.stop: - default: - close(w.stop) - } - <-w.finished -} - -// report ends the watch and describes the dial that just failed. -func (w *dialWatch896) report() string { - w.end() - var d []string - now := tcpExtCounters896() - for _, k := range dialCounters896 { - if v, ok := now[k]; ok { - d = append(d, fmt.Sprintf("%s+%d", k, v-w.netstat[k])) - } - } - d = append(d, fmt.Sprintf("softnet_drop+%d", softnetDrops896()-w.softnet)) - return fmt.Sprintf("sockets on port %d: [%s]; counters: %s; largest gap between 10 ms ticks: %v", - w.port, strings.Join(tcpRows896(w.port), "; "), strings.Join(d, " "), - time.Duration(w.maxGap.Load()).Round(time.Millisecond)) -} - -// tcpStates896 names the st column of /proc/net/tcp. -var tcpStates896 = map[string]string{ - "01": "ESTABLISHED", "02": "SYN_SENT", "03": "SYN_RECV", "04": "FIN_WAIT1", "05": "FIN_WAIT2", - "06": "TIME_WAIT", "07": "CLOSE", "08": "CLOSE_WAIT", "09": "LAST_ACK", "0A": "LISTEN", "0B": "CLOSING", - "0C": "NEW_SYN_RECV", -} - -// tcpRows896 returns every row of /proc/net/tcp and tcp6 whose local or -// remote port is port, in any state, with its state named. For a LISTEN -// socket tx_queue is always 0 and rx_queue is its accept queue alone: a -// connection TCP_DEFER_ACCEPT holds back is in the request queue and is not -// counted, so an empty queue does not say the handshake failed. -func tcpRows896(port int) []string { - var rows []string - want := fmt.Sprintf(":%04X", port) - for _, f := range []string{"/proc/net/tcp", "/proc/net/tcp6"} { - b, err := os.ReadFile(f) - if err != nil { - continue - } - for _, ln := range strings.Split(string(b), "\n") { - fl := strings.Fields(ln) - // sl local_address rem_address st tx_queue:rx_queue ... - if len(fl) < 5 || fl[0] == "sl" || (!strings.HasSuffix(fl[1], want) && !strings.HasSuffix(fl[2], want)) { - continue - } - st := tcpStates896[fl[3]] - if st == "" { - st = fl[3] - } - rows = append(rows, fmt.Sprintf("%s %s>%s %s q(tx:rx)=%s", filepath.Base(f), fl[1], fl[2], st, fl[4])) - } - } - if len(rows) == 0 { - return []string{"none"} - } - return rows -} - -// tcpExtCounters896 parses the TcpExt pair of lines of /proc/net/netstat. -func tcpExtCounters896() map[string]int64 { - out := map[string]int64{} - b, err := os.ReadFile("/proc/net/netstat") - if err != nil { - return out - } - lines := strings.Split(string(b), "\n") - for i := 0; i+1 < len(lines); i++ { - if !strings.HasPrefix(lines[i], "TcpExt:") || !strings.HasPrefix(lines[i+1], "TcpExt:") { - continue - } - names, vals := strings.Fields(lines[i])[1:], strings.Fields(lines[i+1])[1:] - for j := 0; j < len(names) && j < len(vals); j++ { - if v, err := strconv.ParseInt(vals[j], 10, 64); err == nil { - out[names[j]] = v - } - } - break - } - return out -} - -// softnetDrops896 sums the second column (packets dropped for want of room in -// a CPU's backlog) of /proc/net/softnet_stat, whose values are hex. -func softnetDrops896() int64 { - b, err := os.ReadFile("/proc/net/softnet_stat") - if err != nil { - return 0 - } - var n int64 - for _, ln := range strings.Split(string(b), "\n") { - if fl := strings.Fields(ln); len(fl) > 1 { - if v, err := strconv.ParseInt(fl[1], 16, 64); err == nil { - n += v - } - } - } - return n -} - // h2Request896 opens an HTTP/2 connection to addr (prior knowledge) and sends // GET path on stream 1, without reading any response. func h2Request896(t *testing.T, addr, path string) net.Conn { t.Helper() - watch := watchDial896(addr) - c, err := net.DialTimeout("tcp", addr, 2*time.Second) + watch := watchDial896(addr, dialTimeout896) + c, err := net.DialTimeout("tcp", addr, dialTimeout896) if err != nil { t.Fatalf("dial %s: %v (%s)", addr, err, watch.report()) } @@ -498,8 +336,8 @@ func sendDrainIteration896(t *testing.T, calls *atomic.Int64) (string, time.Time h := newHoldHandler896(true) run := startHold896(t, addr, 2, -1, h) defer run.reap(t) - watch := watchDial896(addr) - c, err := net.DialTimeout("tcp", addr, 2*time.Second) + watch := watchDial896(addr, dialTimeout896) + c, err := net.DialTimeout("tcp", addr, dialTimeout896) if err != nil { t.Fatalf("dial %s: %v (%s)", addr, err, watch.report()) } From 2ffc13f6e6a0049cf2d68c16e29d1f2709392149 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 3 Nov 2026 05:12:45 +0100 Subject: [PATCH 8/9] test(iouring): read the softnet backlog from column 11, pin the retransmit base and the snapshot threshold, poll instead of sleeping (Refs #1229) --- .../iouring/dial_watch_1229_linux_test.go | 58 ++++++++++++++----- 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/internal/engine/iouring/dial_watch_1229_linux_test.go b/internal/engine/iouring/dial_watch_1229_linux_test.go index 5450bea0d..e325ac8bc 100644 --- a/internal/engine/iouring/dial_watch_1229_linux_test.go +++ b/internal/engine/iouring/dial_watch_1229_linux_test.go @@ -283,7 +283,9 @@ func parseProcPairs896(text string, prefixes ...string) map[string]int64 { const ( softnetDroppedCol896 = 1 // packets dropped for want of room in the CPU's backlog softnetSqueezeCol896 = 2 // softirq runs that ran out of budget - softnetBacklogCol896 = 14 // length of the CPU's backlog queue, kernels from 5.10 + softnetBacklogCol896 = 11 // the CPU's backlog length, input and process queues together, kernels from 5.10 + // (from 6.7, 13 is the input queue alone and 14 the process queue, the part + // process_backlog has already taken: a SYN waiting for the softirq is not in it) ) // softnetSum896 sums column col (0-based; the values are hex) of @@ -301,9 +303,9 @@ func softnetSum896(text string, col int) int64 { return n } -// softnetBacklog896 sums the backlog column of /proc/net/softnet_stat (the length of -// the CPU's backlog queue, kernels from 5.10) over the CPUs: a SYN that sits -// unprocessed shows here. +// softnetBacklog896 sums the backlog column of /proc/net/softnet_stat (the +// length of the CPU's backlog queue, input and process queues together) over +// the CPUs: a SYN that sits unprocessed shows here. func softnetBacklog896() int64 { b, err := os.ReadFile("/proc/net/softnet_stat") if err != nil { @@ -322,9 +324,14 @@ const ( snmpFixture896 = "Ip: Forwarding\nIp: 1\n" + "Tcp: RtoAlgorithm AttemptFails EstabResets RetransSegs\n" + "Tcp: 1 7 8 9\n" - // two CPUs: processed dropped time_squeeze ... (15 columns, hex) - softnetFixture896 = "0000aaaa 00000010 00000003 0 0 0 0 0 0 0 0 0 0 0 00000002 00000000\n" + - "0000bbbb 00000020 00000005 0 0 0 0 0 0 0 0 0 0 0 00000004 00000001\n" + // two CPUs, hex, as net/core/net-procfs.c prints them: processed, dropped, + // time_squeeze, five zero columns, cpu_collision 0, received_rps, + // flow_limit_count, the backlog length (11), the CPU index (12), and from + // kernel 6.7 the input queue (13) and the process queue (14) the length is + // the sum of. The first row is of that kernel (backlog 5 = 3 + 2), the + // second of 5.10 to 6.6, 13 columns (backlog 4). + softnetFixture896 = "0000aaaa 00000010 00000003 0 0 0 0 0 0 00000007 00000009 00000005 00000000 00000003 00000002\n" + + "0000bbbb 00000020 00000005 0 0 0 0 0 0 00000008 0000000a 00000004 00000001\n" ) // tcpRowFixture896 is a /proc/net/tcp row whose local port is 0x1F90 (8080), @@ -347,8 +354,8 @@ func TestDialWatchReadsTheProcFiles1229(t *testing.T) { t.Errorf("state %s: rows = %q, want one starting %q", st, rows, want) } } - if rows := tcpRowsOf896("tcp", hdr+tcpRowFixture896("02", 3), 8080); len(rows) != 1 || !strings.HasSuffix(rows[0], " retrnsmt=3") { - t.Errorf("a retransmitted SYN_SENT row = %q, want retrnsmt=3", rows) + if rows := tcpRowsOf896("tcp", hdr+tcpRowFixture896("02", 12), 8080); len(rows) != 1 || !strings.HasSuffix(rows[0], " retrnsmt=12") { + t.Errorf("a retransmitted SYN_SENT row = %q, want retrnsmt=12", rows) } // The remote port counts too (the client's row), a port that is neither does not. if rows := tcpRowsOf896("tcp", hdr+tcpRowFixture896("01", 0), 0xC350); len(rows) != 1 { @@ -379,8 +386,8 @@ func TestDialWatchReadsTheProcFiles1229(t *testing.T) { if got := softnetSum896(softnetFixture896, softnetSqueezeCol896); got != 8 { t.Errorf("softnet squeeze = %d, want 8", got) } - if got := softnetSum896(softnetFixture896, softnetBacklogCol896); got != 6 { - t.Errorf("softnet backlog length = %d, want 6", got) + if got := softnetSum896(softnetFixture896, softnetBacklogCol896); got != 9 { + t.Errorf("softnet backlog length = %d, want 9 (5 + 4)", got) } before := map[string]int64{"ListenDrops": 300, "ListenOverflows": 22} @@ -415,7 +422,10 @@ func TestDialWatchGapSeesAStallThatSpansTheDeadline1229(t *testing.T) { w := watchDialClock896("127.0.0.1:1", time.Second, clock) offset.Store(int64(1500 * time.Millisecond)) if !onStop { - time.Sleep(60 * time.Millisecond) // a few ticks after the jump + // a tick after the jump: polled, a starved goroutine is late + for dl := time.Now().Add(time.Second); time.Duration(w.gaps.max.Load()) < 1500*time.Millisecond && time.Now().Before(dl); { + time.Sleep(5 * time.Millisecond) + } if got := time.Duration(w.gaps.max.Load()); got < 1500*time.Millisecond { t.Errorf("stall seen on a tick: largest gap before the stop = %v, want at least 1.5s", got) } @@ -457,14 +467,14 @@ func TestDialWatchDescribesTheSocketsOfAPort1229(t *testing.T) { // A dial nobody answers: a reserved address, whose SYN goes unanswered. const unanswered = "10.255.255.1:41234" - w = watchDial896(unanswered, 600*time.Millisecond) + w = watchDial896(unanswered, time.Second) t0 := time.Now() - c, err = net.DialTimeout("tcp", unanswered, 600*time.Millisecond) + c, err = net.DialTimeout("tcp", unanswered, time.Second) if err == nil { _ = c.Close() t.Skip("something answers 10.255.255.1") } - if time.Since(t0) < 500*time.Millisecond { + if time.Since(t0) < 900*time.Millisecond { t.Skipf("the dial to %s failed at once, not on its timeout: %v", unanswered, err) } got = w.report() @@ -474,3 +484,21 @@ func TestDialWatchDescribesTheSocketsOfAPort1229(t *testing.T) { } w.end() // a second end after report is harmless } + +// TestDialWatchSnapshotsOnlyFromHalfTheTimeout1229: a row seen before half +// the dial's timeout is not kept as the near-deadline snapshot (the dial has +// only just begun, and a snapshot of its first moments would stand in for the +// deadline's); the first snapshot tick is at 250 ms, of a 2 s timeout. +func TestDialWatchSnapshotsOnlyFromHalfTheTimeout1229(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer func() { _ = ln.Close() }() + w := watchDial896(ln.Addr().String(), dialTimeout896) // the LISTEN row is there from the start + time.Sleep(400 * time.Millisecond) // past the first snapshot tick + w.end() + if w.near != nil { + t.Errorf("a snapshot was kept at %v, before half of the %v timeout: %q", w.nearAt, dialTimeout896, w.near) + } +} From 16f215451132f0100e3a808e353cd31c6392940d Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 3 Nov 2026 05:42:02 +0100 Subject: [PATCH 9/9] test(iouring): the dial watch's live test asserts only what holds on any host; gap and snapshot are tested on fixed clocks and rows (Refs #1229) --- .../iouring/dial_watch_1229_linux_test.go | 182 +++++++++++------- 1 file changed, 110 insertions(+), 72 deletions(-) diff --git a/internal/engine/iouring/dial_watch_1229_linux_test.go b/internal/engine/iouring/dial_watch_1229_linux_test.go index e325ac8bc..23b4fb00b 100644 --- a/internal/engine/iouring/dial_watch_1229_linux_test.go +++ b/internal/engine/iouring/dial_watch_1229_linux_test.go @@ -41,7 +41,10 @@ const dialTimeout896 = 2 * time.Second // not running for that long. type dialWatch896 struct { port int - now func() time.Time // time.Now; a test replaces it + now func() time.Time // time.Now; a test replaces it + t0 time.Time // when the watch began + rows func(port int) []string // tcpRows896; a test replaces it + backlog func() int64 // softnetBacklog896 timeout time.Duration before procCounters896 stop chan struct{} @@ -120,7 +123,8 @@ func watchDial896(addr string, timeout time.Duration) *dialWatch896 { } func watchDialClock896(addr string, timeout time.Duration, now func() time.Time) *dialWatch896 { - w := &dialWatch896{now: now, timeout: timeout, stop: make(chan struct{}), finished: make(chan struct{})} + w := &dialWatch896{now: now, timeout: timeout, rows: tcpRows896, backlog: softnetBacklog896, + stop: make(chan struct{}), finished: make(chan struct{})} if _, ps, err := net.SplitHostPort(addr); err == nil { w.port, _ = strconv.Atoi(ps) } @@ -132,26 +136,18 @@ func watchDialClock896(addr string, timeout time.Duration, now func() time.Time) defer tk.Stop() snap := time.NewTicker(250 * time.Millisecond) defer snap.Stop() - t0 := now() - w.gaps.last = t0 + w.t0 = now() + w.gaps.last = w.t0 close(started) for { select { case <-w.stop: - // The stall that ends the dial is the gap since the last tick. - w.gaps.tick(now()) + w.onStop() return case <-tk.C: - w.gaps.tick(now()) + w.onTick() case <-snap.C: - // From half the timeout on, keep the last snapshot with a row: - // the client's socket is closed before the dial returns. - if at := now().Sub(t0); at >= timeout/2 { - if rows := tcpRows896(w.port); len(rows) > 0 { - w.near, w.nearAt = rows, at - w.backlogN = softnetBacklog896() - } - } + w.onSnap() } } }() @@ -159,6 +155,24 @@ func watchDialClock896(addr string, timeout time.Duration, now func() time.Time) return w } +// onTick records the gap since the last tick, with the clock and not with the +// value the ticker delivered. +func (w *dialWatch896) onTick() { w.gaps.tick(w.now()) } + +// onStop records the stall that ends the dial: the gap since the last tick. +func (w *dialWatch896) onStop() { w.gaps.tick(w.now()) } + +// onSnap keeps, from half the timeout on, the last snapshot of the port's rows +// that has a row: the client's socket is closed before the dial returns. +func (w *dialWatch896) onSnap() { + if at := w.now().Sub(w.t0); at >= w.timeout/2 { + if rows := w.rows(w.port); len(rows) > 0 { + w.near, w.nearAt = rows, at + w.backlogN = w.backlog() + } + } +} + // end stops the watch; a watch whose dial succeeded calls only this. func (w *dialWatch896) end() { select { @@ -414,91 +428,115 @@ func TestDialWatchGapSeesAStallThatSpansTheDeadline1229(t *testing.T) { t.Errorf("largest gap = %v, want 1.5s", got) } - // The watch itself: its clock jumps while the (real) ticker runs, and then - // the watch is stopped; both the tick and the stop take the clock's time. + // The watch's own handlers, driven with a clock that jumps and no ticker + // or goroutine: both the tick and the stop take the clock's time. for _, onStop := range []bool{false, true} { - var offset atomic.Int64 - clock := func() time.Time { return time.Now().Add(time.Duration(offset.Load())) } - w := watchDialClock896("127.0.0.1:1", time.Second, clock) - offset.Store(int64(1500 * time.Millisecond)) - if !onStop { - // a tick after the jump: polled, a starved goroutine is late - for dl := time.Now().Add(time.Second); time.Duration(w.gaps.max.Load()) < 1500*time.Millisecond && time.Now().Before(dl); { - time.Sleep(5 * time.Millisecond) - } - if got := time.Duration(w.gaps.max.Load()); got < 1500*time.Millisecond { - t.Errorf("stall seen on a tick: largest gap before the stop = %v, want at least 1.5s", got) - } + now := t0 + w := &dialWatch896{now: func() time.Time { return now }} + w.gaps.last = t0 + now = t0.Add(1500 * time.Millisecond) + if onStop { + w.onStop() + } else { + w.onTick() } - w.end() - if got := time.Duration(w.gaps.max.Load()); got < 1500*time.Millisecond { - t.Errorf("stall seen on a tick (onStop=%v): largest gap = %v, want at least 1.5s", onStop, got) + if got := time.Duration(w.gaps.max.Load()); got != 1500*time.Millisecond { + t.Errorf("stall seen by the %s handler: largest gap = %v, want 1.5s", map[bool]string{false: "tick", true: "stop"}[onStop], got) } } } -// TestDialWatchDescribesTheSocketsOfAPort1229: the failure message of a dial -// that timed out lists the sockets of the port with their states, the -// counters and the tick gap, whether the dial went through or not, and the -// snapshot taken during a dial that nobody answers holds the client in -// SYN_SENT, which the report after the timeout cannot (Go closed the socket). +// TestDialWatchSnapshotKeepsTheLastRowsFromHalfTheTimeout1229: a snapshot +// before half the timeout is not kept (the dial has only just begun and its +// first moments would stand in for the deadline's), later ones replace each +// other, and one without a row does not erase the last that had one. +func TestDialWatchSnapshotKeepsTheLastRowsFromHalfTheTimeout1229(t *testing.T) { + t0 := time.Unix(1000, 0) + now := t0 + rows := []string{"tcp a>b SYN_SENT q(tx:rx)=00000001:00000000"} + var backlog int64 = 7 + w := &dialWatch896{now: func() time.Time { return now }, t0: t0, timeout: 2 * time.Second, port: 8080, + rows: func(int) []string { return rows }, backlog: func() int64 { return backlog }, + stop: make(chan struct{}), finished: make(chan struct{})} + close(w.finished) // no goroutine: report's end() has nothing to wait for + + now = t0.Add(time.Second - time.Millisecond) + w.onSnap() + if w.near != nil { + t.Errorf("a snapshot was kept before half of the timeout: %q at %v", w.near, w.nearAt) + } + now = t0.Add(time.Second) + w.onSnap() + if len(w.near) != 1 || w.nearAt != time.Second || w.backlogN != 7 { + t.Errorf("snapshot at half the timeout = %q at %v backlog %d, want the row at 1s with backlog 7", w.near, w.nearAt, w.backlogN) + } + rows, backlog, now = []string{"tcp c>d TIME_WAIT q(tx:rx)=0:0"}, 9, t0.Add(1500*time.Millisecond) + w.onSnap() + if len(w.near) != 1 || !strings.Contains(w.near[0], "TIME_WAIT") || w.nearAt != 1500*time.Millisecond || w.backlogN != 9 { + t.Errorf("a later snapshot = %q at %v backlog %d, want it to replace the first", w.near, w.nearAt, w.backlogN) + } + rows, now = nil, t0.Add(1800*time.Millisecond) + w.onSnap() + if len(w.near) != 1 || w.nearAt != 1500*time.Millisecond { + t.Errorf("a snapshot without a row replaced %q at %v", w.near, w.nearAt) + } + if got := w.report(); !strings.Contains(got, "near the deadline at 1.5s: [tcp c>d TIME_WAIT") { + t.Errorf("report does not carry the kept snapshot: %s", got) + } +} + +// TestDialWatchDescribesTheSocketsOfAPort1229 runs the watch against live +// sockets and asserts only what holds in any environment: the report is made, +// bounded, names the port and the gap, and ends the watch (the fixture tests +// above carry what the fields say). What the sockets show depends on the +// host, so it is logged; a host that gives a dial nobody answers (a reserved +// address, whose SYN is not answered) is not one the test can rely on, and +// the SYN_SENT row of its snapshot is checked only if the dial was left +// unanswered. func TestDialWatchDescribesTheSocketsOfAPort1229(t *testing.T) { + check := func(t *testing.T, got string, port int) { + t.Helper() + if len(got) > 4000 { + t.Errorf("report of %d bytes is not bounded", len(got)) + } + for _, want := range []string{fmt.Sprintf("sockets on port %d:", port), "largest gap between 10 ms ticks"} { + if !strings.Contains(got, want) { + t.Errorf("report lacks %q: %s", want, got) + } + } + } ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { - t.Fatal(err) + t.Skipf("no loopback listener here: %v", err) } addr := ln.Addr().String() + port := ln.Addr().(*net.TCPAddr).Port w := watchDial896(addr, dialTimeout896) c, err := net.DialTimeout("tcp", addr, dialTimeout896) if err != nil { - t.Fatal(err) + _ = ln.Close() + t.Skipf("no loopback dial here: %v", err) } - time.Sleep(30 * time.Millisecond) // a tick or two got := w.report() t.Logf("report: %s", got) - // The counters are namespace-wide, so only that they are there is asserted. - for _, want := range []string{"LISTEN", "ESTABLISHED", "no snapshot", "largest gap between 10 ms ticks", "ListenDrops+", "softnet_drop+", "RetransSegs+"} { - if !strings.Contains(got, want) { - t.Errorf("report of a dial that went through lacks %q: %s", want, got) - } - } + check(t, got, port) _ = c.Close() _ = ln.Close() - // A dial nobody answers: a reserved address, whose SYN goes unanswered. - const unanswered = "10.255.255.1:41234" + // A dial nobody answers: a reserved address, on a port of our own choosing. + unanswered := fmt.Sprintf("10.255.255.1:%d", port) w = watchDial896(unanswered, time.Second) t0 := time.Now() c, err = net.DialTimeout("tcp", unanswered, time.Second) if err == nil { _ = c.Close() - t.Skip("something answers 10.255.255.1") - } - if time.Since(t0) < 900*time.Millisecond { - t.Skipf("the dial to %s failed at once, not on its timeout: %v", unanswered, err) } got = w.report() t.Logf("report: %s", got) - if !strings.Contains(got, "SYN_SENT") || strings.Contains(got, "no snapshot") { - t.Errorf("report of an unanswered dial lacks the SYN_SENT row of the snapshot: %s", got) - } + check(t, got, port) w.end() // a second end after report is harmless -} - -// TestDialWatchSnapshotsOnlyFromHalfTheTimeout1229: a row seen before half -// the dial's timeout is not kept as the near-deadline snapshot (the dial has -// only just begun, and a snapshot of its first moments would stand in for the -// deadline's); the first snapshot tick is at 250 ms, of a 2 s timeout. -func TestDialWatchSnapshotsOnlyFromHalfTheTimeout1229(t *testing.T) { - ln, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer func() { _ = ln.Close() }() - w := watchDial896(ln.Addr().String(), dialTimeout896) // the LISTEN row is there from the start - time.Sleep(400 * time.Millisecond) // past the first snapshot tick - w.end() - if w.near != nil { - t.Errorf("a snapshot was kept at %v, before half of the %v timeout: %q", w.nearAt, dialTimeout896, w.near) + if err == nil || time.Since(t0) < 900*time.Millisecond || !strings.Contains(got, "SYN_SENT") { + t.Skipf("this host did not leave the dial to %s unanswered with a SYN_SENT row in the snapshot (err %v after %v)", + unanswered, err, time.Since(t0).Round(time.Millisecond)) } }