Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
542 changes: 542 additions & 0 deletions internal/engine/iouring/dial_watch_1229_linux_test.go

Large diffs are not rendered by default.

9 changes: 2 additions & 7 deletions internal/engine/iouring/drain_clock_step_1176_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"context"
"io"
"net"
"sync"
"sync/atomic"
"testing"
"time"
Expand Down Expand Up @@ -356,10 +355,7 @@ func TestPoolWaitGateWaitsItsWindowWhateverTheWallDoes1176(t *testing.T) {

// releaseOnce1176 is h's release, closed at most once, so a test that fails
// before the handler is released still lets it go.
func releaseOnce1176(h *holdHandler896) func() {
var once sync.Once
return func() { once.Do(func() { close(h.release) }) }
}
func releaseOnce1176(h *holdHandler896) func() { return h.unblock }

func isTimeout1176(err error) bool {
ne, ok := err.(net.Error)
Expand Down Expand Up @@ -437,8 +433,7 @@ func TestCancelAcceptWaitIsBoundedByTheMonotonicClock1176(t *testing.T) {
return nil
}
t.Cleanup(func() { shutdownRingWait = orig })
cancelAcceptSeam = func(w *Worker, _ int) { ring.CompareAndSwap(nil, w.ring) }
t.Cleanup(func() { cancelAcceptSeam = nil })
setCancelAcceptSeam1229(t, func(w *Worker, _ int) { ring.CompareAndSwap(nil, w.ring) })
restore := stairClock1176(t, step)

h := newHoldHandler896(false)
Expand Down
161 changes: 161 additions & 0 deletions internal/engine/iouring/engine_cleanup_1229_linux_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
//go:build linux

package iouring

import (
"testing"
"time"
)

// celeris#1229: a test that fails after it started an engine must stop the
// engine. The helpers below start engines for the 896 and 1173 tests and
// used to leave the stopping to the test body, which a failed assertion
// (t.Fatalf: runtime.Goexit) never reaches. On the Coverage job's 8 MiB
// RLIMIT_MEMLOCK four 896 tests failed on a dial timeout, left four engines
// running, and every later test that installs a package seam (ringWaitSeam,
// cancelAcceptSeam) had those workers read it: answered EBUSY for waits it
// never asked about, and raced with the seam's restore.

// engineAfterItsTestEnded1229 runs start in a subtest that returns at once
// without stopping what it started, which is all a failed assertion does to
// the cleanups of its test (runtime.Goexit runs them, and nothing else of the
// test's body), and returns the run the helper started and
// whether Listen had returned by the time the subtest's cleanups had run. A
// run that is still going is cancelled here, so that this test does not leak
// what it looks for.
func engineAfterItsTestEnded1229(t *testing.T, start func(t *testing.T) *engineRun896) (run *engineRun896, stopped bool) {
t.Helper()
t.Run("ended", func(t *testing.T) {
run = start(t)
})
if run == nil {
t.Skip("the engine did not start here")
}
select {
case err := <-run.done:
if err != nil {
t.Errorf("Listen returned %v", err)
}
return run, true
default:
}
run.cancel()
select {
case <-run.done:
case <-time.After(10 * time.Second):
t.Error("the leaked engine did not stop when cancelled")
}
return run, false
}

func TestEngineStartHelpersStopTheirEngineWhenTheTestEnds1229(t *testing.T) {
const leaked = "the engine was still running after the test that started it had ended: " +
"a failed assertion would leave its workers reading the package's test seams"
t.Run("start896", func(t *testing.T) {
if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 {
return start896(t, freeAddr639(t), 2)
}); !stopped {
t.Error(leaked)
}
})
t.Run("startHold896", func(t *testing.T) {
if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 {
return startHold896(t, freeAddr639(t), 2, 0, newHoldHandler896(false))
}); !stopped {
t.Error(leaked)
}
})
// A /hold handler still running at the end is what a failure in the middle
// of an iteration leaves: Listen waits for it, so the engine's cleanup has
// to release it before it waits.
t.Run("startHold896/handler running", func(t *testing.T) {
var h *holdHandler896
if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 {
h = newHoldHandler896(false)
addr := freeAddr639(t)
run := startHold896(t, addr, 2, 0, h)
c := h2Request896(t, addr, "/hold")
t.Cleanup(func() { _ = c.Close() })
select {
case <-h.started:
case <-time.After(5 * time.Second):
t.Fatal("the /hold handler never ran")
}
return run
}); !stopped {
h.unblock()
t.Error(leaked)
}
})
t.Run("startGate1173", func(t *testing.T) {
var g *gate1173
if _, stopped := engineAfterItsTestEnded1229(t, func(t *testing.T) *engineRun896 {
g = &gate1173{started: make(chan struct{}, 1), gate: make(chan struct{})}
run, _ := startGate1173(t, g)
return run
}); !stopped {
t.Error(leaked)
}
})
}

// seamRestoredUnderALateReader1229 installs a seam with install in a subtest,
// whose cleanup restores it, while a goroutine that is not ordered with that
// restore goes on reading the seam through read: a worker that an earlier test
// leaked. The restore is a write the race detector must not find unordered
// with the read, so a seam read and written as a plain variable fails this
// test under -race ("race detected during execution of test"), in most runs
// and not in every one: the detector reports the races it observes, and the
// interleaving is not forced.
func seamRestoredUnderALateReader1229(t *testing.T, install func(t *testing.T), read func()) {
t.Helper()
stop := make(chan struct{})
finished := make(chan struct{})
started := false
t.Run("seam", func(t *testing.T) {
install(t)
started = true
go func() {
defer close(finished)
for {
select {
case <-stop:
return
default:
read()
}
}
}()
})
if !started { // -run filtered the subtest out: no reader to wait for
return
}
close(stop)
<-finished
}

func TestRingWaitSeamRestoreIsSafeAgainstALateReader1229(t *testing.T) {
var w Worker
seamRestoredUnderALateReader1229(t, func(t *testing.T) {
setRingWaitSeam884(t, func(*Worker, ringWaitMode, error) error { return nil })
}, func() { _ = w.ringWaitContinues(waitModeTimed, nil) })
}

func TestCancelAcceptSeamRestoreIsSafeAgainstALateReader1229(t *testing.T) {
r := newAcceptRig896(t, &baseTier{}, false)
seamRestoredUnderALateReader1229(t, func(t *testing.T) {
setCancelAcceptSeam1229(t, func(*Worker, int) {})
}, func() {
// stopAccepting reads the seam first; the listen socket is closed by the
// first call, and a later one finds none and returns.
r.w.stopAccepting(t.Context())
})
}

// setCancelAcceptSeam1229 installs cancelAcceptSeam for the test and restores
// it nil when the test ends.
func setCancelAcceptSeam1229(t *testing.T, fn func(*Worker, int)) {
t.Helper()
cancelAcceptSeam.Store(&fn)
t.Cleanup(func() { cancelAcceptSeam.Store(nil) })
}
7 changes: 6 additions & 1 deletion internal/engine/iouring/listen_cancel_seam_896.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@

package iouring

import "sync/atomic"

// cancelAcceptSeam is a test seam (celeris#896): when set, cancelAccept calls it
// first, on the worker's thread, with the descriptor it is about to cancel. A
// test counts the calls to prove a shutdown reached stopAccepting or the pause
// close, and uses it to make the SQ ring full at that moment. Nil in production.
var cancelAcceptSeam func(w *Worker, lfd int)
// An atomic.Pointer, as h2WriteDrainSeam is, because a worker an earlier test
// left running may still read it when a later test sets or restores it
// (celeris#1229).
var cancelAcceptSeam atomic.Pointer[func(w *Worker, lfd int)]
75 changes: 69 additions & 6 deletions internal/engine/iouring/listen_release_896_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,72 @@ const iterations896 = 200

// engineRun896 is one started engine and the way to stop it.
type engineRun896 struct {
e *Engine
cancel context.CancelFunc
done chan error
e *Engine
cancel context.CancelFunc
done chan error // receives Listen's result
returned chan struct{} // closed once done has it (listenAsync896)
reaper *reaper896 // set by reapAtCleanup
}

// listenAsync896 runs e.Listen(ctx) on a goroutine of its own. done receives
// its result; returned is closed after that, so a cleanup that waits on it
// can still read done.
func listenAsync896(ctx context.Context, e *Engine) (done chan error, returned chan struct{}) {
done = make(chan error, 1)
returned = make(chan struct{})
go func() {
done <- e.Listen(ctx)
close(returned)
}()
return done, returned
}

// reapAtCleanup stops the engine when the test ends, whether or not the test
// stopped it: a test that fails after it started the engine ends in
// runtime.Goexit, which skips the stop in its body, and the workers of an
// engine left running read the package's test seams (ringWaitSeam,
// cancelAcceptSeam) in every test after it (celeris#1229). release, if not
// nil, frees whatever Listen waits for first (a handler held by the test). The
// cleanup is registered when the engine has started, so it runs before the
// restores of the seams the test installed earlier.
func (r *engineRun896) reapAtCleanup(t *testing.T, release func()) {
t.Helper()
r.reaper = &reaper896{cancel: r.cancel, returned: r.returned, release: release}
rp := r.reaper
t.Cleanup(func() { rp.reap(t) })
}

// reap is reapAtCleanup's stop, at once. A loop that starts an engine per
// iteration defers it, so that the engine, and the handler it holds (a 32 MiB
// response in the send-drain loops), are not kept until the test ends.
func (r *engineRun896) reap(t *testing.T) {
t.Helper()
r.reaper.reap(t)
}

// reaper896 is what the cleanup holds of a run: not the engine, so that a
// finished iteration's engine is garbage.
type reaper896 struct {
cancel context.CancelFunc
returned chan struct{}
release func()
once sync.Once
}

func (rp *reaper896) reap(t *testing.T) {
t.Helper()
rp.once.Do(func() {
if rp.release != nil {
rp.release()
}
rp.cancel()
select {
case <-rp.returned:
case <-time.After(10 * time.Second):
t.Errorf("Listen had not returned 10 s after the test ended: a worker is still running")
}
rp.release = nil
})
}

// stop cancels Listen's context and returns the instant Listen has returned,
Expand Down Expand Up @@ -78,11 +141,10 @@ func start896Config(t *testing.T, addr string, workers int, disableDefer bool) *
skipOrFail656(t, "New: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- e.Listen(ctx) }()
done, returned := listenAsync896(ctx, e)
for dl := time.Now().Add(5 * time.Second); time.Now().Before(dl); {
if e.Addr() != nil {
run = &engineRun896{e: e, cancel: cancel, done: done}
run = &engineRun896{e: e, cancel: cancel, done: done, returned: returned}
return nil
}
select {
Expand All @@ -103,6 +165,7 @@ func start896Config(t *testing.T, addr string, workers int, disableDefer bool) *
}
t.Fatalf("engine start on %s: %v", addr, err)
}
run.reapAtCleanup(t, nil)
return run
}

Expand Down
Loading
Loading