diff --git a/driver/internal/eventloop/portable.go b/driver/internal/eventloop/portable.go index 313100494..3556c5c39 100644 --- a/driver/internal/eventloop/portable.go +++ b/driver/internal/eventloop/portable.go @@ -293,6 +293,15 @@ func (w *portableWorker) readLoop(c *portableConn) { func (w *portableWorker) UnregisterConn(fd int) error { w.mu.Lock() c, ok := w.conns[fd] + if ok && c.conn == nil { + // register's placeholder: a RegisterConn of this number is still + // opening the net.Conn. The number is not registered yet, so it is + // unknown to this call, as on the Linux workers, whose RegisterConn + // inserts the conn in one step. Leave the placeholder: register + // installs the conn over it, and an UnregisterConn issued after + // RegisterConn returned finds that (celeris#890). + ok = false + } if ok { delete(w.conns, fd) } @@ -389,14 +398,11 @@ func (w *portableWorker) Write(fd int, data []byte) error { // still delivering onRecv. No lock is held across the close or the callback. // The placeholder register puts in the table while it opens the net.Conn // (conn == nil) has no onClose and no reader, so for it this only marks it -// closing: there is nothing to close and nothing to report. shutdown and -// UnregisterConn both reach it. After shutdown's table swap register sees the +// closing: there is nothing to close and nothing to report. Only shutdown +// reaches it: UnregisterConn treats a placeholder as an unknown number and +// leaves it (celeris#890). After shutdown's table swap register sees the // worker closed and drops the conn it opened; a failed open drops the -// placeholder itself. An UnregisterConn that finds the placeholder deletes it -// and returns nil, but register's second section checks only the worker's -// closed flag, not that the placeholder is still in the table, so it installs -// the opened conn and starts its reader anyway: that UnregisterConn is lost -// (celeris#890). +// placeholder itself. func (c *portableConn) shutdownConn(err error) { c.mu.Lock() if c.closing { diff --git a/driver/internal/eventloop/portable_unregister_opening_890_test.go b/driver/internal/eventloop/portable_unregister_opening_890_test.go new file mode 100644 index 000000000..aa6e7d472 --- /dev/null +++ b/driver/internal/eventloop/portable_unregister_opening_890_test.go @@ -0,0 +1,115 @@ +package eventloop + +// celeris#890 (found by the correctness review of PR #1051): an +// UnregisterConn(fd) that ran while RegisterConn(fd, ...) was still opening +// the net.Conn was lost. register had put a placeholder in the table; +// UnregisterConn found it, deleted it and returned nil; register's second +// section checked only the worker's closed flag, so it installed the opened +// conn and started its reader anyway: the conn stayed in the table, onClose +// never fired and the net.Conn was never closed. +// +// The two outcomes the contract allows are tested (the call is linearizable): +// either UnregisterConn ran before the registration (the number is unknown to +// it: engine.ErrUnknownFD, as on the Linux workers, whose RegisterConn also +// inserts the conn atomically) and the registration stands, fully live; or it +// ran after, returned nil and the conn is fully torn down (onClose fired, the +// net.Conn is closed, the table is clean). Never a nil return with the conn +// left registered. No build tag: the portable worker is compiled everywhere. + +import ( + "context" + "errors" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" +) + +func TestUnregisterConnWhileRegisterConnIsOpening890(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pw := newPortableWorker(ctx, 0) + w := &pw + t.Cleanup(func() { _ = w.shutdown() }) + + const fd = 4242 + ours, theirs := net.Pipe() + t.Cleanup(func() { _ = theirs.Close(); _ = ours.Close() }) + + opening := make(chan struct{}) + release := make(chan struct{}) + open := func() (net.Conn, error) { + close(opening) + <-release + return ours, nil + } + var closes atomic.Int32 + closed := make(chan error, 4) + onClose := func(err error) { + closes.Add(1) + closed <- err + } + + type regResult struct { + fd int + err error + } + regDone := make(chan regResult, 1) + go func() { + got, err := w.register(fd, false, open, func([]byte) {}, onClose) + regDone <- regResult{got, err} + }() + <-opening + + unregErr := w.UnregisterConn(fd) + close(release) + reg := <-regDone + + switch { + case unregErr == nil: + // UnregisterConn took the registration down: onClose must fire, the + // net.Conn must be closed and the table must not hold the number. + select { + case err := <-closed: + if err != nil { + t.Errorf("onClose(%v), want nil after UnregisterConn", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("UnregisterConn returned nil during the open, but the conn was left registered: onClose never fired (register returned fd %d, err %v)", reg.fd, reg.err) + } + _ = theirs.SetReadDeadline(time.Now().Add(2 * time.Second)) + if _, err := theirs.Read(make([]byte, 1)); err == nil || errors.Is(err, context.DeadlineExceeded) { + t.Errorf("the net.Conn is still open after UnregisterConn returned nil: peer read err = %v", err) + } + w.mu.RLock() + _, still := w.conns[fd] + w.mu.RUnlock() + if still { + t.Errorf("the table still holds fd %d after UnregisterConn returned nil", fd) + } + case errors.Is(unregErr, engine.ErrUnknownFD): + // UnregisterConn ran before the registration existed: it did nothing, + // and the registration must be intact. + if reg.err != nil { + t.Fatalf("register failed after an UnregisterConn that reported an unknown fd: %v", reg.err) + } + if closes.Load() != 0 { + t.Fatalf("onClose fired %d times for a registration that was never unregistered", closes.Load()) + } + if err := w.UnregisterConn(fd); err != nil { + t.Fatalf("second UnregisterConn: %v", err) + } + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("the registration did not tear down on its own UnregisterConn: onClose never fired") + } + default: + t.Fatalf("UnregisterConn during the open returned %v", unregErr) + } + if got := closes.Load(); got != 1 { + t.Errorf("onClose fired %d times, want 1", got) + } +} diff --git a/driver/internal/eventloop/register_wait.go b/driver/internal/eventloop/register_wait.go new file mode 100644 index 000000000..021d9555e --- /dev/null +++ b/driver/internal/eventloop/register_wait.go @@ -0,0 +1,97 @@ +package eventloop + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/goceleris/celeris/internal/engine" +) + +// registerWaitBudget bounds how long RegisterConnWait waits for a refused +// number to be finalized. A conn is finalized when the engine has cancelled its +// operations and closed its own duplicate of the descriptor, normally within +// microseconds of UnregisterConn; a close that lingers (SO_LINGER) is closed +// off the worker and can take longer (celeris#735), so the wait is bounded +// instead of unlimited. +const registerWaitBudget = 2 * time.Second + +// errRegisterWaitBudget is wrapped into the error RegisterConnWait returns +// when registerWaitBudget has passed (declared by the tests commit, used by the +// next). +var errRegisterWaitBudget = errors.New("celeris/eventloop: waited for the number to be finalized, budget used up") + +// IsNumberBusy reports whether err is the io_uring engine's refusal of a +// descriptor number whose previous conn has been unregistered but not yet +// finalized: "celeris/iouring: fd already registered". +// [engine.WorkerLoop.UnregisterConn] documents it: the next socket usually +// gets the lowest free number, which is the number of the conn just closed, +// and the engine keeps that number registered until the old conn is finalized. +// It clears by itself, unlike any other RegisterConn error (the number is +// held by an HTTP conn, the worker has shut down, a duplicate registration on +// the epoll engine, whose onClose has run before UnregisterConn returns). +// +// The engine has no sentinel for it, so the text is matched; the tests of +// the drivers register the same number twice on a real io_uring worker to pin +// it. +func IsNumberBusy(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.HasPrefix(msg, "celeris/iouring:") && strings.HasSuffix(msg, "fd already registered") +} + +// RegisterConnWait is loop.RegisterConn that waits out the io_uring engine's +// "fd already registered" refusal (see [IsNumberBusy]) instead of returning it: +// it retries with a growing pause until the number is free, ctx is done, or +// registerWaitBudget has passed, and then returns the refusal wrapped together +// with the reason it stopped (ctx.Err(), or errRegisterWaitBudget), so +// errors.Is sees both. Every other error is returned at once, and a failed +// call leaves fd with the caller, as RegisterConn does. The drivers' dials use +// it: without it Pool.Acquire fails under connection churn on the io_uring +// engine (celeris#887 item 28). +// +// It must not run on the worker goroutine of loop (a handler of the server +// whose engine is that loop's, called inline): the number is finalized by that +// worker, which would be asleep in this wait, so the wait runs to its end, +// stalls every connection of that worker for up to registerWaitBudget, and +// returns the refusal anyway. The drivers do not dial there: with WithEngine on +// io_uring, epoll or adaptive their command pools dial direct net.Conns and +// never call RegisterConn. The one public path left is redis +// Client.Subscribe called from a handler, whose PubSub pool still registers +// with the engine; with the default config its handshake already blocks on the +// worker it occupies, so the case that reaches this wait is ForceRESP2 with no +// password and no DB. The engine gives no cheap, reliable way to tell that the +// caller is on its worker, so this is documented, not guarded. The alternative +// that works on any thread, moving the socket to a free number (F_DUPFD) before +// registering it, changes which descriptor the driver owns and needs its own +// review. +func RegisterConnWait(ctx context.Context, loop engine.WorkerLoop, fd int, onRecv func([]byte), onClose func(error)) error { + err := loop.RegisterConn(fd, onRecv, onClose) + if !IsNumberBusy(err) { + return err + } + deadline := time.Now().Add(registerWaitBudget) + pause := 100 * time.Microsecond + for { + t := time.NewTimer(pause) + select { + case <-ctx.Done(): + t.Stop() + return fmt.Errorf("%w (gave up waiting: %w)", err, ctx.Err()) + case <-t.C: + } + if err = loop.RegisterConn(fd, onRecv, onClose); !IsNumberBusy(err) { + return err + } + if !time.Now().Before(deadline) { + return fmt.Errorf("%w (gave up waiting: %w)", err, errRegisterWaitBudget) + } + if pause < 5*time.Millisecond { + pause *= 2 + } + } +} diff --git a/driver/internal/eventloop/register_wait_887_test.go b/driver/internal/eventloop/register_wait_887_test.go new file mode 100644 index 000000000..26665addc --- /dev/null +++ b/driver/internal/eventloop/register_wait_887_test.go @@ -0,0 +1,100 @@ +package eventloop + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" +) + +type scriptedLoop struct { + engine.WorkerLoop + errs []error // returned in order; the last repeats + calls int +} + +func (l *scriptedLoop) RegisterConn(int, func([]byte), func(error)) error { + i := l.calls + l.calls++ + if i >= len(l.errs) { + i = len(l.errs) - 1 + } + return l.errs[i] +} + +var errBusy = errors.New("celeris/iouring: fd already registered") + +func TestIsNumberBusy887(t *testing.T) { + for _, c := range []struct { + err error + want bool + }{ + {nil, false}, + {errBusy, true}, + {errors.New("celeris/iouring: fd 7 is already an HTTP connection"), false}, + {errors.New("celeris/epoll: fd 7 is already registered"), false}, + {ErrAlreadyRegistered, false}, // a real duplicate of the standalone loop: it does not clear + {fmt.Errorf("celeris/iouring: worker 0 has shut down, cannot register fd 7: %w", errors.New("engine shut down")), false}, + } { + if got := IsNumberBusy(c.err); got != c.want { + t.Errorf("IsNumberBusy(%v) = %v, want %v", c.err, got, c.want) + } + } +} + +func TestRegisterConnWait887(t *testing.T) { + other := errors.New("boom") + for _, c := range []struct { + name string + errs []error + cancelled bool + wantErr error + wantCalls int + }{ + {"free at once", []error{nil}, false, nil, 1}, + {"clears after three refusals", []error{errBusy, errBusy, errBusy, nil}, false, nil, 4}, + {"other error is final", []error{other}, false, other, 1}, + {"other error after a refusal is final", []error{errBusy, other}, false, other, 2}, + {"ctx done returns the refusal", []error{errBusy}, true, errBusy, 1}, + } { + t.Run(c.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if c.cancelled { + cancel() + } + l := &scriptedLoop{errs: c.errs} + err := RegisterConnWait(ctx, l, 3, nil, nil) + if !errors.Is(err, c.wantErr) { + t.Fatalf("err = %v, want %v", err, c.wantErr) + } + if c.cancelled && !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v does not wrap the ctx error: a caller cannot tell a cancelled wait from a refusal", err) + } + if l.calls != c.wantCalls { + t.Fatalf("RegisterConn calls = %d, want %d", l.calls, c.wantCalls) + } + }) + } +} + +// A refusal that never clears is bounded by registerWaitBudget, not by ctx. +func TestRegisterConnWaitIsBoundedWithoutACtxDeadline887(t *testing.T) { + l := &scriptedLoop{errs: []error{errBusy}} + start := time.Now() + err := RegisterConnWait(context.Background(), l, 3, nil, nil) + took := time.Since(start) + if !errors.Is(err, errBusy) { + t.Fatalf("err = %v", err) + } + if !errors.Is(err, errRegisterWaitBudget) { + t.Fatalf("err = %v does not say the wait ran out of budget", err) + } + if took < registerWaitBudget || took > registerWaitBudget+time.Second { + t.Fatalf("gave up after %v, want about %v", took, registerWaitBudget) + } + t.Logf("gave up after %v and %d calls", took, l.calls) +} diff --git a/driver/internal/eventloop/teardown_hooks_890_linux_test.go b/driver/internal/eventloop/teardown_hooks_890_linux_test.go index 061a64876..11d6dc894 100644 --- a/driver/internal/eventloop/teardown_hooks_890_linux_test.go +++ b/driver/internal/eventloop/teardown_hooks_890_linux_test.go @@ -209,8 +209,10 @@ func TestShutdownMarksEveryConnClosedBeforeItClosesTheEpollFd890(t *testing.T) { // the loaded number after it (mutant WAKEATOMIC): FD() already returns -1 once // the close has run, so no hook placed after the close can order that shape, // and the race detector reports nothing (the number is an atomic, the write is -// a syscall). Only the probabilistic oracle of -// TestWriteRacingCloseNeverTouchesTheClosedEventfd862 covers it (celeris#890). +// a syscall). That shape is pinned at the source by +// TestWakeGoesThroughSignalOnly890 (wake_source_guard_890_test.go); the +// probabilistic oracle of TestWriteRacingCloseNeverTouchesTheClosedEventfd862 +// only backs it (celeris#890). func TestAWakeAfterShutdownClosedTheEventfdWritesNothing890(t *testing.T) { t.Cleanup(func() { testHookAfterWakeClose = nil }) // runs last l, err := New(1) diff --git a/driver/internal/eventloop/wake_close_862_linux_test.go b/driver/internal/eventloop/wake_close_862_linux_test.go index 3c5fdcaf1..a6166d969 100644 --- a/driver/internal/eventloop/wake_close_862_linux_test.go +++ b/driver/internal/eventloop/wake_close_862_linux_test.go @@ -65,7 +65,8 @@ func c862FillSendBuffer(t *testing.T, fd int) int { // pinned deterministically by // TestAWakeAfterShutdownClosedTheEventfdWritesNothing890; a wake that loads // w.wakeFD.FD() itself, races the close and writes to the loaded number is -// covered only by the oracle above (celeris#890). +// pinned at the source by TestWakeGoesThroughSignalOnly890; the oracle above +// only backs it (celeris#890). func TestWriteRacingCloseNeverTouchesTheClosedEventfd862(t *testing.T) { const rounds, writers = 64, 4 one := []byte{'w'} diff --git a/driver/internal/eventloop/wake_source_guard_890_test.go b/driver/internal/eventloop/wake_source_guard_890_test.go new file mode 100644 index 000000000..d663218e3 --- /dev/null +++ b/driver/internal/eventloop/wake_source_guard_890_test.go @@ -0,0 +1,317 @@ +package eventloop + +// celeris#890 (the #932 wake test, WAKEATOMIC): a producer's wake must go +// through wakefd.WakeFD.Signal, whose write(2) and the eventfd's close share a +// lock. A wake that loads the eventfd's number (w.wakeFD.FD()) and writes it +// itself races Loop.Close: it can load the number before the close and write +// it after, onto whatever socket or eventfd has taken the number meanwhile. +// +// No test can order that shape. FD() returns -1 once the close has run, so a +// hook placed after the close (testHookAfterWakeClose, +// TestAWakeAfterShutdownClosedTheEventfdWritesNothing890) never sees a stale +// number; the race detector reports nothing (the number is an atomic and the +// write is a syscall); and the only oracle that can catch it, +// TestWriteRacingCloseNeverTouchesTheClosedEventfd862, needs a wake to land in +// the few microseconds between the close and the reuse of the number (0 of 10 +// processes without -race, 1 of 10 with it). The window sits between two +// statements of the mutant's own wake, where the production code has no hook +// and, by design, no such window. So the property is pinned at the source: +// this test parses the package's non-test files and requires that +// +// - every use of the worker's wakeFD field is a method call, and the method +// is the one its function is allowed (newWorker builds it, run reads the +// number on the loop thread, wake signals, shutdown closes); +// - wake's body is the single statement w.wakeFD.Signal(); +// - the eventfd number newWorker creates reaches only wakefd.New, +// unix.EpollCtl and unix.Close: it is never stored in another field, where +// a producer could write it itself (review round 1: a copy kept in the +// worker and written by enqueueFlush has WAKEATOMIC's defect and left +// wakeFD untouched); +// - the only raw write(2) of the package is flushLocked's, to a conn's fd. +// +// Known limit: this is a syntactic pin of the shapes above, not a proof. A +// number reaching a producer by a route outside them (a global, a closure over +// an integer returned by a helper) is not seen; the probabilistic oracle of +// TestWriteRacingCloseNeverTouchesTheClosedEventfd862 is the backstop. +// +// It has no build tag: the files are parsed, not compiled, so it runs on every +// platform the package builds on. A mutant fails it with the position of the +// offending use, for example "(*worker).wake calls wakeFD.FD". + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +// wakeFDUses is the set of (function, method) pairs allowed to touch the +// worker's wakeFD field. "" stands for the composite-literal key. +var wakeFDUses = map[string]bool{ + "newWorker ": true, + "run FD": true, + "wake Signal": true, + "shutdown Close": true, +} + +func TestWakeGoesThroughSignalOnly890(t *testing.T) { + uses, bad, wakeBody := scanWakeFDUses(t, ".") + for _, v := range bad { + t.Error(v) + } + for _, u := range uses { + if !wakeFDUses[u.key()] { + t.Errorf("%s: %s uses wakeFD.%s; only %s are allowed (a producer's wake must be wakefd.WakeFD.Signal, celeris#890 WAKEATOMIC)", + u.pos, u.fn, u.method, allowedList()) + } + } + // A guard that scans nothing passes for the wrong reason: every allowed use + // must be there, so a rename of the field or of a function fails here + // instead of silently emptying the scan. + seen := map[string]bool{} + for _, u := range uses { + seen[u.key()] = true + } + for k := range wakeFDUses { + if !seen[k] { + t.Errorf("the scan found no %q: the worker's wakeFD usage changed shape; update this guard with it", k) + } + } + for _, v := range scanEventfdNumber(t, ".") { + t.Error(v) + } + if wakeBody != "w.wakeFD.Signal()" { + t.Errorf("(*worker).wake is %q, want exactly the statement w.wakeFD.Signal()", wakeBody) + } +} + +type wakeFDUse struct { + pos token.Position + fn string + method string +} + +func (u wakeFDUse) key() string { return u.fn + " " + u.method } + +func allowedList() string { + var ks []string + for k := range wakeFDUses { + ks = append(ks, k) + } + sort.Strings(ks) + return strings.Join(ks, ", ") +} + +// scanWakeFDUses parses the non-test .go files of dir. It returns every +// wakeFD use that is a method call or a composite-literal key, the uses that +// are anything else (a bare read of the field, a copy, a method value) as +// messages, and the source of (*worker).wake's body. +func scanWakeFDUses(t *testing.T, dir string) (uses []wakeFDUse, bad []string, wakeBody string) { + t.Helper() + files, err := filepath.Glob(filepath.Join(dir, "*.go")) + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + parsed := 0 + for _, name := range files { + if strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + f, err := parser.ParseFile(fset, name, src, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + parsed++ + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + if fd.Name.Name == "wake" && fd.Recv != nil && recvName(fd) == "worker" { + wakeBody = bodySource(fset, src, fd.Body) + } + handled := map[*ast.SelectorExpr]bool{} + ast.Inspect(fd.Body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.CallExpr: + if sel, ok := n.Fun.(*ast.SelectorExpr); ok { + if in, ok := sel.X.(*ast.SelectorExpr); ok && in.Sel.Name == "wakeFD" { + handled[in] = true + uses = append(uses, wakeFDUse{fset.Position(n.Pos()), fd.Name.Name, sel.Sel.Name}) + } + } + case *ast.KeyValueExpr: + if id, ok := n.Key.(*ast.Ident); ok && id.Name == "wakeFD" { + uses = append(uses, wakeFDUse{fset.Position(n.Pos()), fd.Name.Name, ""}) + } + } + return true + }) + ast.Inspect(fd.Body, func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok && sel.Sel.Name == "wakeFD" && !handled[sel] { + bad = append(bad, fmt.Sprintf("%s: %s reads the wakeFD field without calling a method on it (a copy or a method value escapes WakeFD's lock)", + fset.Position(sel.Pos()), fd.Name.Name)) + } + return true + }) + } + } + if parsed == 0 { + t.Fatalf("no non-test .go file in %s", dir) + } + return uses, bad, wakeBody +} + +func recvName(fd *ast.FuncDecl) string { + if fd.Recv == nil || len(fd.Recv.List) == 0 { + return "" + } + switch x := fd.Recv.List[0].Type.(type) { + case *ast.StarExpr: + if id, ok := x.X.(*ast.Ident); ok { + return id.Name + } + case *ast.Ident: + return x.Name + } + return "" +} + +// bodySource is the statements of a function body, without braces and +// surrounding space, comments excluded by taking the statement sources. +func bodySource(fset *token.FileSet, src []byte, b *ast.BlockStmt) string { + var parts []string + for _, s := range b.List { + parts = append(parts, string(src[fset.Position(s.Pos()).Offset:fset.Position(s.End()).Offset])) + } + return strings.Join(parts, "; ") +} + +// scanEventfdNumber checks the two rules about the raw eventfd number: in +// newWorker the identifier efd appears only as an argument of wakefd.New, +// unix.EpollCtl or unix.Close; and a raw write call (unix.Write, syscall.Write +// and their variants) appears only in flushLocked, on c.fd. +func scanEventfdNumber(t *testing.T, dir string) (bad []string) { + t.Helper() + files, err := filepath.Glob(filepath.Join(dir, "*.go")) + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + sawNewWorker, sawFlush := false, false + for _, name := range files { + if strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + f, err := parser.ParseFile(fset, name, src, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + if fd.Recv == nil && fd.Name.Name == "newWorker" { + sawNewWorker = true + bad = append(bad, efdUses(fset, fd)...) + } + ast.Inspect(fd.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok || (pkg.Name != "unix" && pkg.Name != "syscall") || !strings.HasPrefix(sel.Sel.Name, "Write") && !strings.HasPrefix(sel.Sel.Name, "Pwrite") { + return true + } + first := "" + if len(call.Args) > 0 { + first = string(src[fset.Position(call.Args[0].Pos()).Offset:fset.Position(call.Args[0].End()).Offset]) + } + if fd.Name.Name == "flushLocked" && first == "c.fd" { + sawFlush = true + return true + } + bad = append(bad, fmt.Sprintf("%s: %s calls %s.%s(%s, ...); the only raw write allowed is flushLocked's, on c.fd (a producer's wake must be wakefd.WakeFD.Signal, celeris#890 WAKEATOMIC)", + fset.Position(call.Pos()), fd.Name.Name, pkg.Name, sel.Sel.Name, first)) + return true + }) + } + } + if !sawNewWorker || !sawFlush { + bad = append(bad, fmt.Sprintf("the scan did not find newWorker (%v) or flushLocked's write (%v): the code changed shape; update this guard with it", sawNewWorker, sawFlush)) + } + return bad +} + +// efdUses reports every use of the identifier efd in fd that is not an +// argument of wakefd.New, unix.EpollCtl or unix.Close (anywhere beneath the +// EpollCtl call, for int32(efd) in the event). +func efdUses(fset *token.FileSet, fd *ast.FuncDecl) (bad []string) { + allowed := map[*ast.Ident]bool{} + ast.Inspect(fd.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok { + return true + } + switch pkg.Name + "." + sel.Sel.Name { + case "wakefd.New", "unix.Close": + for _, a := range call.Args { + if id, ok := a.(*ast.Ident); ok && id.Name == "efd" { + allowed[id] = true + } + } + case "unix.EpollCtl": + ast.Inspect(call, func(m ast.Node) bool { + if id, ok := m.(*ast.Ident); ok && id.Name == "efd" { + allowed[id] = true + } + return true + }) + } + return true + }) + ast.Inspect(fd.Body, func(n ast.Node) bool { + // The definition (efd, err := unix.Eventfd(...)) is an Ident on the + // left of a define; skip it. + if as, ok := n.(*ast.AssignStmt); ok && as.Tok == token.DEFINE { + for _, l := range as.Lhs { + if id, ok := l.(*ast.Ident); ok && id.Name == "efd" { + allowed[id] = true + } + } + } + if id, ok := n.(*ast.Ident); ok && id.Name == "efd" && !allowed[id] { + bad = append(bad, fmt.Sprintf("%s: newWorker uses the eventfd number efd outside wakefd.New, unix.EpollCtl and unix.Close (a copy of the number in another place can be written by a producer, celeris#890 WAKEATOMIC)", fset.Position(id.Pos()))) + } + return true + }) + return bad +} diff --git a/driver/memcached/conn_unix.go b/driver/memcached/conn_unix.go index b09748534..0c02a0e8c 100644 --- a/driver/memcached/conn_unix.go +++ b/driver/memcached/conn_unix.go @@ -10,6 +10,7 @@ import ( "syscall" "time" + "github.com/goceleris/celeris/driver/internal/eventloop" "github.com/goceleris/celeris/internal/engine" ) @@ -90,7 +91,7 @@ func dialMemcachedConn(ctx context.Context, prov engine.EventLoopProvider, cfg C c.pollIsDoneFn = c.pollIsDone c.lastUsedAt.Store(time.Now().UnixNano()) - if err := loop.RegisterConn(fd, c.onRecvFn, c.onCloseFn); err != nil { + if err := eventloop.RegisterConnWait(ctx, loop, fd, c.onRecvFn, c.onCloseFn); err != nil { _ = file.Close() return nil, err } diff --git a/driver/memcached/dial_refused_887_test.go b/driver/memcached/dial_refused_887_test.go new file mode 100644 index 000000000..1a2142463 --- /dev/null +++ b/driver/memcached/dial_refused_887_test.go @@ -0,0 +1,87 @@ +//go:build unix + +package memcached + +// celeris#887 item 28 (see driver/redis/dial_refused_887_test.go): the dial +// must wait out a RegisterConn refusal that the io_uring engine gives for a +// number whose previous conn is not finalized yet. + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +type refusingProvider struct { + engine.EventLoopProvider + refusals atomic.Int32 + calls atomic.Int32 + err error +} + +func (p *refusingProvider) WorkerLoop(n int) engine.WorkerLoop { + return refusingLoop{WorkerLoop: p.EventLoopProvider.WorkerLoop(n), p: p} +} + +type refusingLoop struct { + engine.WorkerLoop + p *refusingProvider +} + +func (l refusingLoop) RegisterConn(fd int, onRecv func([]byte), onClose func(error)) error { + l.p.calls.Add(1) + if r := l.p.refusals.Load(); r != 0 { + if r > 0 { + l.p.refusals.Add(-1) + } + return l.p.err + } + return l.WorkerLoop.RegisterConn(fd, onRecv, onClose) +} + +func newRefusingProvider(t *testing.T, refusals int32, err error) *refusingProvider { + t.Helper() + prov, err2 := eventloop.Resolve(nil) + if err2 != nil { + t.Fatal(err2) + } + t.Cleanup(func() { eventloop.Release(prov) }) + p := &refusingProvider{EventLoopProvider: prov, err: err} + p.refusals.Store(refusals) + return p +} + +func TestDialWaitsOutAnUnfinalizedNumber887(t *testing.T) { + p := newRefusingProvider(t, 3, errors.New("celeris/iouring: fd already registered")) + fake := startFake(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := dialMemcachedConn(ctx, p, Config{Addr: fake.Addr()}, 0) + if err != nil { + t.Fatalf("dial: %v (RegisterConn was called %d times)", err, p.calls.Load()) + } + _ = c.Close() + if got := p.calls.Load(); got != 4 { + t.Fatalf("RegisterConn calls = %d, want 4 (3 refusals, then the registration)", got) + } +} + +func TestDialDoesNotRetryOtherRegisterErrors887(t *testing.T) { + other := errors.New("celeris/iouring: fd 7 is already an HTTP connection") + p := newRefusingProvider(t, -1, other) + fake := startFake(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := dialMemcachedConn(ctx, p, Config{Addr: fake.Addr()}, 0) + if !errors.Is(err, other) { + t.Fatalf("dial error = %v, want %v", err, other) + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("RegisterConn calls = %d, want 1", got) + } +} diff --git a/driver/postgres/conn_unix.go b/driver/postgres/conn_unix.go index 5fed66826..4ae9d14a9 100644 --- a/driver/postgres/conn_unix.go +++ b/driver/postgres/conn_unix.go @@ -11,6 +11,7 @@ import ( "time" "github.com/goceleris/celeris/driver/internal/async" + "github.com/goceleris/celeris/driver/internal/eventloop" "github.com/goceleris/celeris/internal/driver/postgres/protocol" "github.com/goceleris/celeris/internal/engine" ) @@ -116,7 +117,7 @@ func dialConn(ctx context.Context, prov engine.EventLoopProvider, closeLoop func c.onCloseFn = c.onClose c.lastUsedAt.Store(time.Now().UnixNano()) - if err := loop.RegisterConn(fd, c.onRecvFn, c.onCloseFn); err != nil { + if err := eventloop.RegisterConnWait(ctx, loop, fd, c.onRecvFn, c.onCloseFn); err != nil { // Close via file.Close() to disarm the *os.File finalizer. A stray // syscall.Close(fd) here would leave the finalizer armed; when GC // later fires it would close the same fd a SECOND time, possibly diff --git a/driver/postgres/dial_refused_887_test.go b/driver/postgres/dial_refused_887_test.go new file mode 100644 index 000000000..a7decb72f --- /dev/null +++ b/driver/postgres/dial_refused_887_test.go @@ -0,0 +1,106 @@ +//go:build unix + +package postgres + +// celeris#887 item 28 (see driver/redis/dial_refused_887_test.go): the dial +// must wait out a RegisterConn refusal that the io_uring engine gives for a +// number whose previous conn is not finalized yet. + +import ( + "context" + "errors" + "net" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +type refusingProvider struct { + engine.EventLoopProvider + refusals atomic.Int32 + calls atomic.Int32 + err error +} + +func (p *refusingProvider) WorkerLoop(n int) engine.WorkerLoop { + return refusingLoop{WorkerLoop: p.EventLoopProvider.WorkerLoop(n), p: p} +} + +type refusingLoop struct { + engine.WorkerLoop + p *refusingProvider +} + +func (l refusingLoop) RegisterConn(fd int, onRecv func([]byte), onClose func(error)) error { + l.p.calls.Add(1) + if r := l.p.refusals.Load(); r != 0 { + if r > 0 { + l.p.refusals.Add(-1) + } + return l.p.err + } + return l.WorkerLoop.RegisterConn(fd, onRecv, onClose) +} + +// hangUpListener accepts and closes every conn at once: a dial that gets past +// RegisterConn then fails in the startup (EOF), quickly, which is not the +// failure under test. +func hangUpListener(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { + defer close(done) + for { + c, err := ln.Accept() + if err != nil { + return + } + _ = c.Close() + } + }() + t.Cleanup(func() { + _ = ln.Close() + <-done + }) + return ln.Addr().String() +} + +func TestDialWaitsOutAnUnfinalizedNumber887(t *testing.T) { + prov, err := eventloop.Resolve(nil) + if err != nil { + t.Fatal(err) + } + defer eventloop.Release(prov) + p := &refusingProvider{EventLoopProvider: prov, err: errors.New("celeris/iouring: fd already registered")} + p.refusals.Store(3) + + host, port, _ := net.SplitHostPort(hangUpListener(t)) + dsn := DSN{ + Host: host, Port: port, User: "u", Database: "d", + Options: Options{SSLMode: "disable", StatementCacheSize: 16}, + Params: map[string]string{}, + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := dialConn(ctx, p, nil, dsn, 0) + if err == nil { + _ = c.Close() + t.Fatal("dial against a server that hangs up succeeded") + } + // Past the refusals the dial reaches the startup, which the server cannot + // answer; the refusal itself must not be what the caller sees. + if strings.Contains(err.Error(), "already registered") { + t.Fatalf("dial returned the RegisterConn refusal: %v (RegisterConn was called %d times)", err, p.calls.Load()) + } + if got := p.calls.Load(); got != 4 { + t.Fatalf("RegisterConn calls = %d, want 4 (3 refusals, then the registration)", got) + } +} diff --git a/driver/redis/conn_unix.go b/driver/redis/conn_unix.go index b341e2c51..5d464474f 100644 --- a/driver/redis/conn_unix.go +++ b/driver/redis/conn_unix.go @@ -10,6 +10,7 @@ import ( "syscall" "time" + "github.com/goceleris/celeris/driver/internal/eventloop" "github.com/goceleris/celeris/internal/engine" ) @@ -92,7 +93,7 @@ func dialRedisConn(ctx context.Context, prov engine.EventLoopProvider, cfg Confi c.pollBeforeRearmFn = c.pollBeforeRearm c.lastUsedAt.Store(time.Now().UnixNano()) - if err := loop.RegisterConn(fd, c.onRecvFn, c.onCloseFn); err != nil { + if err := eventloop.RegisterConnWait(ctx, loop, fd, c.onRecvFn, c.onCloseFn); err != nil { // Close via file.Close() to disarm the *os.File finalizer; a plain // syscall.Close(fd) would leak the finalizer and risk a double // close on a reassigned fd when GC later runs it. diff --git a/driver/redis/dial_refused_887_test.go b/driver/redis/dial_refused_887_test.go new file mode 100644 index 000000000..61029f1d7 --- /dev/null +++ b/driver/redis/dial_refused_887_test.go @@ -0,0 +1,124 @@ +//go:build unix + +package redis + +// celeris#887 item 28, deterministic half: a WorkerLoop that refuses +// RegisterConn the way the io_uring engine does for a number whose previous +// conn is not finalized yet ("celeris/iouring: fd already registered", see +// engine.WorkerLoop.UnregisterConn). The real-engine half is +// TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887. + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +// refusingProvider wraps the standalone loop. Its workers refuse RegisterConn +// with err while refusals is not 0 (a negative value refuses for ever) and +// count every call. +type refusingProvider struct { + engine.EventLoopProvider + refusals atomic.Int32 + calls atomic.Int32 + err error +} + +func newRefusingProvider(t *testing.T, refusals int32, err error) *refusingProvider { + t.Helper() + prov, err2 := eventloop.Resolve(nil) + if err2 != nil { + t.Fatal(err2) + } + t.Cleanup(func() { eventloop.Release(prov) }) + p := &refusingProvider{EventLoopProvider: prov, err: err} + p.refusals.Store(refusals) + return p +} + +func (p *refusingProvider) WorkerLoop(n int) engine.WorkerLoop { + return refusingLoop{WorkerLoop: p.EventLoopProvider.WorkerLoop(n), p: p} +} + +type refusingLoop struct { + engine.WorkerLoop + p *refusingProvider +} + +func (l refusingLoop) RegisterConn(fd int, onRecv func([]byte), onClose func(error)) error { + l.p.calls.Add(1) + if r := l.p.refusals.Load(); r != 0 { + if r > 0 { + l.p.refusals.Add(-1) + } + return l.p.err + } + return l.WorkerLoop.RegisterConn(fd, onRecv, onClose) +} + +var errIOUringBusy887 = errors.New("celeris/iouring: fd already registered") + +func dialRefusedCfg(t *testing.T) Config { + t.Helper() + fake := startFakeRedis(t, nil) + return Config{Addr: fake.Addr(), ForceRESP2: true} +} + +// A refusal that clears within a few milliseconds must not fail the dial. +func TestDialWaitsOutAnUnfinalizedNumber887(t *testing.T) { + p := newRefusingProvider(t, 3, errIOUringBusy887) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := dialRedisConn(ctx, p, dialRefusedCfg(t), 0) + if err != nil { + t.Fatalf("dial: %v (RegisterConn was called %d times)", err, p.calls.Load()) + } + _ = c.Close() + if got := p.calls.Load(); got != 4 { + t.Fatalf("RegisterConn calls = %d, want 4 (3 refusals, then the registration)", got) + } +} + +// A number that stays registered must not hold the dial for ever: the dial's +// ctx bounds the wait, and the refusal is what the caller sees. +func TestDialGivesUpWhenTheNumberStaysRegistered887(t *testing.T) { + p := newRefusingProvider(t, -1, errIOUringBusy887) + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + start := time.Now() + c, err := dialRedisConn(ctx, p, dialRefusedCfg(t), 0) + took := time.Since(start) + if err == nil { + _ = c.Close() + t.Fatal("dial succeeded against a loop that always refuses") + } + if !strings.Contains(err.Error(), "already registered") { + t.Fatalf("dial error = %v, want the refusal", err) + } + if took > time.Second { + t.Fatalf("dial took %v with a 300 ms ctx: the wait ignores the ctx", took) + } + t.Logf("gave up after %v and %d calls: %v", took, p.calls.Load(), err) +} + +// Only the "number not finalized yet" refusal is retried. Any other +// RegisterConn error is final at once. +func TestDialDoesNotRetryOtherRegisterErrors887(t *testing.T) { + other := errors.New("celeris/iouring: fd 7 is already an HTTP connection") + p := newRefusingProvider(t, -1, other) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := dialRedisConn(ctx, p, dialRefusedCfg(t), 0) + if !errors.Is(err, other) { + t.Fatalf("dial error = %v, want %v", err, other) + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("RegisterConn calls = %d, want 1", got) + } +} diff --git a/driver/redis/dial_reuse_887_linux_test.go b/driver/redis/dial_reuse_887_linux_test.go new file mode 100644 index 000000000..f0a3897e1 --- /dev/null +++ b/driver/redis/dial_reuse_887_linux_test.go @@ -0,0 +1,114 @@ +//go:build linux + +package redis + +// celeris#887 item 28 (the checklist's twenty-eighth box): a driver dial fails +// with "celeris/iouring: fd already registered" when the new socket gets the +// number of a conn that was just unregistered on the same worker and whose +// onClose has not run. engine.WorkerLoop.UnregisterConn documents the refusal; +// dialRedisConn handed it to the caller, so Pool.Acquire could fail under +// connection churn on the io_uring engine. + +import ( + "bufio" + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/sys/unix" + + "github.com/goceleris/celeris/driver/internal/eventloop" +) + +// TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887 closes a conn and +// dials the next one at once, on each engine of the rig. The next socket takes +// the number of the conn just closed, and the io_uring engine keeps that +// number registered on its worker until the old conn is finalized. A dial that +// loses the race must wait for it (bounded) instead of returning the refusal. +func TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887(t *testing.T) { + eachRigEngine(t, func(t *testing.T, eng rigEngine) { + prov := rigProvider(t, eng) + fake := startFakeRedis(t, func(cmd []string, w *bufio.Writer) {}) + // ForceRESP2 with no password and no DB: the handshake sends nothing, + // so the fake needs no replies and the test measures the dial alone. + cfg := Config{Addr: fake.Addr(), ForceRESP2: true} + + // Four goroutines close and dial without pause: the churn makes the + // next socket take a number that was just released, on a worker that + // has not finalized the conn that held it. + const goroutines, perG = 4, 500 + var refused, other atomic.Int32 + var mu sync.Mutex + var first error + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < perG; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + c, err := dialRedisConn(ctx, prov, cfg, -1) + cancel() + if err != nil { + if strings.Contains(err.Error(), "already registered") { + refused.Add(1) + } else { + other.Add(1) + } + mu.Lock() + if first == nil { + first = err + } + mu.Unlock() + continue + } + _ = c.Close() + } + }() + } + wg.Wait() + rounds := goroutines * perG + t.Logf("engine %s: %d dials, %d refused with 'already registered', %d failed otherwise", eng, rounds, refused.Load(), other.Load()) + if first != nil { + t.Errorf("a dial right after a close failed (%d of %d refused, %d other); first error: %v", refused.Load(), rounds, other.Load(), first) + } + }) +} + +// TestTheIOUringRefusalIsWhatTheDialWaitsOut887 pins the match of +// eventloop.IsNumberBusy to the text a real io_uring worker returns: the +// engine has no sentinel for it, so a reworded message would silently turn the +// dial's wait off. +func TestTheIOUringRefusalIsWhatTheDialWaitsOut887(t *testing.T) { + prov := rigProvider(t, rigIOUring) + wl := prov.WorkerLoop(0) + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_NONBLOCK|unix.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + a, b := fds[0], fds[1] + t.Cleanup(func() { _ = unix.Close(b) }) + if err := wl.RegisterConn(a, func([]byte) {}, func(error) {}); err != nil { + _ = unix.Close(a) + t.Fatalf("first RegisterConn: %v", err) + } + t.Cleanup(func() { + _ = wl.UnregisterConn(a) + _ = unix.Close(a) + }) + err = wl.RegisterConn(a, func([]byte) {}, func(error) {}) + if err == nil { + t.Fatal("registering the same number twice succeeded") + } + if !eventloop.IsNumberBusy(err) { + t.Fatalf("IsNumberBusy does not recognise the io_uring engine's refusal: %q", err) + } + other := "celeris/iouring: fd 7 is already an HTTP connection" + if eventloop.IsNumberBusy(errors.New(other)) { + t.Fatalf("IsNumberBusy accepts %q", other) + } +} diff --git a/driver/redis/pool.go b/driver/redis/pool.go index fa683063f..40bd75f60 100644 --- a/driver/redis/pool.go +++ b/driver/redis/pool.go @@ -3,6 +3,7 @@ package redis import ( "context" "runtime" + "sync" "time" "github.com/goceleris/celeris/driver/internal/async" @@ -20,6 +21,8 @@ type redisPool struct { cfg Config provider engine.EventLoopProvider ownsLoop bool + // closeOnce makes Close idempotent. + closeOnce sync.Once } // newDirectCmdPool creates a pool whose cmd conns dial via direct @@ -271,13 +274,21 @@ func (p *redisPool) releasePubSub(c *redisConn) { } // Close closes both pools and releases the event loop if owned. +// +// It is safe to call from several goroutines and more than once: the body runs +// once, and a second caller returns at once. Releasing the shared standalone +// loop twice would drop a reference that belongs to another client +// (celeris#887 item 29: SentinelClient.Close and the give-up of a sentinel +// closed the same Client). func (p *redisPool) Close() error { - _ = p.cmd.Close() - _ = p.pubsub.Close() - if p.ownsLoop && p.provider != nil { - eventloop.Release(p.provider) - p.provider = nil - } + p.closeOnce.Do(func() { + _ = p.cmd.Close() + _ = p.pubsub.Close() + if p.ownsLoop && p.provider != nil { + eventloop.Release(p.provider) + p.provider = nil + } + }) return nil } diff --git a/driver/redis/pubsub.go b/driver/redis/pubsub.go index a199ba163..3c881c1bd 100644 --- a/driver/redis/pubsub.go +++ b/driver/redis/pubsub.go @@ -10,6 +10,11 @@ import ( "github.com/goceleris/celeris/driver/internal/async" ) +// pubsubStableAfter is how long a re-established pub/sub conn must stay up +// before the reconnect backoff starts over from its base delay (celeris#887 +// item 30). A conn that dies sooner is a failed attempt. +var pubsubStableAfter = 5 * time.Second + // Message is one pub/sub notification. Pattern is set only for PSUBSCRIBE // deliveries ("pmessage" push type). Shard is true for shard channel // deliveries ("smessage" push type, Redis 7+). @@ -91,6 +96,20 @@ type PubSub struct { // conn teardown. reconnectingMu sync.Mutex reconnecting bool + // The outage state below is guarded by reconnectingMu (celeris#887 items + // 29 and 30). An outage starts at the first drop of a conn and ends when a + // replayed conn has stayed up for pubsubStableAfter. The backoff runs over + // the whole outage: a conn that dies soon after its replay is one more + // failed attempt, not a fresh start. + // + // downSince is the start of the current outage, zero when there is none. + // upSince is when the last replay succeeded. attempt is the backoff + // attempt the next reconnect loop of this outage starts at. + downSince time.Time + upSince time.Time + attempt int + // stableAfter is pubsubStableAfter as of the PubSub's creation. + stableAfter time.Duration closeMsgOnce sync.Once @@ -175,6 +194,9 @@ func (c *Client) newPubSub(ctx context.Context) (*PubSub, error) { msgCh: make(chan *Message, defaultPubSubChanBuf), closeCh: make(chan struct{}), backoff: async.NewBackoff(50*time.Millisecond, 5*time.Second), + // Read once: the goroutines of the PubSub never touch the package + // variable, which tests set. + stableAfter: pubsubStableAfter, } ps.bindConn(conn) // The conn may have closed while the pool was handing it out, before the @@ -456,10 +478,46 @@ func (ps *PubSub) onConnDrop(_ error) { return } ps.reconnecting = true + if now := time.Now(); ps.downSince.IsZero() || (!ps.upSince.IsZero() && now.Sub(ps.upSince) >= ps.stableAfter) { + // A new outage: this is the first drop, or the conn that replaced the + // last one stayed up long enough to count as recovered. + ps.downSince = now + ps.attempt = 0 + ps.backoff.Reset() + } ps.reconnectingMu.Unlock() go ps.reconnectLoop() } +// downFor reports how long the PubSub has been without a conn it can rely on: +// zero when there is none of that, as for a PubSub whose conn is up and has +// been since its replay stayed up for pubsubStableAfter. During an outage it +// is the time since the outage began while no conn is up or a replay is in +// progress; once a conn is up again it is the time that outage lasted until +// that conn came up, so a conn that keeps dying soon after its replay shows a +// growing figure, and a blip that healed does not (celeris#887 item 29; +// SentinelClient reads it to give up a sentinel that went away). +func (ps *PubSub) downFor() time.Duration { + ps.mu.Lock() + cur := ps.conn + ps.mu.Unlock() + up := cur != nil && !cur.closed.Load() + now := time.Now() + ps.reconnectingMu.Lock() + defer ps.reconnectingMu.Unlock() + if ps.downSince.IsZero() { + return 0 + } + if up && !ps.reconnecting && !ps.upSince.IsZero() { + if now.Sub(ps.upSince) >= ps.stableAfter { + ps.downSince = time.Time{} // recovered: the next drop starts a new outage + return 0 + } + return ps.upSince.Sub(ps.downSince) + } + return now.Sub(ps.downSince) +} + // reconnectLoop re-dials the pubsub conn and replays subscriptions with // exponential backoff. It terminates when the pubsub is closed or the backoff // reaches its cap and a configured retry budget is exhausted (here we keep @@ -481,11 +539,19 @@ func (ps *PubSub) reconnectLoop() { ps.onConnDrop(nil) } }() - for attempt := 0; ; attempt++ { + ps.reconnectingMu.Lock() + attempt := ps.attempt + ps.reconnectingMu.Unlock() + for ; ; attempt++ { if ps.closed.Load() { return } delay := ps.backoff.Next(attempt) + // The next loop of this outage (a conn that dies right after its + // replay starts one) continues from the next attempt. + ps.reconnectingMu.Lock() + ps.attempt = attempt + 1 + ps.reconnectingMu.Unlock() select { case <-time.After(delay): case <-ps.closeCh: @@ -579,7 +645,12 @@ func (ps *PubSub) reconnectLoop() { ps.client.pool.releasePubSub(failed) } if ok { - ps.backoff.Reset() + // No Reset: the backoff starts over when a drop finds this conn + // has stayed up for pubsubStableAfter (onConnDrop), not when the + // replay returns (celeris#887 item 30). + ps.reconnectingMu.Lock() + ps.upSince = time.Now() + ps.reconnectingMu.Unlock() return } } diff --git a/driver/redis/pubsub_backoff_887_linux_test.go b/driver/redis/pubsub_backoff_887_linux_test.go new file mode 100644 index 000000000..003ac1615 --- /dev/null +++ b/driver/redis/pubsub_backoff_887_linux_test.go @@ -0,0 +1,136 @@ +//go:build linux + +package redis + +// celeris#887 item 30: the reconnect loop reset its backoff as soon as the +// replay of the subscriptions returned, so a server that hangs up just after +// the replay was retried at the base delay for ever (the loop's own in-loop +// check covers only a conn that dies during the replay, item 15). The backoff +// must start over only after the replayed conn has stayed up for a while +// (pubsubStableAfter). + +import ( + "bufio" + "context" + "sync" + "testing" + "time" +) + +// hangUpServer is a fake server that answers SUBSCRIBE and then closes the +// conn after lifetime(n), n counting the SUBSCRIBEs it has seen from 1 (0 +// keeps the conn). It records when each SUBSCRIBE arrived. +type hangUpServer struct { + fake *fakeRedis + lifetime func(n int) time.Duration + mu sync.Mutex + stamps []time.Time +} + +func startHangUpServer(t *testing.T, lifetime func(n int) time.Duration) *hangUpServer { + t.Helper() + s := &hangUpServer{lifetime: lifetime} + ready := make(chan struct{}) + s.fake = startFakeRedis(t, func(cmd []string, w *bufio.Writer) { + <-ready + if len(cmd) == 0 || cmd[0] != "SUBSCRIBE" { + writeSimple(w, "OK") + return + } + writeArrayHeader(w, 3) + writeBulk(w, "subscribe") + writeBulk(w, cmd[1]) + writeInt(w, 1) + s.mu.Lock() + s.stamps = append(s.stamps, time.Now()) + n := len(s.stamps) + s.mu.Unlock() + if d := s.lifetime(n); d > 0 { + s.fake.mu.Lock() + c := s.fake.conns[len(s.fake.conns)-1] + s.fake.mu.Unlock() + time.AfterFunc(d, func() { _ = c.Close() }) + } + }) + close(ready) + return s +} + +func (s *hangUpServer) times() []time.Time { + s.mu.Lock() + defer s.mu.Unlock() + return append([]time.Time(nil), s.stamps...) +} + +func subscribeToHangUpServer(t *testing.T, s *hangUpServer) *PubSub { + t.Helper() + cl, err := NewClient(s.fake.Addr(), WithForceRESP2()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cl.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ps, err := cl.Subscribe(ctx, "news") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ps.Close() }) + return ps +} + +// The server hangs up 10 ms after every SUBSCRIBE. The attempts must back off: +// the base delay is 50 ms and doubles up to 5 s, so in 2.5 s the server sees +// about seven SUBSCRIBEs (50+100+200+400+800+1600 ms of delay), not the fifty +// of a loop that starts over from 50 ms every time. +func TestReconnectBacksOffWhenTheServerHangsUpAfterEveryReplay887(t *testing.T) { + s := startHangUpServer(t, func(int) time.Duration { return 10 * time.Millisecond }) + subscribeToHangUpServer(t, s) + time.Sleep(2500 * time.Millisecond) + ts := s.times() + for i := 1; i < len(ts); i++ { + t.Logf("SUBSCRIBE %d came %v after the previous one", i+1, ts[i].Sub(ts[i-1]).Round(time.Millisecond)) + } + if len(ts) > 12 { + t.Errorf("the server saw %d SUBSCRIBEs in 2.5 s: the reconnect backoff started over after every replay", len(ts)) + } + if n := len(ts); n >= 5 { + if gap := ts[n-1].Sub(ts[n-2]); gap < 250*time.Millisecond { + t.Errorf("the gap before SUBSCRIBE %d is %v, want >= 250 ms: the delay did not grow", n, gap) + } + } +} + +// A conn that stays up for pubsubStableAfter ends the outage: the next drop is +// retried from the base delay again. +func TestReconnectBackoffStartsOverAfterTheConnWasStable887(t *testing.T) { + old := pubsubStableAfter + pubsubStableAfter = 400 * time.Millisecond + t.Cleanup(func() { pubsubStableAfter = old }) + + // SUBSCRIBEs 1..5 are cut after 10 ms; the sixth lives 800 ms (longer than + // the stable window), then everything is cut after 10 ms again. + s := startHangUpServer(t, func(n int) time.Duration { + if n == 6 { + return 800 * time.Millisecond + } + return 10 * time.Millisecond + }) + subscribeToHangUpServer(t, s) + waitFor(t, "the seventh SUBSCRIBE", 20*time.Second, func() bool { return len(s.times()) >= 7 }) + ts := s.times() + for i := 1; i < len(ts); i++ { + t.Logf("SUBSCRIBE %d came %v after the previous one", i+1, ts[i].Sub(ts[i-1]).Round(time.Millisecond)) + } + // Before the stable conn the attempts back off (the fifth reconnect waits + // about 800 ms) ... + if gap := ts[4].Sub(ts[3]); gap < 250*time.Millisecond { + t.Errorf("the gap before SUBSCRIBE 5 is %v, want >= 250 ms: the delay did not grow", gap) + } + // ... and after the conn that lived 800 ms the next reconnect is at the + // base delay again. + gap := ts[6].Sub(ts[5]) - 800*time.Millisecond + if gap > 250*time.Millisecond { + t.Errorf("the reconnect after a stable conn waited %v, want about the base delay (50 ms): the backoff did not start over", gap) + } +} diff --git a/driver/redis/pubsub_ctlwrite_929_linux_test.go b/driver/redis/pubsub_ctlwrite_929_linux_test.go index 01ec0c7a3..63c5f9d0d 100644 --- a/driver/redis/pubsub_ctlwrite_929_linux_test.go +++ b/driver/redis/pubsub_ctlwrite_929_linux_test.go @@ -268,22 +268,11 @@ func newPubsubRig(t *testing.T, eng rigEngine) *pubsubRig { // registered on its worker until the old conn's onClose has run, so its // RegisterConn refuses the new conn with "fd already registered" until then // (documented on engine.WorkerLoop.UnregisterConn; with one worker, as on a -// runner whose memlock funds one, every number lands on the same worker). -// The dial is retried for a while on exactly that error (celeris#887 item 10). -// -// The retry hides a real dial failure: Pool.Acquire can fail the same way for a -// caller under connection churn. It is tracked as the item "drivers, io_uring -// dial: ... fd already registered" of the celeris#887 checklist. Remove this -// retry when that dial is fixed, so the test sees the failure again. +// runner whose memlock funds one, every number lands on the same worker). The +// dial waits that out itself (eventloop.RegisterConnWait, celeris#887 item +// 28); this used to retry the dial here, which hid the failure from the test. func (r *pubsubRig) newPubSubAfterReuse(ctx context.Context) (*PubSub, error) { - deadline := time.Now().Add(5 * time.Second) - for { - ps, err := r.cl.newPubSub(ctx) - if err == nil || !strings.Contains(err.Error(), "already registered") || time.Now().After(deadline) { - return ps, err - } - time.Sleep(5 * time.Millisecond) - } + return r.cl.newPubSub(ctx) } func (r *pubsubRig) dropServerSide() { diff --git a/driver/redis/sentinel.go b/driver/redis/sentinel.go index 0d171996e..f4d1a14db 100644 --- a/driver/redis/sentinel.go +++ b/driver/redis/sentinel.go @@ -37,6 +37,13 @@ type SentinelConfig struct { Engine ServerProvider } +// sentinelDownGrace is how long watchFailover lets the PubSub of the sentinel +// it follows stay down (see PubSub.downFor) before it gives that sentinel up +// and lets subscribeLoop connect to the next address of SentinelAddrs +// (celeris#887 item 29). A sentinel that restarts within it keeps the +// subscription. +var sentinelDownGrace = 5 * time.Second + // SentinelClient is a Redis client managed by Redis Sentinel. It discovers the // current primary via Sentinel, connects to it, and automatically reconnects // to the new primary when a failover is detected. @@ -50,6 +57,10 @@ type SentinelClient struct { closeCh chan struct{} closed atomic.Bool + + // downGrace is sentinelDownGrace as of construction; zero (a client built + // without NewSentinelClient) means the default. + downGrace time.Duration } type sentinelConn struct { @@ -73,8 +84,9 @@ func NewSentinelClient(cfg SentinelConfig) (*SentinelClient, error) { cfg.DialTimeout = defaultDialTimeout } s := &SentinelClient{ - cfg: cfg, - closeCh: make(chan struct{}), + cfg: cfg, + closeCh: make(chan struct{}), + downGrace: sentinelDownGrace, } masterAddr, err := s.discoverMaster() if err != nil { @@ -235,11 +247,16 @@ func (s *SentinelClient) dialMaster(addr string) (*Client, error) { func (s *SentinelClient) subscribeLoop() { bo := async.NewBackoff(100*time.Millisecond, 10*time.Second) attempt := 0 + // start is the index of the SentinelAddrs entry connectSentinel tries + // first: 0, and after a sentinel was given up the one after it, so that a + // sentinel that accepts and hangs up does not take the subscription back + // from the next address for ever (celeris#887 item 29). + start := 0 for { if s.closed.Load() { return } - sc, err := s.connectSentinel() + sc, err := s.connectSentinelFrom(start) if err != nil { delay := bo.Next(attempt) attempt++ @@ -263,21 +280,81 @@ func (s *SentinelClient) subscribeLoop() { // exited on read error (that's what triggered this // reconnect iteration). s.sentinelMu.Lock() + if s.closed.Load() { + // Close ran between the check at the top of the loop and here: its + // list was already cleared, so it will never close sc. Whoever + // takes a sentinel out of the list, or declines to put it there, + // closes it (celeris#887 item 29). + s.sentinelMu.Unlock() + s.closeSentinel(sc) + return + } for _, old := range s.sentinelConns { + // The PubSub first: its conn is checked out of the client's pool, + // so closing the client does not stop its reconnect loop. + if old != nil && old.subPS != nil { + _ = old.subPS.Close() + } if old != nil && old.client != nil { _ = old.client.Close() } } s.sentinelConns = append(s.sentinelConns[:0], sc) s.sentinelMu.Unlock() - s.watchFailover(sc) + if s.watchFailover(sc) { + start = s.sentinelIndex(sc.addr) + 1 + } } } +// takeSentinel removes sc from sentinelConns and reports whether it was there. +// The caller that gets true owns sc's close. +func (s *SentinelClient) takeSentinel(sc *sentinelConn) bool { + s.sentinelMu.Lock() + defer s.sentinelMu.Unlock() + for i, c := range s.sentinelConns { + if c == sc { + s.sentinelConns = append(s.sentinelConns[:i], s.sentinelConns[i+1:]...) + return true + } + } + return false +} + +// closeSentinel closes sc's PubSub and then its client: the PubSub's conn is +// checked out of the client's pool, so closing the client alone does not stop +// its reconnect loop. +func (s *SentinelClient) closeSentinel(sc *sentinelConn) { + if sc.subPS != nil { + _ = sc.subPS.Close() + } + if sc.client != nil { + _ = sc.client.Close() + } +} + +// sentinelIndex is the index of addr in SentinelAddrs, or -1. +func (s *SentinelClient) sentinelIndex(addr string) int { + for i, a := range s.cfg.SentinelAddrs { + if a == addr { + return i + } + } + return -1 +} + // connectSentinel connects to the first responsive sentinel and subscribes to // the +switch-master channel. func (s *SentinelClient) connectSentinel() (*sentinelConn, error) { - for _, addr := range s.cfg.SentinelAddrs { + return s.connectSentinelFrom(0) +} + +// connectSentinelFrom is connectSentinel trying the addresses of +// SentinelAddrs in order from index start, wrapping around once. +func (s *SentinelClient) connectSentinelFrom(start int) (*sentinelConn, error) { + n := len(s.cfg.SentinelAddrs) + for k := 0; k < n; k++ { + addr := s.cfg.SentinelAddrs[((start+k)%n+n)%n] var opts []Option opts = append(opts, WithDialTimeout(s.cfg.DialTimeout)) if s.cfg.SentinelPassword != "" { @@ -317,22 +394,47 @@ func (s *SentinelClient) connectSentinel() (*sentinelConn, error) { return nil, errors.New("no responsive sentinel") } -// watchFailover reads +switch-master messages and triggers a master swap. -func (s *SentinelClient) watchFailover(sc *sentinelConn) { +// watchFailover reads +switch-master messages and triggers a master swap. It +// returns true when it gave the sentinel up: the sentinel's PubSub reconnects +// to the same address for ever, so a sentinel that went away would otherwise +// keep the subscription and the failovers of the other sentinels would never +// be heard. The sentinel is given up once its PubSub has been without a +// reliable conn for sentinelDownGrace (PubSub.downFor: gone, or accepting and +// hanging up again and again); the PubSub and the client are closed, and +// subscribeLoop moves on to the next address of SentinelAddrs. It returns +// false when the SentinelClient is closed. +func (s *SentinelClient) watchFailover(sc *sentinelConn) (gaveUp bool) { ch := sc.subPS.Channel() + grace := s.downGrace + if grace <= 0 { + grace = 5 * time.Second + } + tick := time.NewTicker(max(grace/4, 5*time.Millisecond)) + defer tick.Stop() for { select { case msg, ok := <-ch: if !ok { - return + return !s.closed.Load() } if msg == nil { continue } // +switch-master payload: "mymaster old-ip old-port new-ip new-port" s.handleSwitchMaster(msg.Payload) + case <-tick.C: + if sc.subPS.downFor() >= grace { + // Whoever takes sc out of sentinelConns closes it: if + // SentinelClient.Close got there first it has closed (or is + // closing, under sentinelMu) this very client, and a second + // close would race it. + if s.takeSentinel(sc) { + s.closeSentinel(sc) + } + return !s.closed.Load() + } case <-s.closeCh: - return + return false } } } diff --git a/driver/redis/sentinel_close_race_887_linux_test.go b/driver/redis/sentinel_close_race_887_linux_test.go new file mode 100644 index 000000000..1df02787b --- /dev/null +++ b/driver/redis/sentinel_close_race_887_linux_test.go @@ -0,0 +1,132 @@ +//go:build linux + +package redis + +// celeris#887 item 29, review round 1 of the PR: the give-up of watchFailover +// closed the sentinel's client outside sentinelMu while that sentinel was still +// listed, so SentinelClient.Close could close the same Client at the same time. +// redisPool.Close is not safe for that (it reads and writes p.provider with no +// lock and then releases the shared standalone loop), and the sentinel's +// clients, the master client and any other client of the process share that +// loop's refcount. Close racing a subscribeLoop that had already passed its +// closed check also left a freshly connected sentinel open for ever (pre- +// existing). + +import ( + "math/rand/v2" + "sync" + "testing" + "time" +) + +// Two goroutines close one Client. Under -race this is a data race on +// p.provider unless Close is idempotent under concurrency, and a double +// Release of the loop otherwise. +func TestClientCloseTwiceConcurrently887(t *testing.T) { + fake := startFakeRedis(t, nil) + // A bystander on the same standalone loop must keep its conn. + by := startHangUpServer(t, func(int) time.Duration { return 0 }) + ps := subscribeToHangUpServer(t, by) + conn0 := ps.connNow() + for i := 0; i < 50; i++ { + cl, err := NewClient(fake.Addr(), WithForceRESP2()) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + start := make(chan struct{}) + for g := 0; g < 2; g++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _ = cl.Close() + }() + } + close(start) + wg.Wait() + } + time.Sleep(100 * time.Millisecond) + if c := ps.connNow(); c != conn0 || c.closed.Load() { + t.Fatal("the bystander PubSub lost its conn: the standalone loop was released under it") + } +} + +// The give-up and SentinelClient.Close race: both sentinels hang up every 2 ms +// and the grace is 20 ms, so the client gives up one and moves to the other at +// about the rate the test closes it. A bystander on the shared loop must keep +// its conn; under -race the double close is reported. +func TestSentinelCloseVsGiveUp887(t *testing.T) { + oldGrace := sentinelDownGrace + sentinelDownGrace = 20 * time.Millisecond + t.Cleanup(func() { sentinelDownGrace = oldGrace }) + master := startFakeMaster(t) + a := startFakeSentinel(t, master.Addr()) + b := startFakeSentinel(t, master.Addr()) + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + go func() { + for { + select { + case <-stop: + return + case <-time.After(2 * time.Millisecond): + a.dropAll() + b.dropAll() + } + } + }() + by := startHangUpServer(t, func(int) time.Duration { return 0 }) + ps := subscribeToHangUpServer(t, by) + conn0 := ps.connNow() + for i := 0; i < 40; i++ { + sc, err := NewSentinelClient(SentinelConfig{MasterName: "mymaster", SentinelAddrs: []string{a.Addr(), b.Addr()}, DialTimeout: time.Second}) + if err != nil { + continue + } + time.Sleep(time.Duration(rand.IntN(150)) * time.Millisecond) + _ = sc.Close() + } + time.Sleep(300 * time.Millisecond) + if c := ps.connNow(); c != conn0 || c.closed.Load() { + t.Errorf("the bystander PubSub lost its conn: the standalone loop was released under it") + } +} + +// Close at a random moment after NewSentinelClient: after Close has returned +// no sentinel conn may stay open. Before the fix subscribeLoop appended the +// sentinel it had just connected without looking at the closed flag, so it +// stayed in sentinelConns, open, after Close had cleared the list. +func TestSentinelCloseDuringSubscribeLeaksNothing887(t *testing.T) { + master := startFakeMaster(t) + a := startFakeSentinel(t, master.Addr()) + leaks, iters := 0, 300 + for i := 0; i < iters; i++ { + sc, err := NewSentinelClient(SentinelConfig{ + MasterName: "mymaster", + SentinelAddrs: []string{a.Addr()}, + DialTimeout: 2 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Duration(rand.IntN(400)) * time.Microsecond) + _ = sc.Close() + time.Sleep(5 * time.Millisecond) + sc.sentinelMu.Lock() + var left *sentinelConn + if len(sc.sentinelConns) > 0 { + left = sc.sentinelConns[0] + } + sc.sentinelMu.Unlock() + if left != nil && !left.subPS.closed.Load() { + leaks++ + _ = left.subPS.Close() + _ = left.client.Close() + } + } + t.Logf("%d of %d iterations left an open sentinel PubSub and client after Close", leaks, iters) + if leaks > 0 { + t.Errorf("%d of %d Close calls left a sentinel connection open", leaks, iters) + } +} diff --git a/driver/redis/sentinel_down_887_linux_test.go b/driver/redis/sentinel_down_887_linux_test.go new file mode 100644 index 000000000..b8791a0d8 --- /dev/null +++ b/driver/redis/sentinel_down_887_linux_test.go @@ -0,0 +1,146 @@ +//go:build linux + +package redis + +// celeris#887 item 29: watchFailover waited on sc.subPS.Channel(), which a +// conn drop does not close: the sentinel's PubSub reconnected to the same +// address for ever. A sentinel that went away after a successful +// Subscribe("+switch-master") was never replaced by the next address of +// SentinelAddrs, and the failovers it would have announced were missed. +// +// Both shapes of "went away" are tested: the sentinel is gone for good (its +// listener closed), and the sentinel accepts every reconnect and hangs up at +// once, so its PubSub is never down for long. + +import ( + "net" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" +) + +func (fs *fakeSentinel) subscribers() int { + fs.mu.Lock() + defer fs.mu.Unlock() + return len(fs.subWriters) +} + +// dropAll closes every conn the sentinel has accepted so far. +func (fs *fakeSentinel) dropAll() { + fs.fake.mu.Lock() + conns := append([]net.Conn(nil), fs.fake.conns...) + fs.fake.conns = nil + fs.fake.mu.Unlock() + for _, c := range conns { + _ = c.Close() + } +} + +// flapEvery is the period of the hang-ups of the sentinel that stays up; 0 +// means the sentinel is gone for good. +// reconnectLoops counts the PubSub reconnect goroutines of the process. +func reconnectLoops() int { + buf := make([]byte, 1<<20) + buf = buf[:runtime.Stack(buf, true)] + return strings.Count(string(buf), "(*PubSub).reconnectLoop") +} + +func sentinelMovesOn887(t *testing.T, flapEvery time.Duration) { + goneForGood := flapEvery == 0 + oldGrace := sentinelDownGrace + sentinelDownGrace = 600 * time.Millisecond + t.Cleanup(func() { sentinelDownGrace = oldGrace }) + + master1 := startFakeMaster(t) + master2 := startFakeMaster(t) + a := startFakeSentinel(t, master1.Addr()) + b := startFakeSentinel(t, master1.Addr()) + + sc, err := NewSentinelClient(SentinelConfig{ + MasterName: "mymaster", + SentinelAddrs: []string{a.Addr(), b.Addr()}, + DialTimeout: 2 * time.Second, + }) + if err != nil { + t.Fatalf("NewSentinelClient: %v", err) + } + t.Cleanup(func() { _ = sc.Close() }) + waitFor(t, "the client to subscribe to the first sentinel", 5*time.Second, func() bool { return a.subscribers() >= 1 }) + if b.subscribers() != 0 { + t.Fatalf("fixture: the second sentinel has %d subscribers before the first went away", b.subscribers()) + } + + loopsBefore := reconnectLoops() + + // The first sentinel goes away after the client subscribed to it. + stop := make(chan struct{}) + var stopped atomic.Bool + t.Cleanup(func() { + if stopped.CompareAndSwap(false, true) { + close(stop) + } + }) + if goneForGood { + _ = a.fake.ln.Close() + a.dropAll() + } else { + go func() { + for { + select { + case <-stop: + return + case <-time.After(flapEvery): + a.dropAll() + } + } + }() + } + + // The client must move on to the second sentinel ... + waitFor(t, "the client to subscribe to the next sentinel after the first went away", 8*time.Second, func() bool { return b.subscribers() >= 1 }) + + // The sentinel that was given up must not leave its PubSub reconnecting + // for ever, against a closed pool if it no longer reaches the sentinel: no + // reconnect goroutine is left from it ... + waitFor(t, "the abandoned sentinel's PubSub reconnect loop to exit", 5*time.Second, func() bool { return reconnectLoops() <= loopsBefore }) + // ... and a sentinel that still accepts sees no new SUBSCRIBE from this + // client once it has moved on (the backoff of the abandoned PubSub is long + // by then, so the window is 2.5 s). + if !goneForGood { + close(stop) + stopped.Store(true) + time.Sleep(50 * time.Millisecond) + before := a.subscribers() + time.Sleep(2500 * time.Millisecond) + if after := a.subscribers(); after != before { + t.Errorf("the sentinel that was given up got %d more SUBSCRIBEs after the client moved on: its PubSub is still reconnecting", after-before) + } + } + + // ... and see the failover that sentinel announces. + b.SetMaster(master2.Addr()) + waitFor(t, "the client to follow the failover announced by the next sentinel", 5*time.Second, func() bool { + if err := sc.Set(t.Context(), "after", "2", 0); err != nil { + return false + } + master2.mem.mu.Lock() + defer master2.mem.mu.Unlock() + return master2.mem.kv["after"] == "2" + }) +} + +func TestSentinelMovesOnWhenItsSentinelIsGone887(t *testing.T) { + sentinelMovesOn887(t, 0) +} + +func TestSentinelMovesOnWhenItsSentinelHangsUpAfterEveryReplay887(t *testing.T) { + sentinelMovesOn887(t, 5*time.Millisecond) +} + +// The conn lives 200 ms between hang-ups, so a check that looks at "is the +// conn up now" sees a healthy PubSub about as often as a down one. +func TestSentinelMovesOnWhenItsConnKeepsDyingAfterAWhile887(t *testing.T) { + sentinelMovesOn887(t, 200*time.Millisecond) +}