From 0e27c4175ad431ee8f58a1fcbeb3c02e9cfd9ef7 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 03:12:18 +0100 Subject: [PATCH 1/4] test(driver): probes for the redis dial refusal, sentinel move-on, pubsub backoff, portable unregister during open, wake source guard (Refs #887 #890) Tests only, plus two declared and unused knobs the tests set (sentinelDownGrace, pubsubStableAfter). They compile on main and fail there as assertions, except the wake source guard, which pins code that is already correct and is checked by mutants. --- .../portable_unregister_opening_890_test.go | 115 +++++++++++ .../eventloop/wake_source_guard_890_test.go | 184 ++++++++++++++++++ driver/memcached/dial_refused_887_test.go | 87 +++++++++ driver/postgres/dial_refused_887_test.go | 106 ++++++++++ driver/redis/dial_refused_887_test.go | 124 ++++++++++++ driver/redis/dial_reuse_887_linux_test.go | 75 +++++++ driver/redis/pubsub.go | 5 + driver/redis/pubsub_backoff_887_linux_test.go | 136 +++++++++++++ driver/redis/sentinel.go | 7 + driver/redis/sentinel_down_887_linux_test.go | 146 ++++++++++++++ 10 files changed, 985 insertions(+) create mode 100644 driver/internal/eventloop/portable_unregister_opening_890_test.go create mode 100644 driver/internal/eventloop/wake_source_guard_890_test.go create mode 100644 driver/memcached/dial_refused_887_test.go create mode 100644 driver/postgres/dial_refused_887_test.go create mode 100644 driver/redis/dial_refused_887_test.go create mode 100644 driver/redis/dial_reuse_887_linux_test.go create mode 100644 driver/redis/pubsub_backoff_887_linux_test.go create mode 100644 driver/redis/sentinel_down_887_linux_test.go diff --git a/driver/internal/eventloop/portable_unregister_opening_890_test.go b/driver/internal/eventloop/portable_unregister_opening_890_test.go new file mode 100644 index 000000000..aa6e7d472 --- /dev/null +++ b/driver/internal/eventloop/portable_unregister_opening_890_test.go @@ -0,0 +1,115 @@ +package eventloop + +// celeris#890 (found by the correctness review of PR #1051): an +// UnregisterConn(fd) that ran while RegisterConn(fd, ...) was still opening +// the net.Conn was lost. register had put a placeholder in the table; +// UnregisterConn found it, deleted it and returned nil; register's second +// section checked only the worker's closed flag, so it installed the opened +// conn and started its reader anyway: the conn stayed in the table, onClose +// never fired and the net.Conn was never closed. +// +// The two outcomes the contract allows are tested (the call is linearizable): +// either UnregisterConn ran before the registration (the number is unknown to +// it: engine.ErrUnknownFD, as on the Linux workers, whose RegisterConn also +// inserts the conn atomically) and the registration stands, fully live; or it +// ran after, returned nil and the conn is fully torn down (onClose fired, the +// net.Conn is closed, the table is clean). Never a nil return with the conn +// left registered. No build tag: the portable worker is compiled everywhere. + +import ( + "context" + "errors" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" +) + +func TestUnregisterConnWhileRegisterConnIsOpening890(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pw := newPortableWorker(ctx, 0) + w := &pw + t.Cleanup(func() { _ = w.shutdown() }) + + const fd = 4242 + ours, theirs := net.Pipe() + t.Cleanup(func() { _ = theirs.Close(); _ = ours.Close() }) + + opening := make(chan struct{}) + release := make(chan struct{}) + open := func() (net.Conn, error) { + close(opening) + <-release + return ours, nil + } + var closes atomic.Int32 + closed := make(chan error, 4) + onClose := func(err error) { + closes.Add(1) + closed <- err + } + + type regResult struct { + fd int + err error + } + regDone := make(chan regResult, 1) + go func() { + got, err := w.register(fd, false, open, func([]byte) {}, onClose) + regDone <- regResult{got, err} + }() + <-opening + + unregErr := w.UnregisterConn(fd) + close(release) + reg := <-regDone + + switch { + case unregErr == nil: + // UnregisterConn took the registration down: onClose must fire, the + // net.Conn must be closed and the table must not hold the number. + select { + case err := <-closed: + if err != nil { + t.Errorf("onClose(%v), want nil after UnregisterConn", err) + } + case <-time.After(2 * time.Second): + t.Fatalf("UnregisterConn returned nil during the open, but the conn was left registered: onClose never fired (register returned fd %d, err %v)", reg.fd, reg.err) + } + _ = theirs.SetReadDeadline(time.Now().Add(2 * time.Second)) + if _, err := theirs.Read(make([]byte, 1)); err == nil || errors.Is(err, context.DeadlineExceeded) { + t.Errorf("the net.Conn is still open after UnregisterConn returned nil: peer read err = %v", err) + } + w.mu.RLock() + _, still := w.conns[fd] + w.mu.RUnlock() + if still { + t.Errorf("the table still holds fd %d after UnregisterConn returned nil", fd) + } + case errors.Is(unregErr, engine.ErrUnknownFD): + // UnregisterConn ran before the registration existed: it did nothing, + // and the registration must be intact. + if reg.err != nil { + t.Fatalf("register failed after an UnregisterConn that reported an unknown fd: %v", reg.err) + } + if closes.Load() != 0 { + t.Fatalf("onClose fired %d times for a registration that was never unregistered", closes.Load()) + } + if err := w.UnregisterConn(fd); err != nil { + t.Fatalf("second UnregisterConn: %v", err) + } + select { + case <-closed: + case <-time.After(2 * time.Second): + t.Fatal("the registration did not tear down on its own UnregisterConn: onClose never fired") + } + default: + t.Fatalf("UnregisterConn during the open returned %v", unregErr) + } + if got := closes.Load(); got != 1 { + t.Errorf("onClose fired %d times, want 1", got) + } +} diff --git a/driver/internal/eventloop/wake_source_guard_890_test.go b/driver/internal/eventloop/wake_source_guard_890_test.go new file mode 100644 index 000000000..bdd5bfd2a --- /dev/null +++ b/driver/internal/eventloop/wake_source_guard_890_test.go @@ -0,0 +1,184 @@ +package eventloop + +// celeris#890 (the #932 wake test, WAKEATOMIC): a producer's wake must go +// through wakefd.WakeFD.Signal, whose write(2) and the eventfd's close share a +// lock. A wake that loads the eventfd's number (w.wakeFD.FD()) and writes it +// itself races Loop.Close: it can load the number before the close and write +// it after, onto whatever socket or eventfd has taken the number meanwhile. +// +// No test can order that shape. FD() returns -1 once the close has run, so a +// hook placed after the close (testHookAfterWakeClose, +// TestAWakeAfterShutdownClosedTheEventfdWritesNothing890) never sees a stale +// number; the race detector reports nothing (the number is an atomic and the +// write is a syscall); and the only oracle that can catch it, +// TestWriteRacingCloseNeverTouchesTheClosedEventfd862, needs a wake to land in +// the few microseconds between the close and the reuse of the number (0 of 10 +// processes without -race, 1 of 10 with it). The window sits between two +// statements of the mutant's own wake, where the production code has no hook +// and, by design, no such window. So the property is pinned at the source: +// this test parses the package's non-test files and requires that +// +// - every use of the worker's wakeFD field is a method call, and the method +// is the one its function is allowed (newWorker builds it, run reads the +// number on the loop thread, wake signals, shutdown closes); +// - wake's body is the single statement w.wakeFD.Signal(). +// +// It has no build tag: the files are parsed, not compiled, so it runs on every +// platform the package builds on. A mutant fails it with the position of the +// offending use, for example "(*worker).wake calls wakeFD.FD". + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +// wakeFDUses is the set of (function, method) pairs allowed to touch the +// worker's wakeFD field. "" stands for the composite-literal key. +var wakeFDUses = map[string]bool{ + "newWorker ": true, + "run FD": true, + "wake Signal": true, + "shutdown Close": true, +} + +func TestWakeGoesThroughSignalOnly890(t *testing.T) { + uses, bad, wakeBody := scanWakeFDUses(t, ".") + for _, v := range bad { + t.Error(v) + } + for _, u := range uses { + if !wakeFDUses[u.key()] { + t.Errorf("%s: %s uses wakeFD.%s; only %s are allowed (a producer's wake must be wakefd.WakeFD.Signal, celeris#890 WAKEATOMIC)", + u.pos, u.fn, u.method, allowedList()) + } + } + // A guard that scans nothing passes for the wrong reason: every allowed use + // must be there, so a rename of the field or of a function fails here + // instead of silently emptying the scan. + seen := map[string]bool{} + for _, u := range uses { + seen[u.key()] = true + } + for k := range wakeFDUses { + if !seen[k] { + t.Errorf("the scan found no %q: the worker's wakeFD usage changed shape; update this guard with it", k) + } + } + if wakeBody != "w.wakeFD.Signal()" { + t.Errorf("(*worker).wake is %q, want exactly the statement w.wakeFD.Signal()", wakeBody) + } +} + +type wakeFDUse struct { + pos token.Position + fn string + method string +} + +func (u wakeFDUse) key() string { return u.fn + " " + u.method } + +func allowedList() string { + var ks []string + for k := range wakeFDUses { + ks = append(ks, k) + } + sort.Strings(ks) + return strings.Join(ks, ", ") +} + +// scanWakeFDUses parses the non-test .go files of dir. It returns every +// wakeFD use that is a method call or a composite-literal key, the uses that +// are anything else (a bare read of the field, a copy, a method value) as +// messages, and the source of (*worker).wake's body. +func scanWakeFDUses(t *testing.T, dir string) (uses []wakeFDUse, bad []string, wakeBody string) { + t.Helper() + files, err := filepath.Glob(filepath.Join(dir, "*.go")) + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + parsed := 0 + for _, name := range files { + if strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + f, err := parser.ParseFile(fset, name, src, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + parsed++ + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + if fd.Name.Name == "wake" && fd.Recv != nil && recvName(fd) == "worker" { + wakeBody = bodySource(fset, src, fd.Body) + } + handled := map[*ast.SelectorExpr]bool{} + ast.Inspect(fd.Body, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.CallExpr: + if sel, ok := n.Fun.(*ast.SelectorExpr); ok { + if in, ok := sel.X.(*ast.SelectorExpr); ok && in.Sel.Name == "wakeFD" { + handled[in] = true + uses = append(uses, wakeFDUse{fset.Position(n.Pos()), fd.Name.Name, sel.Sel.Name}) + } + } + case *ast.KeyValueExpr: + if id, ok := n.Key.(*ast.Ident); ok && id.Name == "wakeFD" { + uses = append(uses, wakeFDUse{fset.Position(n.Pos()), fd.Name.Name, ""}) + } + } + return true + }) + ast.Inspect(fd.Body, func(n ast.Node) bool { + if sel, ok := n.(*ast.SelectorExpr); ok && sel.Sel.Name == "wakeFD" && !handled[sel] { + bad = append(bad, fmt.Sprintf("%s: %s reads the wakeFD field without calling a method on it (a copy or a method value escapes WakeFD's lock)", + fset.Position(sel.Pos()), fd.Name.Name)) + } + return true + }) + } + } + if parsed == 0 { + t.Fatalf("no non-test .go file in %s", dir) + } + return uses, bad, wakeBody +} + +func recvName(fd *ast.FuncDecl) string { + if fd.Recv == nil || len(fd.Recv.List) == 0 { + return "" + } + switch x := fd.Recv.List[0].Type.(type) { + case *ast.StarExpr: + if id, ok := x.X.(*ast.Ident); ok { + return id.Name + } + case *ast.Ident: + return x.Name + } + return "" +} + +// bodySource is the statements of a function body, without braces and +// surrounding space, comments excluded by taking the statement sources. +func bodySource(fset *token.FileSet, src []byte, b *ast.BlockStmt) string { + var parts []string + for _, s := range b.List { + parts = append(parts, string(src[fset.Position(s.Pos()).Offset:fset.Position(s.End()).Offset])) + } + return strings.Join(parts, "; ") +} diff --git a/driver/memcached/dial_refused_887_test.go b/driver/memcached/dial_refused_887_test.go new file mode 100644 index 000000000..1a2142463 --- /dev/null +++ b/driver/memcached/dial_refused_887_test.go @@ -0,0 +1,87 @@ +//go:build unix + +package memcached + +// celeris#887 item 28 (see driver/redis/dial_refused_887_test.go): the dial +// must wait out a RegisterConn refusal that the io_uring engine gives for a +// number whose previous conn is not finalized yet. + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +type refusingProvider struct { + engine.EventLoopProvider + refusals atomic.Int32 + calls atomic.Int32 + err error +} + +func (p *refusingProvider) WorkerLoop(n int) engine.WorkerLoop { + return refusingLoop{WorkerLoop: p.EventLoopProvider.WorkerLoop(n), p: p} +} + +type refusingLoop struct { + engine.WorkerLoop + p *refusingProvider +} + +func (l refusingLoop) RegisterConn(fd int, onRecv func([]byte), onClose func(error)) error { + l.p.calls.Add(1) + if r := l.p.refusals.Load(); r != 0 { + if r > 0 { + l.p.refusals.Add(-1) + } + return l.p.err + } + return l.WorkerLoop.RegisterConn(fd, onRecv, onClose) +} + +func newRefusingProvider(t *testing.T, refusals int32, err error) *refusingProvider { + t.Helper() + prov, err2 := eventloop.Resolve(nil) + if err2 != nil { + t.Fatal(err2) + } + t.Cleanup(func() { eventloop.Release(prov) }) + p := &refusingProvider{EventLoopProvider: prov, err: err} + p.refusals.Store(refusals) + return p +} + +func TestDialWaitsOutAnUnfinalizedNumber887(t *testing.T) { + p := newRefusingProvider(t, 3, errors.New("celeris/iouring: fd already registered")) + fake := startFake(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := dialMemcachedConn(ctx, p, Config{Addr: fake.Addr()}, 0) + if err != nil { + t.Fatalf("dial: %v (RegisterConn was called %d times)", err, p.calls.Load()) + } + _ = c.Close() + if got := p.calls.Load(); got != 4 { + t.Fatalf("RegisterConn calls = %d, want 4 (3 refusals, then the registration)", got) + } +} + +func TestDialDoesNotRetryOtherRegisterErrors887(t *testing.T) { + other := errors.New("celeris/iouring: fd 7 is already an HTTP connection") + p := newRefusingProvider(t, -1, other) + fake := startFake(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := dialMemcachedConn(ctx, p, Config{Addr: fake.Addr()}, 0) + if !errors.Is(err, other) { + t.Fatalf("dial error = %v, want %v", err, other) + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("RegisterConn calls = %d, want 1", got) + } +} diff --git a/driver/postgres/dial_refused_887_test.go b/driver/postgres/dial_refused_887_test.go new file mode 100644 index 000000000..a7decb72f --- /dev/null +++ b/driver/postgres/dial_refused_887_test.go @@ -0,0 +1,106 @@ +//go:build unix + +package postgres + +// celeris#887 item 28 (see driver/redis/dial_refused_887_test.go): the dial +// must wait out a RegisterConn refusal that the io_uring engine gives for a +// number whose previous conn is not finalized yet. + +import ( + "context" + "errors" + "net" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +type refusingProvider struct { + engine.EventLoopProvider + refusals atomic.Int32 + calls atomic.Int32 + err error +} + +func (p *refusingProvider) WorkerLoop(n int) engine.WorkerLoop { + return refusingLoop{WorkerLoop: p.EventLoopProvider.WorkerLoop(n), p: p} +} + +type refusingLoop struct { + engine.WorkerLoop + p *refusingProvider +} + +func (l refusingLoop) RegisterConn(fd int, onRecv func([]byte), onClose func(error)) error { + l.p.calls.Add(1) + if r := l.p.refusals.Load(); r != 0 { + if r > 0 { + l.p.refusals.Add(-1) + } + return l.p.err + } + return l.WorkerLoop.RegisterConn(fd, onRecv, onClose) +} + +// hangUpListener accepts and closes every conn at once: a dial that gets past +// RegisterConn then fails in the startup (EOF), quickly, which is not the +// failure under test. +func hangUpListener(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done := make(chan struct{}) + go func() { + defer close(done) + for { + c, err := ln.Accept() + if err != nil { + return + } + _ = c.Close() + } + }() + t.Cleanup(func() { + _ = ln.Close() + <-done + }) + return ln.Addr().String() +} + +func TestDialWaitsOutAnUnfinalizedNumber887(t *testing.T) { + prov, err := eventloop.Resolve(nil) + if err != nil { + t.Fatal(err) + } + defer eventloop.Release(prov) + p := &refusingProvider{EventLoopProvider: prov, err: errors.New("celeris/iouring: fd already registered")} + p.refusals.Store(3) + + host, port, _ := net.SplitHostPort(hangUpListener(t)) + dsn := DSN{ + Host: host, Port: port, User: "u", Database: "d", + Options: Options{SSLMode: "disable", StatementCacheSize: 16}, + Params: map[string]string{}, + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := dialConn(ctx, p, nil, dsn, 0) + if err == nil { + _ = c.Close() + t.Fatal("dial against a server that hangs up succeeded") + } + // Past the refusals the dial reaches the startup, which the server cannot + // answer; the refusal itself must not be what the caller sees. + if strings.Contains(err.Error(), "already registered") { + t.Fatalf("dial returned the RegisterConn refusal: %v (RegisterConn was called %d times)", err, p.calls.Load()) + } + if got := p.calls.Load(); got != 4 { + t.Fatalf("RegisterConn calls = %d, want 4 (3 refusals, then the registration)", got) + } +} diff --git a/driver/redis/dial_refused_887_test.go b/driver/redis/dial_refused_887_test.go new file mode 100644 index 000000000..61029f1d7 --- /dev/null +++ b/driver/redis/dial_refused_887_test.go @@ -0,0 +1,124 @@ +//go:build unix + +package redis + +// celeris#887 item 28, deterministic half: a WorkerLoop that refuses +// RegisterConn the way the io_uring engine does for a number whose previous +// conn is not finalized yet ("celeris/iouring: fd already registered", see +// engine.WorkerLoop.UnregisterConn). The real-engine half is +// TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887. + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +// refusingProvider wraps the standalone loop. Its workers refuse RegisterConn +// with err while refusals is not 0 (a negative value refuses for ever) and +// count every call. +type refusingProvider struct { + engine.EventLoopProvider + refusals atomic.Int32 + calls atomic.Int32 + err error +} + +func newRefusingProvider(t *testing.T, refusals int32, err error) *refusingProvider { + t.Helper() + prov, err2 := eventloop.Resolve(nil) + if err2 != nil { + t.Fatal(err2) + } + t.Cleanup(func() { eventloop.Release(prov) }) + p := &refusingProvider{EventLoopProvider: prov, err: err} + p.refusals.Store(refusals) + return p +} + +func (p *refusingProvider) WorkerLoop(n int) engine.WorkerLoop { + return refusingLoop{WorkerLoop: p.EventLoopProvider.WorkerLoop(n), p: p} +} + +type refusingLoop struct { + engine.WorkerLoop + p *refusingProvider +} + +func (l refusingLoop) RegisterConn(fd int, onRecv func([]byte), onClose func(error)) error { + l.p.calls.Add(1) + if r := l.p.refusals.Load(); r != 0 { + if r > 0 { + l.p.refusals.Add(-1) + } + return l.p.err + } + return l.WorkerLoop.RegisterConn(fd, onRecv, onClose) +} + +var errIOUringBusy887 = errors.New("celeris/iouring: fd already registered") + +func dialRefusedCfg(t *testing.T) Config { + t.Helper() + fake := startFakeRedis(t, nil) + return Config{Addr: fake.Addr(), ForceRESP2: true} +} + +// A refusal that clears within a few milliseconds must not fail the dial. +func TestDialWaitsOutAnUnfinalizedNumber887(t *testing.T) { + p := newRefusingProvider(t, 3, errIOUringBusy887) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := dialRedisConn(ctx, p, dialRefusedCfg(t), 0) + if err != nil { + t.Fatalf("dial: %v (RegisterConn was called %d times)", err, p.calls.Load()) + } + _ = c.Close() + if got := p.calls.Load(); got != 4 { + t.Fatalf("RegisterConn calls = %d, want 4 (3 refusals, then the registration)", got) + } +} + +// A number that stays registered must not hold the dial for ever: the dial's +// ctx bounds the wait, and the refusal is what the caller sees. +func TestDialGivesUpWhenTheNumberStaysRegistered887(t *testing.T) { + p := newRefusingProvider(t, -1, errIOUringBusy887) + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + start := time.Now() + c, err := dialRedisConn(ctx, p, dialRefusedCfg(t), 0) + took := time.Since(start) + if err == nil { + _ = c.Close() + t.Fatal("dial succeeded against a loop that always refuses") + } + if !strings.Contains(err.Error(), "already registered") { + t.Fatalf("dial error = %v, want the refusal", err) + } + if took > time.Second { + t.Fatalf("dial took %v with a 300 ms ctx: the wait ignores the ctx", took) + } + t.Logf("gave up after %v and %d calls: %v", took, p.calls.Load(), err) +} + +// Only the "number not finalized yet" refusal is retried. Any other +// RegisterConn error is final at once. +func TestDialDoesNotRetryOtherRegisterErrors887(t *testing.T) { + other := errors.New("celeris/iouring: fd 7 is already an HTTP connection") + p := newRefusingProvider(t, -1, other) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err := dialRedisConn(ctx, p, dialRefusedCfg(t), 0) + if !errors.Is(err, other) { + t.Fatalf("dial error = %v, want %v", err, other) + } + if got := p.calls.Load(); got != 1 { + t.Fatalf("RegisterConn calls = %d, want 1", got) + } +} diff --git a/driver/redis/dial_reuse_887_linux_test.go b/driver/redis/dial_reuse_887_linux_test.go new file mode 100644 index 000000000..e52cac7b4 --- /dev/null +++ b/driver/redis/dial_reuse_887_linux_test.go @@ -0,0 +1,75 @@ +//go:build linux + +package redis + +// celeris#887 item 28 (the checklist's twenty-eighth box): a driver dial fails +// with "celeris/iouring: fd already registered" when the new socket gets the +// number of a conn that was just unregistered on the same worker and whose +// onClose has not run. engine.WorkerLoop.UnregisterConn documents the refusal; +// dialRedisConn handed it to the caller, so Pool.Acquire could fail under +// connection churn on the io_uring engine. + +import ( + "bufio" + "context" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887 closes a conn and +// dials the next one at once, on each engine of the rig. The next socket takes +// the number of the conn just closed, and the io_uring engine keeps that +// number registered on its worker until the old conn is finalized. A dial that +// loses the race must wait for it (bounded) instead of returning the refusal. +func TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887(t *testing.T) { + eachRigEngine(t, func(t *testing.T, eng rigEngine) { + prov := rigProvider(t, eng) + fake := startFakeRedis(t, func(cmd []string, w *bufio.Writer) {}) + // ForceRESP2 with no password and no DB: the handshake sends nothing, + // so the fake needs no replies and the test measures the dial alone. + cfg := Config{Addr: fake.Addr(), ForceRESP2: true} + + // Four goroutines close and dial without pause: the churn makes the + // next socket take a number that was just released, on a worker that + // has not finalized the conn that held it. + const goroutines, perG = 4, 500 + var refused, other atomic.Int32 + var mu sync.Mutex + var first error + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < perG; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + c, err := dialRedisConn(ctx, prov, cfg, -1) + cancel() + if err != nil { + if strings.Contains(err.Error(), "already registered") { + refused.Add(1) + } else { + other.Add(1) + } + mu.Lock() + if first == nil { + first = err + } + mu.Unlock() + continue + } + _ = c.Close() + } + }() + } + wg.Wait() + rounds := goroutines * perG + t.Logf("engine %s: %d dials, %d refused with 'already registered', %d failed otherwise", eng, rounds, refused.Load(), other.Load()) + if first != nil { + t.Errorf("a dial right after a close failed (%d of %d refused, %d other); first error: %v", refused.Load(), rounds, other.Load(), first) + } + }) +} diff --git a/driver/redis/pubsub.go b/driver/redis/pubsub.go index a199ba163..99122d633 100644 --- a/driver/redis/pubsub.go +++ b/driver/redis/pubsub.go @@ -10,6 +10,11 @@ import ( "github.com/goceleris/celeris/driver/internal/async" ) +// pubsubStableAfter is how long a re-established pub/sub conn must stay up +// before the reconnect backoff starts over from its base delay (celeris#887 +// item 30). A conn that dies sooner is a failed attempt. +var pubsubStableAfter = 5 * time.Second + // Message is one pub/sub notification. Pattern is set only for PSUBSCRIBE // deliveries ("pmessage" push type). Shard is true for shard channel // deliveries ("smessage" push type, Redis 7+). diff --git a/driver/redis/pubsub_backoff_887_linux_test.go b/driver/redis/pubsub_backoff_887_linux_test.go new file mode 100644 index 000000000..003ac1615 --- /dev/null +++ b/driver/redis/pubsub_backoff_887_linux_test.go @@ -0,0 +1,136 @@ +//go:build linux + +package redis + +// celeris#887 item 30: the reconnect loop reset its backoff as soon as the +// replay of the subscriptions returned, so a server that hangs up just after +// the replay was retried at the base delay for ever (the loop's own in-loop +// check covers only a conn that dies during the replay, item 15). The backoff +// must start over only after the replayed conn has stayed up for a while +// (pubsubStableAfter). + +import ( + "bufio" + "context" + "sync" + "testing" + "time" +) + +// hangUpServer is a fake server that answers SUBSCRIBE and then closes the +// conn after lifetime(n), n counting the SUBSCRIBEs it has seen from 1 (0 +// keeps the conn). It records when each SUBSCRIBE arrived. +type hangUpServer struct { + fake *fakeRedis + lifetime func(n int) time.Duration + mu sync.Mutex + stamps []time.Time +} + +func startHangUpServer(t *testing.T, lifetime func(n int) time.Duration) *hangUpServer { + t.Helper() + s := &hangUpServer{lifetime: lifetime} + ready := make(chan struct{}) + s.fake = startFakeRedis(t, func(cmd []string, w *bufio.Writer) { + <-ready + if len(cmd) == 0 || cmd[0] != "SUBSCRIBE" { + writeSimple(w, "OK") + return + } + writeArrayHeader(w, 3) + writeBulk(w, "subscribe") + writeBulk(w, cmd[1]) + writeInt(w, 1) + s.mu.Lock() + s.stamps = append(s.stamps, time.Now()) + n := len(s.stamps) + s.mu.Unlock() + if d := s.lifetime(n); d > 0 { + s.fake.mu.Lock() + c := s.fake.conns[len(s.fake.conns)-1] + s.fake.mu.Unlock() + time.AfterFunc(d, func() { _ = c.Close() }) + } + }) + close(ready) + return s +} + +func (s *hangUpServer) times() []time.Time { + s.mu.Lock() + defer s.mu.Unlock() + return append([]time.Time(nil), s.stamps...) +} + +func subscribeToHangUpServer(t *testing.T, s *hangUpServer) *PubSub { + t.Helper() + cl, err := NewClient(s.fake.Addr(), WithForceRESP2()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cl.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ps, err := cl.Subscribe(ctx, "news") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ps.Close() }) + return ps +} + +// The server hangs up 10 ms after every SUBSCRIBE. The attempts must back off: +// the base delay is 50 ms and doubles up to 5 s, so in 2.5 s the server sees +// about seven SUBSCRIBEs (50+100+200+400+800+1600 ms of delay), not the fifty +// of a loop that starts over from 50 ms every time. +func TestReconnectBacksOffWhenTheServerHangsUpAfterEveryReplay887(t *testing.T) { + s := startHangUpServer(t, func(int) time.Duration { return 10 * time.Millisecond }) + subscribeToHangUpServer(t, s) + time.Sleep(2500 * time.Millisecond) + ts := s.times() + for i := 1; i < len(ts); i++ { + t.Logf("SUBSCRIBE %d came %v after the previous one", i+1, ts[i].Sub(ts[i-1]).Round(time.Millisecond)) + } + if len(ts) > 12 { + t.Errorf("the server saw %d SUBSCRIBEs in 2.5 s: the reconnect backoff started over after every replay", len(ts)) + } + if n := len(ts); n >= 5 { + if gap := ts[n-1].Sub(ts[n-2]); gap < 250*time.Millisecond { + t.Errorf("the gap before SUBSCRIBE %d is %v, want >= 250 ms: the delay did not grow", n, gap) + } + } +} + +// A conn that stays up for pubsubStableAfter ends the outage: the next drop is +// retried from the base delay again. +func TestReconnectBackoffStartsOverAfterTheConnWasStable887(t *testing.T) { + old := pubsubStableAfter + pubsubStableAfter = 400 * time.Millisecond + t.Cleanup(func() { pubsubStableAfter = old }) + + // SUBSCRIBEs 1..5 are cut after 10 ms; the sixth lives 800 ms (longer than + // the stable window), then everything is cut after 10 ms again. + s := startHangUpServer(t, func(n int) time.Duration { + if n == 6 { + return 800 * time.Millisecond + } + return 10 * time.Millisecond + }) + subscribeToHangUpServer(t, s) + waitFor(t, "the seventh SUBSCRIBE", 20*time.Second, func() bool { return len(s.times()) >= 7 }) + ts := s.times() + for i := 1; i < len(ts); i++ { + t.Logf("SUBSCRIBE %d came %v after the previous one", i+1, ts[i].Sub(ts[i-1]).Round(time.Millisecond)) + } + // Before the stable conn the attempts back off (the fifth reconnect waits + // about 800 ms) ... + if gap := ts[4].Sub(ts[3]); gap < 250*time.Millisecond { + t.Errorf("the gap before SUBSCRIBE 5 is %v, want >= 250 ms: the delay did not grow", gap) + } + // ... and after the conn that lived 800 ms the next reconnect is at the + // base delay again. + gap := ts[6].Sub(ts[5]) - 800*time.Millisecond + if gap > 250*time.Millisecond { + t.Errorf("the reconnect after a stable conn waited %v, want about the base delay (50 ms): the backoff did not start over", gap) + } +} diff --git a/driver/redis/sentinel.go b/driver/redis/sentinel.go index 0d171996e..a169471fe 100644 --- a/driver/redis/sentinel.go +++ b/driver/redis/sentinel.go @@ -37,6 +37,13 @@ type SentinelConfig struct { Engine ServerProvider } +// sentinelDownGrace is how long watchFailover lets the PubSub of the sentinel +// it follows stay down (conn lost and not re-established, or re-established +// and lost again before it was stable) before it gives that sentinel up and +// lets subscribeLoop connect to the next address of SentinelAddrs +// (celeris#887 item 29). +var sentinelDownGrace = 5 * time.Second + // SentinelClient is a Redis client managed by Redis Sentinel. It discovers the // current primary via Sentinel, connects to it, and automatically reconnects // to the new primary when a failover is detected. diff --git a/driver/redis/sentinel_down_887_linux_test.go b/driver/redis/sentinel_down_887_linux_test.go new file mode 100644 index 000000000..b8791a0d8 --- /dev/null +++ b/driver/redis/sentinel_down_887_linux_test.go @@ -0,0 +1,146 @@ +//go:build linux + +package redis + +// celeris#887 item 29: watchFailover waited on sc.subPS.Channel(), which a +// conn drop does not close: the sentinel's PubSub reconnected to the same +// address for ever. A sentinel that went away after a successful +// Subscribe("+switch-master") was never replaced by the next address of +// SentinelAddrs, and the failovers it would have announced were missed. +// +// Both shapes of "went away" are tested: the sentinel is gone for good (its +// listener closed), and the sentinel accepts every reconnect and hangs up at +// once, so its PubSub is never down for long. + +import ( + "net" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" +) + +func (fs *fakeSentinel) subscribers() int { + fs.mu.Lock() + defer fs.mu.Unlock() + return len(fs.subWriters) +} + +// dropAll closes every conn the sentinel has accepted so far. +func (fs *fakeSentinel) dropAll() { + fs.fake.mu.Lock() + conns := append([]net.Conn(nil), fs.fake.conns...) + fs.fake.conns = nil + fs.fake.mu.Unlock() + for _, c := range conns { + _ = c.Close() + } +} + +// flapEvery is the period of the hang-ups of the sentinel that stays up; 0 +// means the sentinel is gone for good. +// reconnectLoops counts the PubSub reconnect goroutines of the process. +func reconnectLoops() int { + buf := make([]byte, 1<<20) + buf = buf[:runtime.Stack(buf, true)] + return strings.Count(string(buf), "(*PubSub).reconnectLoop") +} + +func sentinelMovesOn887(t *testing.T, flapEvery time.Duration) { + goneForGood := flapEvery == 0 + oldGrace := sentinelDownGrace + sentinelDownGrace = 600 * time.Millisecond + t.Cleanup(func() { sentinelDownGrace = oldGrace }) + + master1 := startFakeMaster(t) + master2 := startFakeMaster(t) + a := startFakeSentinel(t, master1.Addr()) + b := startFakeSentinel(t, master1.Addr()) + + sc, err := NewSentinelClient(SentinelConfig{ + MasterName: "mymaster", + SentinelAddrs: []string{a.Addr(), b.Addr()}, + DialTimeout: 2 * time.Second, + }) + if err != nil { + t.Fatalf("NewSentinelClient: %v", err) + } + t.Cleanup(func() { _ = sc.Close() }) + waitFor(t, "the client to subscribe to the first sentinel", 5*time.Second, func() bool { return a.subscribers() >= 1 }) + if b.subscribers() != 0 { + t.Fatalf("fixture: the second sentinel has %d subscribers before the first went away", b.subscribers()) + } + + loopsBefore := reconnectLoops() + + // The first sentinel goes away after the client subscribed to it. + stop := make(chan struct{}) + var stopped atomic.Bool + t.Cleanup(func() { + if stopped.CompareAndSwap(false, true) { + close(stop) + } + }) + if goneForGood { + _ = a.fake.ln.Close() + a.dropAll() + } else { + go func() { + for { + select { + case <-stop: + return + case <-time.After(flapEvery): + a.dropAll() + } + } + }() + } + + // The client must move on to the second sentinel ... + waitFor(t, "the client to subscribe to the next sentinel after the first went away", 8*time.Second, func() bool { return b.subscribers() >= 1 }) + + // The sentinel that was given up must not leave its PubSub reconnecting + // for ever, against a closed pool if it no longer reaches the sentinel: no + // reconnect goroutine is left from it ... + waitFor(t, "the abandoned sentinel's PubSub reconnect loop to exit", 5*time.Second, func() bool { return reconnectLoops() <= loopsBefore }) + // ... and a sentinel that still accepts sees no new SUBSCRIBE from this + // client once it has moved on (the backoff of the abandoned PubSub is long + // by then, so the window is 2.5 s). + if !goneForGood { + close(stop) + stopped.Store(true) + time.Sleep(50 * time.Millisecond) + before := a.subscribers() + time.Sleep(2500 * time.Millisecond) + if after := a.subscribers(); after != before { + t.Errorf("the sentinel that was given up got %d more SUBSCRIBEs after the client moved on: its PubSub is still reconnecting", after-before) + } + } + + // ... and see the failover that sentinel announces. + b.SetMaster(master2.Addr()) + waitFor(t, "the client to follow the failover announced by the next sentinel", 5*time.Second, func() bool { + if err := sc.Set(t.Context(), "after", "2", 0); err != nil { + return false + } + master2.mem.mu.Lock() + defer master2.mem.mu.Unlock() + return master2.mem.kv["after"] == "2" + }) +} + +func TestSentinelMovesOnWhenItsSentinelIsGone887(t *testing.T) { + sentinelMovesOn887(t, 0) +} + +func TestSentinelMovesOnWhenItsSentinelHangsUpAfterEveryReplay887(t *testing.T) { + sentinelMovesOn887(t, 5*time.Millisecond) +} + +// The conn lives 200 ms between hang-ups, so a check that looks at "is the +// conn up now" sees a healthy PubSub about as often as a down one. +func TestSentinelMovesOnWhenItsConnKeepsDyingAfterAWhile887(t *testing.T) { + sentinelMovesOn887(t, 200*time.Millisecond) +} From 502a4f4a57d9fdcde870ee6d3ac544104f1c1523 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 03:17:39 +0100 Subject: [PATCH 2/4] driver/redis: dial waits out the io_uring number refusal, sentinel moves on from a lost sentinel, pubsub backoff survives a hang-up after the replay; eventloop: portable UnregisterConn ignores a registration still opening, wake source guard wording (Refs #887 #890) 887 item 28: eventloop.RegisterConnWait retries RegisterConn on the io_uring engine's 'fd already registered' (bounded 2 s and by ctx); the redis, memcached and postgres dials use it. The retry in the PubSub test rig that hid the failure is removed. 887 item 29: watchFailover gives up a sentinel whose PubSub has been down (or flapping) for sentinelDownGrace; subscribeLoop moves to the next address and closes the replaced sentinel's PubSub. 887 item 30: the reconnect backoff runs over the whole outage and starts over only when a replayed conn has stayed up for pubsubStableAfter. 890 item 89: portable UnregisterConn treats register's placeholder as an unknown number. 890 item 50: comments that said the WAKEATOMIC shape is covered only by the probabilistic oracle now point at the source guard. --- driver/internal/eventloop/portable.go | 20 ++-- driver/internal/eventloop/register_wait.go | 72 ++++++++++++++ .../eventloop/register_wait_887_test.go | 94 +++++++++++++++++++ .../teardown_hooks_890_linux_test.go | 6 +- .../eventloop/wake_close_862_linux_test.go | 3 +- driver/memcached/conn_unix.go | 3 +- driver/postgres/conn_unix.go | 3 +- driver/redis/conn_unix.go | 3 +- driver/redis/dial_reuse_887_linux_test.go | 39 ++++++++ driver/redis/pubsub.go | 70 +++++++++++++- .../redis/pubsub_ctlwrite_929_linux_test.go | 19 +--- driver/redis/sentinel.go | 81 +++++++++++++--- 12 files changed, 370 insertions(+), 43 deletions(-) create mode 100644 driver/internal/eventloop/register_wait.go create mode 100644 driver/internal/eventloop/register_wait_887_test.go diff --git a/driver/internal/eventloop/portable.go b/driver/internal/eventloop/portable.go index 313100494..3556c5c39 100644 --- a/driver/internal/eventloop/portable.go +++ b/driver/internal/eventloop/portable.go @@ -293,6 +293,15 @@ func (w *portableWorker) readLoop(c *portableConn) { func (w *portableWorker) UnregisterConn(fd int) error { w.mu.Lock() c, ok := w.conns[fd] + if ok && c.conn == nil { + // register's placeholder: a RegisterConn of this number is still + // opening the net.Conn. The number is not registered yet, so it is + // unknown to this call, as on the Linux workers, whose RegisterConn + // inserts the conn in one step. Leave the placeholder: register + // installs the conn over it, and an UnregisterConn issued after + // RegisterConn returned finds that (celeris#890). + ok = false + } if ok { delete(w.conns, fd) } @@ -389,14 +398,11 @@ func (w *portableWorker) Write(fd int, data []byte) error { // still delivering onRecv. No lock is held across the close or the callback. // The placeholder register puts in the table while it opens the net.Conn // (conn == nil) has no onClose and no reader, so for it this only marks it -// closing: there is nothing to close and nothing to report. shutdown and -// UnregisterConn both reach it. After shutdown's table swap register sees the +// closing: there is nothing to close and nothing to report. Only shutdown +// reaches it: UnregisterConn treats a placeholder as an unknown number and +// leaves it (celeris#890). After shutdown's table swap register sees the // worker closed and drops the conn it opened; a failed open drops the -// placeholder itself. An UnregisterConn that finds the placeholder deletes it -// and returns nil, but register's second section checks only the worker's -// closed flag, not that the placeholder is still in the table, so it installs -// the opened conn and starts its reader anyway: that UnregisterConn is lost -// (celeris#890). +// placeholder itself. func (c *portableConn) shutdownConn(err error) { c.mu.Lock() if c.closing { diff --git a/driver/internal/eventloop/register_wait.go b/driver/internal/eventloop/register_wait.go new file mode 100644 index 000000000..54e5339fb --- /dev/null +++ b/driver/internal/eventloop/register_wait.go @@ -0,0 +1,72 @@ +package eventloop + +import ( + "context" + "strings" + "time" + + "github.com/goceleris/celeris/internal/engine" +) + +// registerWaitBudget bounds how long RegisterConnWait waits for a refused +// number to be finalized. A conn is finalized when the engine has cancelled its +// operations and closed its own duplicate of the descriptor, normally within +// microseconds of UnregisterConn; a close that lingers (SO_LINGER) is closed +// off the worker and can take longer (celeris#735), so the wait is bounded +// instead of unlimited. +const registerWaitBudget = 2 * time.Second + +// IsNumberBusy reports whether err is the io_uring engine's refusal of a +// descriptor number whose previous conn has been unregistered but not yet +// finalized: "celeris/iouring: fd already registered". +// [engine.WorkerLoop.UnregisterConn] documents it: the next socket usually +// gets the lowest free number, which is the number of the conn just closed, +// and the engine keeps that number registered until the old conn is finalized. +// It clears by itself, unlike any other RegisterConn error (the number is +// held by an HTTP conn, the worker has shut down, a duplicate registration on +// the epoll engine, whose onClose has run before UnregisterConn returns). +// +// The engine has no sentinel for it, so the text is matched; the tests of +// the drivers register the same number twice on a real io_uring worker to pin +// it. +func IsNumberBusy(err error) bool { + if err == nil { + return false + } + msg := err.Error() + return strings.HasPrefix(msg, "celeris/iouring:") && strings.HasSuffix(msg, "fd already registered") +} + +// RegisterConnWait is loop.RegisterConn that waits out the io_uring engine's +// "fd already registered" refusal (see [IsNumberBusy]) instead of returning it: +// it retries with a growing pause until the number is free, ctx is done, or +// registerWaitBudget has passed, and then returns the refusal. Every other +// error is returned at once, and a failed call leaves fd with the caller, as +// RegisterConn does. The drivers' dials use it: without it Pool.Acquire fails +// under connection churn on the io_uring engine (celeris#887 item 28). +func RegisterConnWait(ctx context.Context, loop engine.WorkerLoop, fd int, onRecv func([]byte), onClose func(error)) error { + err := loop.RegisterConn(fd, onRecv, onClose) + if !IsNumberBusy(err) { + return err + } + deadline := time.Now().Add(registerWaitBudget) + pause := 100 * time.Microsecond + for { + t := time.NewTimer(pause) + select { + case <-ctx.Done(): + t.Stop() + return err + case <-t.C: + } + if err = loop.RegisterConn(fd, onRecv, onClose); !IsNumberBusy(err) { + return err + } + if !time.Now().Before(deadline) { + return err + } + if pause < 5*time.Millisecond { + pause *= 2 + } + } +} diff --git a/driver/internal/eventloop/register_wait_887_test.go b/driver/internal/eventloop/register_wait_887_test.go new file mode 100644 index 000000000..f7356b38a --- /dev/null +++ b/driver/internal/eventloop/register_wait_887_test.go @@ -0,0 +1,94 @@ +package eventloop + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/goceleris/celeris/internal/engine" +) + +type scriptedLoop struct { + engine.WorkerLoop + errs []error // returned in order; the last repeats + calls int +} + +func (l *scriptedLoop) RegisterConn(int, func([]byte), func(error)) error { + i := l.calls + l.calls++ + if i >= len(l.errs) { + i = len(l.errs) - 1 + } + return l.errs[i] +} + +var errBusy = errors.New("celeris/iouring: fd already registered") + +func TestIsNumberBusy887(t *testing.T) { + for _, c := range []struct { + err error + want bool + }{ + {nil, false}, + {errBusy, true}, + {errors.New("celeris/iouring: fd 7 is already an HTTP connection"), false}, + {errors.New("celeris/epoll: fd 7 is already registered"), false}, + {ErrAlreadyRegistered, false}, // a real duplicate of the standalone loop: it does not clear + {fmt.Errorf("celeris/iouring: worker 0 has shut down, cannot register fd 7: %w", errors.New("engine shut down")), false}, + } { + if got := IsNumberBusy(c.err); got != c.want { + t.Errorf("IsNumberBusy(%v) = %v, want %v", c.err, got, c.want) + } + } +} + +func TestRegisterConnWait887(t *testing.T) { + other := errors.New("boom") + for _, c := range []struct { + name string + errs []error + cancelled bool + wantErr error + wantCalls int + }{ + {"free at once", []error{nil}, false, nil, 1}, + {"clears after three refusals", []error{errBusy, errBusy, errBusy, nil}, false, nil, 4}, + {"other error is final", []error{other}, false, other, 1}, + {"other error after a refusal is final", []error{errBusy, other}, false, other, 2}, + {"ctx done returns the refusal", []error{errBusy}, true, errBusy, 1}, + } { + t.Run(c.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + if c.cancelled { + cancel() + } + l := &scriptedLoop{errs: c.errs} + err := RegisterConnWait(ctx, l, 3, nil, nil) + if !errors.Is(err, c.wantErr) { + t.Fatalf("err = %v, want %v", err, c.wantErr) + } + if l.calls != c.wantCalls { + t.Fatalf("RegisterConn calls = %d, want %d", l.calls, c.wantCalls) + } + }) + } +} + +// A refusal that never clears is bounded by registerWaitBudget, not by ctx. +func TestRegisterConnWaitIsBoundedWithoutACtxDeadline887(t *testing.T) { + l := &scriptedLoop{errs: []error{errBusy}} + start := time.Now() + err := RegisterConnWait(context.Background(), l, 3, nil, nil) + took := time.Since(start) + if !errors.Is(err, errBusy) { + t.Fatalf("err = %v", err) + } + if took < registerWaitBudget || took > registerWaitBudget+time.Second { + t.Fatalf("gave up after %v, want about %v", took, registerWaitBudget) + } + t.Logf("gave up after %v and %d calls", took, l.calls) +} diff --git a/driver/internal/eventloop/teardown_hooks_890_linux_test.go b/driver/internal/eventloop/teardown_hooks_890_linux_test.go index 061a64876..11d6dc894 100644 --- a/driver/internal/eventloop/teardown_hooks_890_linux_test.go +++ b/driver/internal/eventloop/teardown_hooks_890_linux_test.go @@ -209,8 +209,10 @@ func TestShutdownMarksEveryConnClosedBeforeItClosesTheEpollFd890(t *testing.T) { // the loaded number after it (mutant WAKEATOMIC): FD() already returns -1 once // the close has run, so no hook placed after the close can order that shape, // and the race detector reports nothing (the number is an atomic, the write is -// a syscall). Only the probabilistic oracle of -// TestWriteRacingCloseNeverTouchesTheClosedEventfd862 covers it (celeris#890). +// a syscall). That shape is pinned at the source by +// TestWakeGoesThroughSignalOnly890 (wake_source_guard_890_test.go); the +// probabilistic oracle of TestWriteRacingCloseNeverTouchesTheClosedEventfd862 +// only backs it (celeris#890). func TestAWakeAfterShutdownClosedTheEventfdWritesNothing890(t *testing.T) { t.Cleanup(func() { testHookAfterWakeClose = nil }) // runs last l, err := New(1) diff --git a/driver/internal/eventloop/wake_close_862_linux_test.go b/driver/internal/eventloop/wake_close_862_linux_test.go index 3c5fdcaf1..a6166d969 100644 --- a/driver/internal/eventloop/wake_close_862_linux_test.go +++ b/driver/internal/eventloop/wake_close_862_linux_test.go @@ -65,7 +65,8 @@ func c862FillSendBuffer(t *testing.T, fd int) int { // pinned deterministically by // TestAWakeAfterShutdownClosedTheEventfdWritesNothing890; a wake that loads // w.wakeFD.FD() itself, races the close and writes to the loaded number is -// covered only by the oracle above (celeris#890). +// pinned at the source by TestWakeGoesThroughSignalOnly890; the oracle above +// only backs it (celeris#890). func TestWriteRacingCloseNeverTouchesTheClosedEventfd862(t *testing.T) { const rounds, writers = 64, 4 one := []byte{'w'} diff --git a/driver/memcached/conn_unix.go b/driver/memcached/conn_unix.go index b09748534..0c02a0e8c 100644 --- a/driver/memcached/conn_unix.go +++ b/driver/memcached/conn_unix.go @@ -10,6 +10,7 @@ import ( "syscall" "time" + "github.com/goceleris/celeris/driver/internal/eventloop" "github.com/goceleris/celeris/internal/engine" ) @@ -90,7 +91,7 @@ func dialMemcachedConn(ctx context.Context, prov engine.EventLoopProvider, cfg C c.pollIsDoneFn = c.pollIsDone c.lastUsedAt.Store(time.Now().UnixNano()) - if err := loop.RegisterConn(fd, c.onRecvFn, c.onCloseFn); err != nil { + if err := eventloop.RegisterConnWait(ctx, loop, fd, c.onRecvFn, c.onCloseFn); err != nil { _ = file.Close() return nil, err } diff --git a/driver/postgres/conn_unix.go b/driver/postgres/conn_unix.go index 5fed66826..4ae9d14a9 100644 --- a/driver/postgres/conn_unix.go +++ b/driver/postgres/conn_unix.go @@ -11,6 +11,7 @@ import ( "time" "github.com/goceleris/celeris/driver/internal/async" + "github.com/goceleris/celeris/driver/internal/eventloop" "github.com/goceleris/celeris/internal/driver/postgres/protocol" "github.com/goceleris/celeris/internal/engine" ) @@ -116,7 +117,7 @@ func dialConn(ctx context.Context, prov engine.EventLoopProvider, closeLoop func c.onCloseFn = c.onClose c.lastUsedAt.Store(time.Now().UnixNano()) - if err := loop.RegisterConn(fd, c.onRecvFn, c.onCloseFn); err != nil { + if err := eventloop.RegisterConnWait(ctx, loop, fd, c.onRecvFn, c.onCloseFn); err != nil { // Close via file.Close() to disarm the *os.File finalizer. A stray // syscall.Close(fd) here would leave the finalizer armed; when GC // later fires it would close the same fd a SECOND time, possibly diff --git a/driver/redis/conn_unix.go b/driver/redis/conn_unix.go index b341e2c51..5d464474f 100644 --- a/driver/redis/conn_unix.go +++ b/driver/redis/conn_unix.go @@ -10,6 +10,7 @@ import ( "syscall" "time" + "github.com/goceleris/celeris/driver/internal/eventloop" "github.com/goceleris/celeris/internal/engine" ) @@ -92,7 +93,7 @@ func dialRedisConn(ctx context.Context, prov engine.EventLoopProvider, cfg Confi c.pollBeforeRearmFn = c.pollBeforeRearm c.lastUsedAt.Store(time.Now().UnixNano()) - if err := loop.RegisterConn(fd, c.onRecvFn, c.onCloseFn); err != nil { + if err := eventloop.RegisterConnWait(ctx, loop, fd, c.onRecvFn, c.onCloseFn); err != nil { // Close via file.Close() to disarm the *os.File finalizer; a plain // syscall.Close(fd) would leak the finalizer and risk a double // close on a reassigned fd when GC later runs it. diff --git a/driver/redis/dial_reuse_887_linux_test.go b/driver/redis/dial_reuse_887_linux_test.go index e52cac7b4..f0a3897e1 100644 --- a/driver/redis/dial_reuse_887_linux_test.go +++ b/driver/redis/dial_reuse_887_linux_test.go @@ -12,11 +12,16 @@ package redis import ( "bufio" "context" + "errors" "strings" "sync" "sync/atomic" "testing" "time" + + "golang.org/x/sys/unix" + + "github.com/goceleris/celeris/driver/internal/eventloop" ) // TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887 closes a conn and @@ -73,3 +78,37 @@ func TestDialAfterTheCloseOfAConnOnTheSameNumberSucceeds887(t *testing.T) { } }) } + +// TestTheIOUringRefusalIsWhatTheDialWaitsOut887 pins the match of +// eventloop.IsNumberBusy to the text a real io_uring worker returns: the +// engine has no sentinel for it, so a reworded message would silently turn the +// dial's wait off. +func TestTheIOUringRefusalIsWhatTheDialWaitsOut887(t *testing.T) { + prov := rigProvider(t, rigIOUring) + wl := prov.WorkerLoop(0) + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_NONBLOCK|unix.SOCK_CLOEXEC, 0) + if err != nil { + t.Fatal(err) + } + a, b := fds[0], fds[1] + t.Cleanup(func() { _ = unix.Close(b) }) + if err := wl.RegisterConn(a, func([]byte) {}, func(error) {}); err != nil { + _ = unix.Close(a) + t.Fatalf("first RegisterConn: %v", err) + } + t.Cleanup(func() { + _ = wl.UnregisterConn(a) + _ = unix.Close(a) + }) + err = wl.RegisterConn(a, func([]byte) {}, func(error) {}) + if err == nil { + t.Fatal("registering the same number twice succeeded") + } + if !eventloop.IsNumberBusy(err) { + t.Fatalf("IsNumberBusy does not recognise the io_uring engine's refusal: %q", err) + } + other := "celeris/iouring: fd 7 is already an HTTP connection" + if eventloop.IsNumberBusy(errors.New(other)) { + t.Fatalf("IsNumberBusy accepts %q", other) + } +} diff --git a/driver/redis/pubsub.go b/driver/redis/pubsub.go index 99122d633..3c881c1bd 100644 --- a/driver/redis/pubsub.go +++ b/driver/redis/pubsub.go @@ -96,6 +96,20 @@ type PubSub struct { // conn teardown. reconnectingMu sync.Mutex reconnecting bool + // The outage state below is guarded by reconnectingMu (celeris#887 items + // 29 and 30). An outage starts at the first drop of a conn and ends when a + // replayed conn has stayed up for pubsubStableAfter. The backoff runs over + // the whole outage: a conn that dies soon after its replay is one more + // failed attempt, not a fresh start. + // + // downSince is the start of the current outage, zero when there is none. + // upSince is when the last replay succeeded. attempt is the backoff + // attempt the next reconnect loop of this outage starts at. + downSince time.Time + upSince time.Time + attempt int + // stableAfter is pubsubStableAfter as of the PubSub's creation. + stableAfter time.Duration closeMsgOnce sync.Once @@ -180,6 +194,9 @@ func (c *Client) newPubSub(ctx context.Context) (*PubSub, error) { msgCh: make(chan *Message, defaultPubSubChanBuf), closeCh: make(chan struct{}), backoff: async.NewBackoff(50*time.Millisecond, 5*time.Second), + // Read once: the goroutines of the PubSub never touch the package + // variable, which tests set. + stableAfter: pubsubStableAfter, } ps.bindConn(conn) // The conn may have closed while the pool was handing it out, before the @@ -461,10 +478,46 @@ func (ps *PubSub) onConnDrop(_ error) { return } ps.reconnecting = true + if now := time.Now(); ps.downSince.IsZero() || (!ps.upSince.IsZero() && now.Sub(ps.upSince) >= ps.stableAfter) { + // A new outage: this is the first drop, or the conn that replaced the + // last one stayed up long enough to count as recovered. + ps.downSince = now + ps.attempt = 0 + ps.backoff.Reset() + } ps.reconnectingMu.Unlock() go ps.reconnectLoop() } +// downFor reports how long the PubSub has been without a conn it can rely on: +// zero when there is none of that, as for a PubSub whose conn is up and has +// been since its replay stayed up for pubsubStableAfter. During an outage it +// is the time since the outage began while no conn is up or a replay is in +// progress; once a conn is up again it is the time that outage lasted until +// that conn came up, so a conn that keeps dying soon after its replay shows a +// growing figure, and a blip that healed does not (celeris#887 item 29; +// SentinelClient reads it to give up a sentinel that went away). +func (ps *PubSub) downFor() time.Duration { + ps.mu.Lock() + cur := ps.conn + ps.mu.Unlock() + up := cur != nil && !cur.closed.Load() + now := time.Now() + ps.reconnectingMu.Lock() + defer ps.reconnectingMu.Unlock() + if ps.downSince.IsZero() { + return 0 + } + if up && !ps.reconnecting && !ps.upSince.IsZero() { + if now.Sub(ps.upSince) >= ps.stableAfter { + ps.downSince = time.Time{} // recovered: the next drop starts a new outage + return 0 + } + return ps.upSince.Sub(ps.downSince) + } + return now.Sub(ps.downSince) +} + // reconnectLoop re-dials the pubsub conn and replays subscriptions with // exponential backoff. It terminates when the pubsub is closed or the backoff // reaches its cap and a configured retry budget is exhausted (here we keep @@ -486,11 +539,19 @@ func (ps *PubSub) reconnectLoop() { ps.onConnDrop(nil) } }() - for attempt := 0; ; attempt++ { + ps.reconnectingMu.Lock() + attempt := ps.attempt + ps.reconnectingMu.Unlock() + for ; ; attempt++ { if ps.closed.Load() { return } delay := ps.backoff.Next(attempt) + // The next loop of this outage (a conn that dies right after its + // replay starts one) continues from the next attempt. + ps.reconnectingMu.Lock() + ps.attempt = attempt + 1 + ps.reconnectingMu.Unlock() select { case <-time.After(delay): case <-ps.closeCh: @@ -584,7 +645,12 @@ func (ps *PubSub) reconnectLoop() { ps.client.pool.releasePubSub(failed) } if ok { - ps.backoff.Reset() + // No Reset: the backoff starts over when a drop finds this conn + // has stayed up for pubsubStableAfter (onConnDrop), not when the + // replay returns (celeris#887 item 30). + ps.reconnectingMu.Lock() + ps.upSince = time.Now() + ps.reconnectingMu.Unlock() return } } diff --git a/driver/redis/pubsub_ctlwrite_929_linux_test.go b/driver/redis/pubsub_ctlwrite_929_linux_test.go index 01ec0c7a3..63c5f9d0d 100644 --- a/driver/redis/pubsub_ctlwrite_929_linux_test.go +++ b/driver/redis/pubsub_ctlwrite_929_linux_test.go @@ -268,22 +268,11 @@ func newPubsubRig(t *testing.T, eng rigEngine) *pubsubRig { // registered on its worker until the old conn's onClose has run, so its // RegisterConn refuses the new conn with "fd already registered" until then // (documented on engine.WorkerLoop.UnregisterConn; with one worker, as on a -// runner whose memlock funds one, every number lands on the same worker). -// The dial is retried for a while on exactly that error (celeris#887 item 10). -// -// The retry hides a real dial failure: Pool.Acquire can fail the same way for a -// caller under connection churn. It is tracked as the item "drivers, io_uring -// dial: ... fd already registered" of the celeris#887 checklist. Remove this -// retry when that dial is fixed, so the test sees the failure again. +// runner whose memlock funds one, every number lands on the same worker). The +// dial waits that out itself (eventloop.RegisterConnWait, celeris#887 item +// 28); this used to retry the dial here, which hid the failure from the test. func (r *pubsubRig) newPubSubAfterReuse(ctx context.Context) (*PubSub, error) { - deadline := time.Now().Add(5 * time.Second) - for { - ps, err := r.cl.newPubSub(ctx) - if err == nil || !strings.Contains(err.Error(), "already registered") || time.Now().After(deadline) { - return ps, err - } - time.Sleep(5 * time.Millisecond) - } + return r.cl.newPubSub(ctx) } func (r *pubsubRig) dropServerSide() { diff --git a/driver/redis/sentinel.go b/driver/redis/sentinel.go index a169471fe..bd027acfb 100644 --- a/driver/redis/sentinel.go +++ b/driver/redis/sentinel.go @@ -38,10 +38,10 @@ type SentinelConfig struct { } // sentinelDownGrace is how long watchFailover lets the PubSub of the sentinel -// it follows stay down (conn lost and not re-established, or re-established -// and lost again before it was stable) before it gives that sentinel up and -// lets subscribeLoop connect to the next address of SentinelAddrs -// (celeris#887 item 29). +// it follows stay down (see PubSub.downFor) before it gives that sentinel up +// and lets subscribeLoop connect to the next address of SentinelAddrs +// (celeris#887 item 29). A sentinel that restarts within it keeps the +// subscription. var sentinelDownGrace = 5 * time.Second // SentinelClient is a Redis client managed by Redis Sentinel. It discovers the @@ -57,6 +57,10 @@ type SentinelClient struct { closeCh chan struct{} closed atomic.Bool + + // downGrace is sentinelDownGrace as of construction; zero (a client built + // without NewSentinelClient) means the default. + downGrace time.Duration } type sentinelConn struct { @@ -80,8 +84,9 @@ func NewSentinelClient(cfg SentinelConfig) (*SentinelClient, error) { cfg.DialTimeout = defaultDialTimeout } s := &SentinelClient{ - cfg: cfg, - closeCh: make(chan struct{}), + cfg: cfg, + closeCh: make(chan struct{}), + downGrace: sentinelDownGrace, } masterAddr, err := s.discoverMaster() if err != nil { @@ -242,11 +247,16 @@ func (s *SentinelClient) dialMaster(addr string) (*Client, error) { func (s *SentinelClient) subscribeLoop() { bo := async.NewBackoff(100*time.Millisecond, 10*time.Second) attempt := 0 + // start is the index of the SentinelAddrs entry connectSentinel tries + // first: 0, and after a sentinel was given up the one after it, so that a + // sentinel that accepts and hangs up does not take the subscription back + // from the next address for ever (celeris#887 item 29). + start := 0 for { if s.closed.Load() { return } - sc, err := s.connectSentinel() + sc, err := s.connectSentinelFrom(start) if err != nil { delay := bo.Next(attempt) attempt++ @@ -271,20 +281,45 @@ func (s *SentinelClient) subscribeLoop() { // reconnect iteration). s.sentinelMu.Lock() for _, old := range s.sentinelConns { + // The PubSub first: its conn is checked out of the client's pool, + // so closing the client does not stop its reconnect loop. + if old != nil && old.subPS != nil { + _ = old.subPS.Close() + } if old != nil && old.client != nil { _ = old.client.Close() } } s.sentinelConns = append(s.sentinelConns[:0], sc) s.sentinelMu.Unlock() - s.watchFailover(sc) + if s.watchFailover(sc) { + start = s.sentinelIndex(sc.addr) + 1 + } + } +} + +// sentinelIndex is the index of addr in SentinelAddrs, or -1. +func (s *SentinelClient) sentinelIndex(addr string) int { + for i, a := range s.cfg.SentinelAddrs { + if a == addr { + return i + } } + return -1 } // connectSentinel connects to the first responsive sentinel and subscribes to // the +switch-master channel. func (s *SentinelClient) connectSentinel() (*sentinelConn, error) { - for _, addr := range s.cfg.SentinelAddrs { + return s.connectSentinelFrom(0) +} + +// connectSentinelFrom is connectSentinel trying the addresses of +// SentinelAddrs in order from index start, wrapping around once. +func (s *SentinelClient) connectSentinelFrom(start int) (*sentinelConn, error) { + n := len(s.cfg.SentinelAddrs) + for k := 0; k < n; k++ { + addr := s.cfg.SentinelAddrs[((start+k)%n+n)%n] var opts []Option opts = append(opts, WithDialTimeout(s.cfg.DialTimeout)) if s.cfg.SentinelPassword != "" { @@ -324,22 +359,42 @@ func (s *SentinelClient) connectSentinel() (*sentinelConn, error) { return nil, errors.New("no responsive sentinel") } -// watchFailover reads +switch-master messages and triggers a master swap. -func (s *SentinelClient) watchFailover(sc *sentinelConn) { +// watchFailover reads +switch-master messages and triggers a master swap. It +// returns true when it gave the sentinel up: the sentinel's PubSub reconnects +// to the same address for ever, so a sentinel that went away would otherwise +// keep the subscription and the failovers of the other sentinels would never +// be heard. The sentinel is given up once its PubSub has been without a +// reliable conn for sentinelDownGrace (PubSub.downFor: gone, or accepting and +// hanging up again and again); the PubSub and the client are closed, and +// subscribeLoop moves on to the next address of SentinelAddrs. It returns +// false when the SentinelClient is closed. +func (s *SentinelClient) watchFailover(sc *sentinelConn) (gaveUp bool) { ch := sc.subPS.Channel() + grace := s.downGrace + if grace <= 0 { + grace = 5 * time.Second + } + tick := time.NewTicker(max(grace/4, 5*time.Millisecond)) + defer tick.Stop() for { select { case msg, ok := <-ch: if !ok { - return + return !s.closed.Load() } if msg == nil { continue } // +switch-master payload: "mymaster old-ip old-port new-ip new-port" s.handleSwitchMaster(msg.Payload) + case <-tick.C: + if sc.subPS.downFor() >= grace { + _ = sc.subPS.Close() + _ = sc.client.Close() + return true + } case <-s.closeCh: - return + return false } } } From 51522375ff495e45170845e8d9808e2bc0f63d9f Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 04:34:37 +0100 Subject: [PATCH 3/4] test(driver): probes for the review of the sentinel give-up, the wake guard bypass, the wait's error (Refs #887 #890) Tests only, plus one declared and unused error (errRegisterWaitBudget) the test checks. Client.Close from two goroutines and SentinelClient.Close against the give-up race on the shared standalone loop; Close during subscribe leaves a sentinel conn open; the wake guard did not see a copy of the eventfd number that enqueueFlush writes; RegisterConnWait's error did not wrap the ctx or budget cause. --- driver/internal/eventloop/register_wait.go | 6 + .../eventloop/register_wait_887_test.go | 6 + .../eventloop/wake_source_guard_890_test.go | 135 +++++++++++++++++- .../sentinel_close_race_887_linux_test.go | 132 +++++++++++++++++ 4 files changed, 278 insertions(+), 1 deletion(-) create mode 100644 driver/redis/sentinel_close_race_887_linux_test.go diff --git a/driver/internal/eventloop/register_wait.go b/driver/internal/eventloop/register_wait.go index 54e5339fb..cab6e8231 100644 --- a/driver/internal/eventloop/register_wait.go +++ b/driver/internal/eventloop/register_wait.go @@ -2,6 +2,7 @@ package eventloop import ( "context" + "errors" "strings" "time" @@ -16,6 +17,11 @@ import ( // instead of unlimited. const registerWaitBudget = 2 * time.Second +// errRegisterWaitBudget is wrapped into the error RegisterConnWait returns +// when registerWaitBudget has passed (declared by the tests commit, used by the +// next). +var errRegisterWaitBudget = errors.New("celeris/eventloop: waited for the number to be finalized, budget used up") + // IsNumberBusy reports whether err is the io_uring engine's refusal of a // descriptor number whose previous conn has been unregistered but not yet // finalized: "celeris/iouring: fd already registered". diff --git a/driver/internal/eventloop/register_wait_887_test.go b/driver/internal/eventloop/register_wait_887_test.go index f7356b38a..26665addc 100644 --- a/driver/internal/eventloop/register_wait_887_test.go +++ b/driver/internal/eventloop/register_wait_887_test.go @@ -71,6 +71,9 @@ func TestRegisterConnWait887(t *testing.T) { if !errors.Is(err, c.wantErr) { t.Fatalf("err = %v, want %v", err, c.wantErr) } + if c.cancelled && !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v does not wrap the ctx error: a caller cannot tell a cancelled wait from a refusal", err) + } if l.calls != c.wantCalls { t.Fatalf("RegisterConn calls = %d, want %d", l.calls, c.wantCalls) } @@ -87,6 +90,9 @@ func TestRegisterConnWaitIsBoundedWithoutACtxDeadline887(t *testing.T) { if !errors.Is(err, errBusy) { t.Fatalf("err = %v", err) } + if !errors.Is(err, errRegisterWaitBudget) { + t.Fatalf("err = %v does not say the wait ran out of budget", err) + } if took < registerWaitBudget || took > registerWaitBudget+time.Second { t.Fatalf("gave up after %v, want about %v", took, registerWaitBudget) } diff --git a/driver/internal/eventloop/wake_source_guard_890_test.go b/driver/internal/eventloop/wake_source_guard_890_test.go index bdd5bfd2a..d663218e3 100644 --- a/driver/internal/eventloop/wake_source_guard_890_test.go +++ b/driver/internal/eventloop/wake_source_guard_890_test.go @@ -21,7 +21,18 @@ package eventloop // - every use of the worker's wakeFD field is a method call, and the method // is the one its function is allowed (newWorker builds it, run reads the // number on the loop thread, wake signals, shutdown closes); -// - wake's body is the single statement w.wakeFD.Signal(). +// - wake's body is the single statement w.wakeFD.Signal(); +// - the eventfd number newWorker creates reaches only wakefd.New, +// unix.EpollCtl and unix.Close: it is never stored in another field, where +// a producer could write it itself (review round 1: a copy kept in the +// worker and written by enqueueFlush has WAKEATOMIC's defect and left +// wakeFD untouched); +// - the only raw write(2) of the package is flushLocked's, to a conn's fd. +// +// Known limit: this is a syntactic pin of the shapes above, not a proof. A +// number reaching a producer by a route outside them (a global, a closure over +// an integer returned by a helper) is not seen; the probabilistic oracle of +// TestWriteRacingCloseNeverTouchesTheClosedEventfd862 is the backstop. // // It has no build tag: the files are parsed, not compiled, so it runs on every // platform the package builds on. A mutant fails it with the position of the @@ -71,6 +82,9 @@ func TestWakeGoesThroughSignalOnly890(t *testing.T) { t.Errorf("the scan found no %q: the worker's wakeFD usage changed shape; update this guard with it", k) } } + for _, v := range scanEventfdNumber(t, ".") { + t.Error(v) + } if wakeBody != "w.wakeFD.Signal()" { t.Errorf("(*worker).wake is %q, want exactly the statement w.wakeFD.Signal()", wakeBody) } @@ -182,3 +196,122 @@ func bodySource(fset *token.FileSet, src []byte, b *ast.BlockStmt) string { } return strings.Join(parts, "; ") } + +// scanEventfdNumber checks the two rules about the raw eventfd number: in +// newWorker the identifier efd appears only as an argument of wakefd.New, +// unix.EpollCtl or unix.Close; and a raw write call (unix.Write, syscall.Write +// and their variants) appears only in flushLocked, on c.fd. +func scanEventfdNumber(t *testing.T, dir string) (bad []string) { + t.Helper() + files, err := filepath.Glob(filepath.Join(dir, "*.go")) + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + sawNewWorker, sawFlush := false, false + for _, name := range files { + if strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + f, err := parser.ParseFile(fset, name, src, 0) + if err != nil { + t.Fatalf("parse %s: %v", name, err) + } + for _, decl := range f.Decls { + fd, ok := decl.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + if fd.Recv == nil && fd.Name.Name == "newWorker" { + sawNewWorker = true + bad = append(bad, efdUses(fset, fd)...) + } + ast.Inspect(fd.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok || (pkg.Name != "unix" && pkg.Name != "syscall") || !strings.HasPrefix(sel.Sel.Name, "Write") && !strings.HasPrefix(sel.Sel.Name, "Pwrite") { + return true + } + first := "" + if len(call.Args) > 0 { + first = string(src[fset.Position(call.Args[0].Pos()).Offset:fset.Position(call.Args[0].End()).Offset]) + } + if fd.Name.Name == "flushLocked" && first == "c.fd" { + sawFlush = true + return true + } + bad = append(bad, fmt.Sprintf("%s: %s calls %s.%s(%s, ...); the only raw write allowed is flushLocked's, on c.fd (a producer's wake must be wakefd.WakeFD.Signal, celeris#890 WAKEATOMIC)", + fset.Position(call.Pos()), fd.Name.Name, pkg.Name, sel.Sel.Name, first)) + return true + }) + } + } + if !sawNewWorker || !sawFlush { + bad = append(bad, fmt.Sprintf("the scan did not find newWorker (%v) or flushLocked's write (%v): the code changed shape; update this guard with it", sawNewWorker, sawFlush)) + } + return bad +} + +// efdUses reports every use of the identifier efd in fd that is not an +// argument of wakefd.New, unix.EpollCtl or unix.Close (anywhere beneath the +// EpollCtl call, for int32(efd) in the event). +func efdUses(fset *token.FileSet, fd *ast.FuncDecl) (bad []string) { + allowed := map[*ast.Ident]bool{} + ast.Inspect(fd.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok { + return true + } + switch pkg.Name + "." + sel.Sel.Name { + case "wakefd.New", "unix.Close": + for _, a := range call.Args { + if id, ok := a.(*ast.Ident); ok && id.Name == "efd" { + allowed[id] = true + } + } + case "unix.EpollCtl": + ast.Inspect(call, func(m ast.Node) bool { + if id, ok := m.(*ast.Ident); ok && id.Name == "efd" { + allowed[id] = true + } + return true + }) + } + return true + }) + ast.Inspect(fd.Body, func(n ast.Node) bool { + // The definition (efd, err := unix.Eventfd(...)) is an Ident on the + // left of a define; skip it. + if as, ok := n.(*ast.AssignStmt); ok && as.Tok == token.DEFINE { + for _, l := range as.Lhs { + if id, ok := l.(*ast.Ident); ok && id.Name == "efd" { + allowed[id] = true + } + } + } + if id, ok := n.(*ast.Ident); ok && id.Name == "efd" && !allowed[id] { + bad = append(bad, fmt.Sprintf("%s: newWorker uses the eventfd number efd outside wakefd.New, unix.EpollCtl and unix.Close (a copy of the number in another place can be written by a producer, celeris#890 WAKEATOMIC)", fset.Position(id.Pos()))) + } + return true + }) + return bad +} diff --git a/driver/redis/sentinel_close_race_887_linux_test.go b/driver/redis/sentinel_close_race_887_linux_test.go new file mode 100644 index 000000000..1df02787b --- /dev/null +++ b/driver/redis/sentinel_close_race_887_linux_test.go @@ -0,0 +1,132 @@ +//go:build linux + +package redis + +// celeris#887 item 29, review round 1 of the PR: the give-up of watchFailover +// closed the sentinel's client outside sentinelMu while that sentinel was still +// listed, so SentinelClient.Close could close the same Client at the same time. +// redisPool.Close is not safe for that (it reads and writes p.provider with no +// lock and then releases the shared standalone loop), and the sentinel's +// clients, the master client and any other client of the process share that +// loop's refcount. Close racing a subscribeLoop that had already passed its +// closed check also left a freshly connected sentinel open for ever (pre- +// existing). + +import ( + "math/rand/v2" + "sync" + "testing" + "time" +) + +// Two goroutines close one Client. Under -race this is a data race on +// p.provider unless Close is idempotent under concurrency, and a double +// Release of the loop otherwise. +func TestClientCloseTwiceConcurrently887(t *testing.T) { + fake := startFakeRedis(t, nil) + // A bystander on the same standalone loop must keep its conn. + by := startHangUpServer(t, func(int) time.Duration { return 0 }) + ps := subscribeToHangUpServer(t, by) + conn0 := ps.connNow() + for i := 0; i < 50; i++ { + cl, err := NewClient(fake.Addr(), WithForceRESP2()) + if err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + start := make(chan struct{}) + for g := 0; g < 2; g++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _ = cl.Close() + }() + } + close(start) + wg.Wait() + } + time.Sleep(100 * time.Millisecond) + if c := ps.connNow(); c != conn0 || c.closed.Load() { + t.Fatal("the bystander PubSub lost its conn: the standalone loop was released under it") + } +} + +// The give-up and SentinelClient.Close race: both sentinels hang up every 2 ms +// and the grace is 20 ms, so the client gives up one and moves to the other at +// about the rate the test closes it. A bystander on the shared loop must keep +// its conn; under -race the double close is reported. +func TestSentinelCloseVsGiveUp887(t *testing.T) { + oldGrace := sentinelDownGrace + sentinelDownGrace = 20 * time.Millisecond + t.Cleanup(func() { sentinelDownGrace = oldGrace }) + master := startFakeMaster(t) + a := startFakeSentinel(t, master.Addr()) + b := startFakeSentinel(t, master.Addr()) + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + go func() { + for { + select { + case <-stop: + return + case <-time.After(2 * time.Millisecond): + a.dropAll() + b.dropAll() + } + } + }() + by := startHangUpServer(t, func(int) time.Duration { return 0 }) + ps := subscribeToHangUpServer(t, by) + conn0 := ps.connNow() + for i := 0; i < 40; i++ { + sc, err := NewSentinelClient(SentinelConfig{MasterName: "mymaster", SentinelAddrs: []string{a.Addr(), b.Addr()}, DialTimeout: time.Second}) + if err != nil { + continue + } + time.Sleep(time.Duration(rand.IntN(150)) * time.Millisecond) + _ = sc.Close() + } + time.Sleep(300 * time.Millisecond) + if c := ps.connNow(); c != conn0 || c.closed.Load() { + t.Errorf("the bystander PubSub lost its conn: the standalone loop was released under it") + } +} + +// Close at a random moment after NewSentinelClient: after Close has returned +// no sentinel conn may stay open. Before the fix subscribeLoop appended the +// sentinel it had just connected without looking at the closed flag, so it +// stayed in sentinelConns, open, after Close had cleared the list. +func TestSentinelCloseDuringSubscribeLeaksNothing887(t *testing.T) { + master := startFakeMaster(t) + a := startFakeSentinel(t, master.Addr()) + leaks, iters := 0, 300 + for i := 0; i < iters; i++ { + sc, err := NewSentinelClient(SentinelConfig{ + MasterName: "mymaster", + SentinelAddrs: []string{a.Addr()}, + DialTimeout: 2 * time.Second, + }) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Duration(rand.IntN(400)) * time.Microsecond) + _ = sc.Close() + time.Sleep(5 * time.Millisecond) + sc.sentinelMu.Lock() + var left *sentinelConn + if len(sc.sentinelConns) > 0 { + left = sc.sentinelConns[0] + } + sc.sentinelMu.Unlock() + if left != nil && !left.subPS.closed.Load() { + leaks++ + _ = left.subPS.Close() + _ = left.client.Close() + } + } + t.Logf("%d of %d iterations left an open sentinel PubSub and client after Close", leaks, iters) + if leaks > 0 { + t.Errorf("%d of %d Close calls left a sentinel connection open", leaks, iters) + } +} From 4340ecd3bb1c82e0bf5ae941c84f1ee1a4aa24d1 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Tue, 27 Oct 2026 04:38:10 +0100 Subject: [PATCH 4/4] driver/redis: sentinel give-up and Close own the close, Client.Close idempotent, Close during subscribe leaks nothing; eventloop: RegisterConnWait wraps why it stopped, wake guard sees a copy of the eventfd number (Refs #887 #890) Review round 1 of the PR. redisPool.Close runs its body once. The give-up of watchFailover takes the sentinel out of sentinelConns under sentinelMu and closes it only if it was still listed, so SentinelClient.Close and the give-up never close the same Client; subscribeLoop closes a sentinel it connected after Close instead of listing it. RegisterConnWait returns the refusal together with ctx.Err() or the budget, and its godoc says it must not run on the worker it waits for. The wake guard also pins where the eventfd number goes (only wakefd.New, unix.EpollCtl, unix.Close) and that the only raw write is flushLocked's. --- driver/internal/eventloop/register_wait.go | 31 ++++++++++++--- driver/redis/pool.go | 23 ++++++++--- driver/redis/sentinel.go | 46 ++++++++++++++++++++-- 3 files changed, 85 insertions(+), 15 deletions(-) diff --git a/driver/internal/eventloop/register_wait.go b/driver/internal/eventloop/register_wait.go index cab6e8231..021d9555e 100644 --- a/driver/internal/eventloop/register_wait.go +++ b/driver/internal/eventloop/register_wait.go @@ -3,6 +3,7 @@ package eventloop import ( "context" "errors" + "fmt" "strings" "time" @@ -46,10 +47,28 @@ func IsNumberBusy(err error) bool { // RegisterConnWait is loop.RegisterConn that waits out the io_uring engine's // "fd already registered" refusal (see [IsNumberBusy]) instead of returning it: // it retries with a growing pause until the number is free, ctx is done, or -// registerWaitBudget has passed, and then returns the refusal. Every other -// error is returned at once, and a failed call leaves fd with the caller, as -// RegisterConn does. The drivers' dials use it: without it Pool.Acquire fails -// under connection churn on the io_uring engine (celeris#887 item 28). +// registerWaitBudget has passed, and then returns the refusal wrapped together +// with the reason it stopped (ctx.Err(), or errRegisterWaitBudget), so +// errors.Is sees both. Every other error is returned at once, and a failed +// call leaves fd with the caller, as RegisterConn does. The drivers' dials use +// it: without it Pool.Acquire fails under connection churn on the io_uring +// engine (celeris#887 item 28). +// +// It must not run on the worker goroutine of loop (a handler of the server +// whose engine is that loop's, called inline): the number is finalized by that +// worker, which would be asleep in this wait, so the wait runs to its end, +// stalls every connection of that worker for up to registerWaitBudget, and +// returns the refusal anyway. The drivers do not dial there: with WithEngine on +// io_uring, epoll or adaptive their command pools dial direct net.Conns and +// never call RegisterConn. The one public path left is redis +// Client.Subscribe called from a handler, whose PubSub pool still registers +// with the engine; with the default config its handshake already blocks on the +// worker it occupies, so the case that reaches this wait is ForceRESP2 with no +// password and no DB. The engine gives no cheap, reliable way to tell that the +// caller is on its worker, so this is documented, not guarded. The alternative +// that works on any thread, moving the socket to a free number (F_DUPFD) before +// registering it, changes which descriptor the driver owns and needs its own +// review. func RegisterConnWait(ctx context.Context, loop engine.WorkerLoop, fd int, onRecv func([]byte), onClose func(error)) error { err := loop.RegisterConn(fd, onRecv, onClose) if !IsNumberBusy(err) { @@ -62,14 +81,14 @@ func RegisterConnWait(ctx context.Context, loop engine.WorkerLoop, fd int, onRec select { case <-ctx.Done(): t.Stop() - return err + return fmt.Errorf("%w (gave up waiting: %w)", err, ctx.Err()) case <-t.C: } if err = loop.RegisterConn(fd, onRecv, onClose); !IsNumberBusy(err) { return err } if !time.Now().Before(deadline) { - return err + return fmt.Errorf("%w (gave up waiting: %w)", err, errRegisterWaitBudget) } if pause < 5*time.Millisecond { pause *= 2 diff --git a/driver/redis/pool.go b/driver/redis/pool.go index fa683063f..40bd75f60 100644 --- a/driver/redis/pool.go +++ b/driver/redis/pool.go @@ -3,6 +3,7 @@ package redis import ( "context" "runtime" + "sync" "time" "github.com/goceleris/celeris/driver/internal/async" @@ -20,6 +21,8 @@ type redisPool struct { cfg Config provider engine.EventLoopProvider ownsLoop bool + // closeOnce makes Close idempotent. + closeOnce sync.Once } // newDirectCmdPool creates a pool whose cmd conns dial via direct @@ -271,13 +274,21 @@ func (p *redisPool) releasePubSub(c *redisConn) { } // Close closes both pools and releases the event loop if owned. +// +// It is safe to call from several goroutines and more than once: the body runs +// once, and a second caller returns at once. Releasing the shared standalone +// loop twice would drop a reference that belongs to another client +// (celeris#887 item 29: SentinelClient.Close and the give-up of a sentinel +// closed the same Client). func (p *redisPool) Close() error { - _ = p.cmd.Close() - _ = p.pubsub.Close() - if p.ownsLoop && p.provider != nil { - eventloop.Release(p.provider) - p.provider = nil - } + p.closeOnce.Do(func() { + _ = p.cmd.Close() + _ = p.pubsub.Close() + if p.ownsLoop && p.provider != nil { + eventloop.Release(p.provider) + p.provider = nil + } + }) return nil } diff --git a/driver/redis/sentinel.go b/driver/redis/sentinel.go index bd027acfb..f4d1a14db 100644 --- a/driver/redis/sentinel.go +++ b/driver/redis/sentinel.go @@ -280,6 +280,15 @@ func (s *SentinelClient) subscribeLoop() { // exited on read error (that's what triggered this // reconnect iteration). s.sentinelMu.Lock() + if s.closed.Load() { + // Close ran between the check at the top of the loop and here: its + // list was already cleared, so it will never close sc. Whoever + // takes a sentinel out of the list, or declines to put it there, + // closes it (celeris#887 item 29). + s.sentinelMu.Unlock() + s.closeSentinel(sc) + return + } for _, old := range s.sentinelConns { // The PubSub first: its conn is checked out of the client's pool, // so closing the client does not stop its reconnect loop. @@ -298,6 +307,32 @@ func (s *SentinelClient) subscribeLoop() { } } +// takeSentinel removes sc from sentinelConns and reports whether it was there. +// The caller that gets true owns sc's close. +func (s *SentinelClient) takeSentinel(sc *sentinelConn) bool { + s.sentinelMu.Lock() + defer s.sentinelMu.Unlock() + for i, c := range s.sentinelConns { + if c == sc { + s.sentinelConns = append(s.sentinelConns[:i], s.sentinelConns[i+1:]...) + return true + } + } + return false +} + +// closeSentinel closes sc's PubSub and then its client: the PubSub's conn is +// checked out of the client's pool, so closing the client alone does not stop +// its reconnect loop. +func (s *SentinelClient) closeSentinel(sc *sentinelConn) { + if sc.subPS != nil { + _ = sc.subPS.Close() + } + if sc.client != nil { + _ = sc.client.Close() + } +} + // sentinelIndex is the index of addr in SentinelAddrs, or -1. func (s *SentinelClient) sentinelIndex(addr string) int { for i, a := range s.cfg.SentinelAddrs { @@ -389,9 +424,14 @@ func (s *SentinelClient) watchFailover(sc *sentinelConn) (gaveUp bool) { s.handleSwitchMaster(msg.Payload) case <-tick.C: if sc.subPS.downFor() >= grace { - _ = sc.subPS.Close() - _ = sc.client.Close() - return true + // Whoever takes sc out of sentinelConns closes it: if + // SentinelClient.Close got there first it has closed (or is + // closing, under sentinelMu) this very client, and a second + // close would race it. + if s.takeSentinel(sc) { + s.closeSentinel(sc) + } + return !s.closed.Load() } case <-s.closeCh: return false