From 3edf21c817bbfbad0889b00d1d8ad03f4a88b402 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sun, 1 Nov 2026 11:26:05 +0100 Subject: [PATCH 1/3] driver/redis: tests for a handshake that DialTimeout does not bound (Refs #1205) A node that accepts TCP and never answers HELLO holds the dial, and so Client.Subscribe, ClusterClient.Subscribe/SSubscribe and any Do that needs a new conn, until the caller's context ends: for ever under context.Background. These tests fail on main (all three dial paths, Client and ClusterClient entry points). --- driver/redis/handshake_bound_1205_test.go | 219 ++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 driver/redis/handshake_bound_1205_test.go diff --git a/driver/redis/handshake_bound_1205_test.go b/driver/redis/handshake_bound_1205_test.go new file mode 100644 index 000000000..6b91e28bc --- /dev/null +++ b/driver/redis/handshake_bound_1205_test.go @@ -0,0 +1,219 @@ +package redis + +// celeris#1205: DialTimeout bounded only net.Dialer.DialContext. The handshake +// (HELLO, then AUTH/SELECT) ran on the caller's context, so a node that accepts +// TCP and never answers HELLO held ClusterClient.Subscribe, ClusterClient.SSubscribe, +// Client.Subscribe and every Do that needed a new conn until the caller's context +// ended: for ever under context.Background. Found by the review of PR #1154; the +// code is older. + +import ( + "context" + "errors" + "net" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +// startSilentNode accepts TCP conns and never reads or answers them. +func startSilentNode(t *testing.T) string { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var mu sync.Mutex + var conns []net.Conn + t.Cleanup(func() { + _ = ln.Close() + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + _ = c.Close() + } + }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + } + }() + return ln.Addr().String() +} + +// within runs call with a context that has no deadline (the case that hung for +// ever) and fails the test if call is still running after limit, so the failure +// is a message and not a test timeout. +func within(t *testing.T, limit time.Duration, what string, call func(ctx context.Context) error) error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + start := time.Now() + go func() { done <- call(ctx) }() + select { + case err := <-done: + if err == nil { + t.Fatalf("%s succeeded against a node that never answers HELLO", what) + } + if d := time.Since(start); d > limit { + t.Fatalf("%s returned after %v, want within %v", what, d, limit) + } + return err + case <-time.After(limit): + // Not waited for: a direct-mode conn does not watch the context's + // cancellation, only its deadline, so the call may stay blocked until + // the node's cleanup closes its conns. + cancel() + t.Fatalf("%s still blocked after %v: the handshake is not bounded by DialTimeout", what, limit) + return nil + } +} + +// The three dial paths, called directly: each must give up on a silent node +// within about DialTimeout. +func TestDialHandshakeIsBoundedByDialTimeout1205(t *testing.T) { + addr := startSilentNode(t) + cfg := Config{Addr: addr, DialTimeout: time.Second} + + prov, err := eventloop.Resolve(nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { eventloop.Release(prov) }) + portable := netConnProvider(t) + + cases := []struct { + name string + dial func(ctx context.Context) error + }{ + {"direct", func(ctx context.Context) error { + c, err := dialDirectRedisConn(ctx, cfg) + if c != nil { + _ = c.Close() + } + return err + }}, + {"eventloop", func(ctx context.Context) error { + c, err := dialRedisConn(ctx, prov, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + {"netconn", func(ctx context.Context) error { + var p engine.EventLoopProvider = portable + c, err := dialRedisConnNetConn(ctx, p, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + err := within(t, 2*time.Second, tc.name+" dial", tc.dial) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want one matching context.DeadlineExceeded (the handshake timed out)", err) + } + }) + } +} + +// A caller's own, shorter deadline still wins, and is not reported as the +// driver's. +func TestDialHandshakeKeepsTheCallersShorterDeadline1205(t *testing.T) { + addr := startSilentNode(t) + cfg := Config{Addr: addr, DialTimeout: 30 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + c, err := dialDirectRedisConn(ctx, cfg) + if c != nil { + _ = c.Close() + } + if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("err = %v, want a deadline error", err) + } + if errors.Is(err, context.DeadlineExceeded) && strings.Contains(err.Error(), "handshake timed out") { + t.Fatalf("err = %v: the caller's own deadline is reported as the driver's DialTimeout", err) + } + if d := time.Since(start); d > time.Second { + t.Fatalf("returned after %v, want about the caller's 200ms", d) + } +} + +// The public entry points: Client.Subscribe, and a Do that needs a new conn. +func TestClientSubscribeAndDoBoundTheHandshake1205(t *testing.T) { + addr := startSilentNode(t) + c, err := NewClient(addr, WithDialTimeout(time.Second)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + t.Run("Subscribe", func(t *testing.T) { + t.Parallel() + _ = within(t, 2*time.Second, "Client.Subscribe", func(ctx context.Context) error { + ps, err := c.Subscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + }) + }) + t.Run("Do", func(t *testing.T) { + t.Parallel() + _ = within(t, 2*time.Second, "Client.Do", func(ctx context.Context) error { + _, err := c.Do(ctx, "PING") + return err + }) + }) +} + +// The cluster constructors: the node's command conn is up, a new (pubsub) conn +// to it is accepted and never answered. +func TestClusterSubscribeBoundsTheHandshake1205(t *testing.T) { + cl := newRPCluster(t, 1) + cc, err := NewClusterClient(ClusterConfig{Addrs: cl.addrs(), DialTimeout: 500 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cc.Close() }) + n := cl.nodes[0] + n.mu.Lock() + n.hang = true + n.mu.Unlock() + + t.Run("Subscribe", func(t *testing.T) { + t.Parallel() + _ = within(t, 2*time.Second, "ClusterClient.Subscribe", func(ctx context.Context) error { + ps, err := cc.Subscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + }) + }) + t.Run("SSubscribe", func(t *testing.T) { + t.Parallel() + _ = within(t, 2*time.Second, "ClusterClient.SSubscribe", func(ctx context.Context) error { + ps, err := cc.SSubscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + }) + }) +} From 361a5a7cef73c6c92dbb177f82bee91a22e0e0cd Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sun, 1 Nov 2026 11:32:52 +0100 Subject: [PATCH 2/3] driver/redis: bound the connection handshake by DialTimeout (Refs #1205) DialTimeout only covered net.Dialer.DialContext. The handshake (HELLO, then AUTH/SELECT) ran on the caller's context, so a node that accepts TCP and never answers HELLO held ClusterClient.Subscribe, ClusterClient.SSubscribe, Client.Subscribe and any Do that needs a new conn until the caller's context ended: for ever under context.Background. handshake now runs under context.WithTimeout(ctx, DialTimeout) (the 5 s default when unset). All three dial paths (unix event loop, net.Conn loop, direct) and so the pool's command and pub/sub dials end in handshake. A caller's shorter deadline still wins and is reported as the caller's; the driver's bound is reported as an error that matches context.DeadlineExceeded and keeps the underlying cause in the chain. The DialTimeout docs (Config, ClusterConfig, SentinelConfig, WithDialTimeout) now say it bounds the connect and, separately, the handshake. --- driver/redis/cluster.go | 8 +++++--- driver/redis/conn.go | 35 ++++++++++++++++++++++++++++++++++- driver/redis/options.go | 8 ++++++-- driver/redis/sentinel.go | 10 ++++++---- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/driver/redis/cluster.go b/driver/redis/cluster.go index 8d31e385b..e90b58fc9 100644 --- a/driver/redis/cluster.go +++ b/driver/redis/cluster.go @@ -48,9 +48,11 @@ type ClusterConfig struct { PoolSize int MaxIdlePerWorker int - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration + // DialTimeout bounds each new node connection's TCP connect and, separately, + // its handshake; default 5s. See [Config].DialTimeout. + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration // MaxRedirects is the maximum number of MOVED/ASK redirects before an // error is returned. Default: 3. MaxRedirects int diff --git a/driver/redis/conn.go b/driver/redis/conn.go index d9ed7aa03..2c78d3bb8 100644 --- a/driver/redis/conn.go +++ b/driver/redis/conn.go @@ -777,8 +777,41 @@ func (c *redisConn) releaseHandshakeReq(req *redisRequest) { } // handshake runs HELLO (with optional fallback to AUTH+SELECT) to negotiate -// protocol version and authentication. +// protocol version and authentication. The whole exchange is bounded by +// Config.DialTimeout (default 5 s), on top of whatever deadline ctx carries: the +// dialer's timeout covers only the TCP connect (connect and handshake are two +// budgets, each DialTimeout), and a node that accepts and +// never answers HELLO would otherwise hold the dial until ctx ends, which is +// never under context.Background (celeris#1205). The three dial paths +// (dialRedisConn on unix and on the net.Conn loop, dialDirectRedisConn) all end +// here, so the pool's command and pub/sub dials are bounded alike. func (c *redisConn) handshake(ctx context.Context) error { + timeout := c.cfg.DialTimeout + if timeout <= 0 { + timeout = defaultDialTimeout + } + // Is the caller's own deadline the earlier one? Then it ends the handshake + // and is reported as the caller's. + callerFirst := false + if dl, ok := ctx.Deadline(); ok && !dl.After(time.Now().Add(timeout)) { + callerFirst = true + } + hctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + err := c.runHandshake(hctx) + if err != nil && !callerFirst && ctx.Err() == nil { + // The direct path reports the socket's deadline (it can fire a hair + // before hctx's timer), the others the context's. + var rerr *RedisError + if (hctx.Err() != nil || errors.Is(err, os.ErrDeadlineExceeded)) && !errors.As(err, &rerr) { + return fmt.Errorf("celeris-redis: handshake timed out after %v: %w: %w", timeout, context.DeadlineExceeded, err) + } + } + return err +} + +// runHandshake is handshake without the bound. +func (c *redisConn) runHandshake(ctx context.Context) error { if c.cfg.ForceRESP2 { return c.legacyAuth(ctx) } diff --git a/driver/redis/options.go b/driver/redis/options.go index 8537e4c96..99d44eac5 100644 --- a/driver/redis/options.go +++ b/driver/redis/options.go @@ -25,7 +25,10 @@ type Config struct { Username string // DB is the numeric database index applied via SELECT post-HELLO. DB int - // DialTimeout is the TCP dial timeout; default 5s. + // DialTimeout bounds each new connection's TCP connect and, separately, + // its handshake (HELLO, then AUTH and SELECT); default 5s. A server that + // accepts the connection and does not answer the handshake fails the dial + // after DialTimeout, also when the caller's context has no deadline. DialTimeout time.Duration // ReadTimeout is unused currently — responses are driven by the event // loop and cancellation is via the request context. @@ -76,7 +79,8 @@ func WithUsername(s string) Option { return func(c *Config) { c.Username = s } } // WithDB selects a database index. func WithDB(db int) Option { return func(c *Config) { c.DB = db } } -// WithDialTimeout sets the TCP dial timeout. +// WithDialTimeout sets Config.DialTimeout: the bound of a new connection's TCP +// connect and of its handshake, each. func WithDialTimeout(d time.Duration) Option { return func(c *Config) { c.DialTimeout = d } } // WithReadTimeout sets the read timeout (currently advisory). diff --git a/driver/redis/sentinel.go b/driver/redis/sentinel.go index 327244df3..7252b8be2 100644 --- a/driver/redis/sentinel.go +++ b/driver/redis/sentinel.go @@ -31,10 +31,12 @@ type SentinelConfig struct { PoolSize int MaxIdlePerWorker int - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - Engine ServerProvider + // DialTimeout bounds each new connection's TCP connect and, separately, + // its handshake; default 5s. See [Config].DialTimeout. + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + Engine ServerProvider } // sentinelDownGrace is how long watchFailover lets the PubSub of the sentinel From 397ed7f319225ac014900edaaa16e5604d82fe68 Mon Sep 17 00:00:00 2001 From: Albert Bausili Date: Sun, 1 Nov 2026 12:11:34 +0100 Subject: [PATCH 3/3] driver/redis: test the default handshake bound and the caller's cancellation; say the timeout once (Refs #1205) Review r4 of #1209: - a client with DialTimeout unset still bounds a silent HELLO node: a test hook lowers the default so the test does not wait 5 s, and the bound is checked to be the default's; - a caller that cancels during the handshake gets context.Canceled back at once, not the driver's bound, and the half-open conn is closed; - the loop paths' error no longer repeats the context error ("handshake timed out after 1s: context deadline exceeded"); the direct path keeps its socket cause in the chain; - the Client and ClusterClient entry-point tests assert the handshake timeout, not any error. --- driver/redis/conn.go | 12 ++ driver/redis/handshake_bound_1205_test.go | 142 ++++++++++++++++++++-- 2 files changed, 141 insertions(+), 13 deletions(-) diff --git a/driver/redis/conn.go b/driver/redis/conn.go index 2c78d3bb8..c8092a7d6 100644 --- a/driver/redis/conn.go +++ b/driver/redis/conn.go @@ -776,6 +776,12 @@ func (c *redisConn) releaseHandshakeReq(req *redisRequest) { c.releaseResult(req) } +// testHookDefaultHandshakeTimeout, when non-nil, replaces defaultDialTimeout as +// the handshake bound of a conn whose Config.DialTimeout is unset. It is nil in +// production; a test sets it to see that the default is applied without waiting +// 5 s, and resets it to nil when it is done. +var testHookDefaultHandshakeTimeout atomic.Pointer[time.Duration] + // handshake runs HELLO (with optional fallback to AUTH+SELECT) to negotiate // protocol version and authentication. The whole exchange is bounded by // Config.DialTimeout (default 5 s), on top of whatever deadline ctx carries: the @@ -789,6 +795,9 @@ func (c *redisConn) handshake(ctx context.Context) error { timeout := c.cfg.DialTimeout if timeout <= 0 { timeout = defaultDialTimeout + if h := testHookDefaultHandshakeTimeout.Load(); h != nil { + timeout = *h + } } // Is the caller's own deadline the earlier one? Then it ends the handshake // and is reported as the caller's. @@ -804,6 +813,9 @@ func (c *redisConn) handshake(ctx context.Context) error { // before hctx's timer), the others the context's. var rerr *RedisError if (hctx.Err() != nil || errors.Is(err, os.ErrDeadlineExceeded)) && !errors.As(err, &rerr) { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("celeris-redis: handshake timed out after %v: %w", timeout, err) + } return fmt.Errorf("celeris-redis: handshake timed out after %v: %w: %w", timeout, context.DeadlineExceeded, err) } } diff --git a/driver/redis/handshake_bound_1205_test.go b/driver/redis/handshake_bound_1205_test.go index 6b91e28bc..52b284eba 100644 --- a/driver/redis/handshake_bound_1205_test.go +++ b/driver/redis/handshake_bound_1205_test.go @@ -10,10 +10,12 @@ package redis import ( "context" "errors" + "io" "net" "os" "strings" "sync" + "sync/atomic" "testing" "time" @@ -21,9 +23,18 @@ import ( "github.com/goceleris/celeris/internal/engine" ) -// startSilentNode accepts TCP conns and never reads or answers them. +// startSilentNode accepts TCP conns and never answers them. func startSilentNode(t *testing.T) string { t.Helper() + addr, _ := startSilentNodeCounting(t) + return addr +} + +// startSilentNodeCounting is startSilentNode that also counts the conns the +// client has closed (the server read their end of stream). +func startSilentNodeCounting(t *testing.T) (string, *atomic.Int32) { + t.Helper() + closed := new(atomic.Int32) ln, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) @@ -47,9 +58,13 @@ func startSilentNode(t *testing.T) string { mu.Lock() conns = append(conns, c) mu.Unlock() + go func() { + _, _ = io.Copy(io.Discard, c) // reads the HELLO, never answers + closed.Add(1) + }() } }() - return ln.Addr().String() + return ln.Addr().String(), closed } // within runs call with a context that has no deadline (the case that hung for @@ -81,6 +96,22 @@ func within(t *testing.T, limit time.Duration, what string, call func(ctx contex } } +// wantTimeout fails the test unless err is the driver's handshake bound: an +// error that matches context.DeadlineExceeded and names the handshake, once +// (the loop paths' context error is not repeated). +func wantTimeout(t *testing.T, err error) { + t.Helper() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want one matching context.DeadlineExceeded", err) + } + if !strings.Contains(err.Error(), "handshake timed out") { + t.Fatalf("err = %v, want it to say the handshake timed out", err) + } + if strings.Count(err.Error(), "context deadline exceeded") > 1 { + t.Fatalf("err = %q repeats the context error", err) + } +} + // The three dial paths, called directly: each must give up on a silent node // within about DialTimeout. func TestDialHandshakeIsBoundedByDialTimeout1205(t *testing.T) { @@ -125,9 +156,7 @@ func TestDialHandshakeIsBoundedByDialTimeout1205(t *testing.T) { t.Run(tc.name, func(t *testing.T) { t.Parallel() err := within(t, 2*time.Second, tc.name+" dial", tc.dial) - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("err = %v, want one matching context.DeadlineExceeded (the handshake timed out)", err) - } + wantTimeout(t, err) }) } } @@ -165,20 +194,20 @@ func TestClientSubscribeAndDoBoundTheHandshake1205(t *testing.T) { t.Cleanup(func() { _ = c.Close() }) t.Run("Subscribe", func(t *testing.T) { t.Parallel() - _ = within(t, 2*time.Second, "Client.Subscribe", func(ctx context.Context) error { + wantTimeout(t, within(t, 2*time.Second, "Client.Subscribe", func(ctx context.Context) error { ps, err := c.Subscribe(ctx, "ch") if ps != nil { _ = ps.Close() } return err - }) + })) }) t.Run("Do", func(t *testing.T) { t.Parallel() - _ = within(t, 2*time.Second, "Client.Do", func(ctx context.Context) error { + wantTimeout(t, within(t, 2*time.Second, "Client.Do", func(ctx context.Context) error { _, err := c.Do(ctx, "PING") return err - }) + })) }) } @@ -198,22 +227,109 @@ func TestClusterSubscribeBoundsTheHandshake1205(t *testing.T) { t.Run("Subscribe", func(t *testing.T) { t.Parallel() - _ = within(t, 2*time.Second, "ClusterClient.Subscribe", func(ctx context.Context) error { + wantTimeout(t, within(t, 2*time.Second, "ClusterClient.Subscribe", func(ctx context.Context) error { ps, err := cc.Subscribe(ctx, "ch") if ps != nil { _ = ps.Close() } return err - }) + })) }) t.Run("SSubscribe", func(t *testing.T) { t.Parallel() - _ = within(t, 2*time.Second, "ClusterClient.SSubscribe", func(ctx context.Context) error { + wantTimeout(t, within(t, 2*time.Second, "ClusterClient.SSubscribe", func(ctx context.Context) error { ps, err := cc.SSubscribe(ctx, "ch") if ps != nil { _ = ps.Close() } return err - }) + })) }) } + +// A client whose DialTimeout is unset gets the default (5 s) as the handshake +// bound. The test lowers the default through the test hook, so it does not wait +// 5 s, and checks that the bound is the default's, not none. +func TestHandshakeIsBoundedByTheDefaultDialTimeout1205(t *testing.T) { + addr := startSilentNode(t) + d := 300 * time.Millisecond + testHookDefaultHandshakeTimeout.Store(&d) + t.Cleanup(func() { testHookDefaultHandshakeTimeout.Store(nil) }) + + c, err := NewClient(addr) // no WithDialTimeout + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + start := time.Now() + err = within(t, 2*time.Second, "Client.Subscribe with the default DialTimeout", func(ctx context.Context) error { + ps, err := c.Subscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + }) + wantTimeout(t, err) + if !strings.Contains(err.Error(), "after "+d.String()) { + t.Fatalf("err = %v, want the bound to be the default (%v)", err, d) + } + if el := time.Since(start); el < d { + t.Fatalf("returned after %v, before the %v bound", el, d) + } +} + +// A caller that cancels its context during the handshake gets its own +// cancellation back at once, not relabelled as the driver's bound, and the +// half-open conn is closed (the node sees the end of stream). The direct path +// does not watch a context's cancellation (only its deadline), so it is not in +// the cases. +func TestHandshakeKeepsTheCallersCancellation1205(t *testing.T) { + prov, err := eventloop.Resolve(nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { eventloop.Release(prov) }) + portable := netConnProvider(t) + + cases := []struct { + name string + dial func(ctx context.Context, cfg Config) error + }{ + {"eventloop", func(ctx context.Context, cfg Config) error { + c, err := dialRedisConn(ctx, prov, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + {"netconn", func(ctx context.Context, cfg Config) error { + var p engine.EventLoopProvider = portable + c, err := dialRedisConnNetConn(ctx, p, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + addr, closed := startSilentNodeCounting(t) + cfg := Config{Addr: addr, DialTimeout: 30 * time.Second} + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(200*time.Millisecond, cancel) + start := time.Now() + err := tc.dial(ctx, cfg) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if strings.Contains(err.Error(), "handshake timed out") { + t.Fatalf("err = %v: the caller's cancellation is reported as the driver's bound", err) + } + if el := time.Since(start); el > time.Second { + t.Fatalf("returned after %v, want about the cancel at 200ms", el) + } + poll(t, "the node to see the conn closed", 2*time.Second, func() bool { return closed.Load() == 1 }) + }) + } +}