diff --git a/driver/redis/cluster.go b/driver/redis/cluster.go index 8d31e385b..e90b58fc9 100644 --- a/driver/redis/cluster.go +++ b/driver/redis/cluster.go @@ -48,9 +48,11 @@ type ClusterConfig struct { PoolSize int MaxIdlePerWorker int - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration + // DialTimeout bounds each new node connection's TCP connect and, separately, + // its handshake; default 5s. See [Config].DialTimeout. + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration // MaxRedirects is the maximum number of MOVED/ASK redirects before an // error is returned. Default: 3. MaxRedirects int diff --git a/driver/redis/conn.go b/driver/redis/conn.go index d9ed7aa03..c8092a7d6 100644 --- a/driver/redis/conn.go +++ b/driver/redis/conn.go @@ -776,9 +776,54 @@ func (c *redisConn) releaseHandshakeReq(req *redisRequest) { c.releaseResult(req) } +// testHookDefaultHandshakeTimeout, when non-nil, replaces defaultDialTimeout as +// the handshake bound of a conn whose Config.DialTimeout is unset. It is nil in +// production; a test sets it to see that the default is applied without waiting +// 5 s, and resets it to nil when it is done. +var testHookDefaultHandshakeTimeout atomic.Pointer[time.Duration] + // handshake runs HELLO (with optional fallback to AUTH+SELECT) to negotiate -// protocol version and authentication. +// protocol version and authentication. The whole exchange is bounded by +// Config.DialTimeout (default 5 s), on top of whatever deadline ctx carries: the +// dialer's timeout covers only the TCP connect (connect and handshake are two +// budgets, each DialTimeout), and a node that accepts and +// never answers HELLO would otherwise hold the dial until ctx ends, which is +// never under context.Background (celeris#1205). The three dial paths +// (dialRedisConn on unix and on the net.Conn loop, dialDirectRedisConn) all end +// here, so the pool's command and pub/sub dials are bounded alike. func (c *redisConn) handshake(ctx context.Context) error { + timeout := c.cfg.DialTimeout + if timeout <= 0 { + timeout = defaultDialTimeout + if h := testHookDefaultHandshakeTimeout.Load(); h != nil { + timeout = *h + } + } + // Is the caller's own deadline the earlier one? Then it ends the handshake + // and is reported as the caller's. + callerFirst := false + if dl, ok := ctx.Deadline(); ok && !dl.After(time.Now().Add(timeout)) { + callerFirst = true + } + hctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + err := c.runHandshake(hctx) + if err != nil && !callerFirst && ctx.Err() == nil { + // The direct path reports the socket's deadline (it can fire a hair + // before hctx's timer), the others the context's. + var rerr *RedisError + if (hctx.Err() != nil || errors.Is(err, os.ErrDeadlineExceeded)) && !errors.As(err, &rerr) { + if errors.Is(err, context.DeadlineExceeded) { + return fmt.Errorf("celeris-redis: handshake timed out after %v: %w", timeout, err) + } + return fmt.Errorf("celeris-redis: handshake timed out after %v: %w: %w", timeout, context.DeadlineExceeded, err) + } + } + return err +} + +// runHandshake is handshake without the bound. +func (c *redisConn) runHandshake(ctx context.Context) error { if c.cfg.ForceRESP2 { return c.legacyAuth(ctx) } diff --git a/driver/redis/handshake_bound_1205_test.go b/driver/redis/handshake_bound_1205_test.go new file mode 100644 index 000000000..52b284eba --- /dev/null +++ b/driver/redis/handshake_bound_1205_test.go @@ -0,0 +1,335 @@ +package redis + +// celeris#1205: DialTimeout bounded only net.Dialer.DialContext. The handshake +// (HELLO, then AUTH/SELECT) ran on the caller's context, so a node that accepts +// TCP and never answers HELLO held ClusterClient.Subscribe, ClusterClient.SSubscribe, +// Client.Subscribe and every Do that needed a new conn until the caller's context +// ended: for ever under context.Background. Found by the review of PR #1154; the +// code is older. + +import ( + "context" + "errors" + "io" + "net" + "os" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/goceleris/celeris/driver/internal/eventloop" + "github.com/goceleris/celeris/internal/engine" +) + +// startSilentNode accepts TCP conns and never answers them. +func startSilentNode(t *testing.T) string { + t.Helper() + addr, _ := startSilentNodeCounting(t) + return addr +} + +// startSilentNodeCounting is startSilentNode that also counts the conns the +// client has closed (the server read their end of stream). +func startSilentNodeCounting(t *testing.T) (string, *atomic.Int32) { + t.Helper() + closed := new(atomic.Int32) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + var mu sync.Mutex + var conns []net.Conn + t.Cleanup(func() { + _ = ln.Close() + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + _ = c.Close() + } + }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + go func() { + _, _ = io.Copy(io.Discard, c) // reads the HELLO, never answers + closed.Add(1) + }() + } + }() + return ln.Addr().String(), closed +} + +// within runs call with a context that has no deadline (the case that hung for +// ever) and fails the test if call is still running after limit, so the failure +// is a message and not a test timeout. +func within(t *testing.T, limit time.Duration, what string, call func(ctx context.Context) error) error { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + start := time.Now() + go func() { done <- call(ctx) }() + select { + case err := <-done: + if err == nil { + t.Fatalf("%s succeeded against a node that never answers HELLO", what) + } + if d := time.Since(start); d > limit { + t.Fatalf("%s returned after %v, want within %v", what, d, limit) + } + return err + case <-time.After(limit): + // Not waited for: a direct-mode conn does not watch the context's + // cancellation, only its deadline, so the call may stay blocked until + // the node's cleanup closes its conns. + cancel() + t.Fatalf("%s still blocked after %v: the handshake is not bounded by DialTimeout", what, limit) + return nil + } +} + +// wantTimeout fails the test unless err is the driver's handshake bound: an +// error that matches context.DeadlineExceeded and names the handshake, once +// (the loop paths' context error is not repeated). +func wantTimeout(t *testing.T, err error) { + t.Helper() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want one matching context.DeadlineExceeded", err) + } + if !strings.Contains(err.Error(), "handshake timed out") { + t.Fatalf("err = %v, want it to say the handshake timed out", err) + } + if strings.Count(err.Error(), "context deadline exceeded") > 1 { + t.Fatalf("err = %q repeats the context error", err) + } +} + +// The three dial paths, called directly: each must give up on a silent node +// within about DialTimeout. +func TestDialHandshakeIsBoundedByDialTimeout1205(t *testing.T) { + addr := startSilentNode(t) + cfg := Config{Addr: addr, DialTimeout: time.Second} + + prov, err := eventloop.Resolve(nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { eventloop.Release(prov) }) + portable := netConnProvider(t) + + cases := []struct { + name string + dial func(ctx context.Context) error + }{ + {"direct", func(ctx context.Context) error { + c, err := dialDirectRedisConn(ctx, cfg) + if c != nil { + _ = c.Close() + } + return err + }}, + {"eventloop", func(ctx context.Context) error { + c, err := dialRedisConn(ctx, prov, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + {"netconn", func(ctx context.Context) error { + var p engine.EventLoopProvider = portable + c, err := dialRedisConnNetConn(ctx, p, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + err := within(t, 2*time.Second, tc.name+" dial", tc.dial) + wantTimeout(t, err) + }) + } +} + +// A caller's own, shorter deadline still wins, and is not reported as the +// driver's. +func TestDialHandshakeKeepsTheCallersShorterDeadline1205(t *testing.T) { + addr := startSilentNode(t) + cfg := Config{Addr: addr, DialTimeout: 30 * time.Second} + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + start := time.Now() + c, err := dialDirectRedisConn(ctx, cfg) + if c != nil { + _ = c.Close() + } + if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, os.ErrDeadlineExceeded) { + t.Fatalf("err = %v, want a deadline error", err) + } + if errors.Is(err, context.DeadlineExceeded) && strings.Contains(err.Error(), "handshake timed out") { + t.Fatalf("err = %v: the caller's own deadline is reported as the driver's DialTimeout", err) + } + if d := time.Since(start); d > time.Second { + t.Fatalf("returned after %v, want about the caller's 200ms", d) + } +} + +// The public entry points: Client.Subscribe, and a Do that needs a new conn. +func TestClientSubscribeAndDoBoundTheHandshake1205(t *testing.T) { + addr := startSilentNode(t) + c, err := NewClient(addr, WithDialTimeout(time.Second)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + t.Run("Subscribe", func(t *testing.T) { + t.Parallel() + wantTimeout(t, within(t, 2*time.Second, "Client.Subscribe", func(ctx context.Context) error { + ps, err := c.Subscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + })) + }) + t.Run("Do", func(t *testing.T) { + t.Parallel() + wantTimeout(t, within(t, 2*time.Second, "Client.Do", func(ctx context.Context) error { + _, err := c.Do(ctx, "PING") + return err + })) + }) +} + +// The cluster constructors: the node's command conn is up, a new (pubsub) conn +// to it is accepted and never answered. +func TestClusterSubscribeBoundsTheHandshake1205(t *testing.T) { + cl := newRPCluster(t, 1) + cc, err := NewClusterClient(ClusterConfig{Addrs: cl.addrs(), DialTimeout: 500 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cc.Close() }) + n := cl.nodes[0] + n.mu.Lock() + n.hang = true + n.mu.Unlock() + + t.Run("Subscribe", func(t *testing.T) { + t.Parallel() + wantTimeout(t, within(t, 2*time.Second, "ClusterClient.Subscribe", func(ctx context.Context) error { + ps, err := cc.Subscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + })) + }) + t.Run("SSubscribe", func(t *testing.T) { + t.Parallel() + wantTimeout(t, within(t, 2*time.Second, "ClusterClient.SSubscribe", func(ctx context.Context) error { + ps, err := cc.SSubscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + })) + }) +} + +// A client whose DialTimeout is unset gets the default (5 s) as the handshake +// bound. The test lowers the default through the test hook, so it does not wait +// 5 s, and checks that the bound is the default's, not none. +func TestHandshakeIsBoundedByTheDefaultDialTimeout1205(t *testing.T) { + addr := startSilentNode(t) + d := 300 * time.Millisecond + testHookDefaultHandshakeTimeout.Store(&d) + t.Cleanup(func() { testHookDefaultHandshakeTimeout.Store(nil) }) + + c, err := NewClient(addr) // no WithDialTimeout + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = c.Close() }) + start := time.Now() + err = within(t, 2*time.Second, "Client.Subscribe with the default DialTimeout", func(ctx context.Context) error { + ps, err := c.Subscribe(ctx, "ch") + if ps != nil { + _ = ps.Close() + } + return err + }) + wantTimeout(t, err) + if !strings.Contains(err.Error(), "after "+d.String()) { + t.Fatalf("err = %v, want the bound to be the default (%v)", err, d) + } + if el := time.Since(start); el < d { + t.Fatalf("returned after %v, before the %v bound", el, d) + } +} + +// A caller that cancels its context during the handshake gets its own +// cancellation back at once, not relabelled as the driver's bound, and the +// half-open conn is closed (the node sees the end of stream). The direct path +// does not watch a context's cancellation (only its deadline), so it is not in +// the cases. +func TestHandshakeKeepsTheCallersCancellation1205(t *testing.T) { + prov, err := eventloop.Resolve(nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { eventloop.Release(prov) }) + portable := netConnProvider(t) + + cases := []struct { + name string + dial func(ctx context.Context, cfg Config) error + }{ + {"eventloop", func(ctx context.Context, cfg Config) error { + c, err := dialRedisConn(ctx, prov, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + {"netconn", func(ctx context.Context, cfg Config) error { + var p engine.EventLoopProvider = portable + c, err := dialRedisConnNetConn(ctx, p, cfg, -1) + if c != nil { + _ = c.Close() + } + return err + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + addr, closed := startSilentNodeCounting(t) + cfg := Config{Addr: addr, DialTimeout: 30 * time.Second} + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(200*time.Millisecond, cancel) + start := time.Now() + err := tc.dial(ctx, cfg) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if strings.Contains(err.Error(), "handshake timed out") { + t.Fatalf("err = %v: the caller's cancellation is reported as the driver's bound", err) + } + if el := time.Since(start); el > time.Second { + t.Fatalf("returned after %v, want about the cancel at 200ms", el) + } + poll(t, "the node to see the conn closed", 2*time.Second, func() bool { return closed.Load() == 1 }) + }) + } +} diff --git a/driver/redis/options.go b/driver/redis/options.go index 8537e4c96..99d44eac5 100644 --- a/driver/redis/options.go +++ b/driver/redis/options.go @@ -25,7 +25,10 @@ type Config struct { Username string // DB is the numeric database index applied via SELECT post-HELLO. DB int - // DialTimeout is the TCP dial timeout; default 5s. + // DialTimeout bounds each new connection's TCP connect and, separately, + // its handshake (HELLO, then AUTH and SELECT); default 5s. A server that + // accepts the connection and does not answer the handshake fails the dial + // after DialTimeout, also when the caller's context has no deadline. DialTimeout time.Duration // ReadTimeout is unused currently — responses are driven by the event // loop and cancellation is via the request context. @@ -76,7 +79,8 @@ func WithUsername(s string) Option { return func(c *Config) { c.Username = s } } // WithDB selects a database index. func WithDB(db int) Option { return func(c *Config) { c.DB = db } } -// WithDialTimeout sets the TCP dial timeout. +// WithDialTimeout sets Config.DialTimeout: the bound of a new connection's TCP +// connect and of its handshake, each. func WithDialTimeout(d time.Duration) Option { return func(c *Config) { c.DialTimeout = d } } // WithReadTimeout sets the read timeout (currently advisory). diff --git a/driver/redis/sentinel.go b/driver/redis/sentinel.go index 327244df3..7252b8be2 100644 --- a/driver/redis/sentinel.go +++ b/driver/redis/sentinel.go @@ -31,10 +31,12 @@ type SentinelConfig struct { PoolSize int MaxIdlePerWorker int - DialTimeout time.Duration - ReadTimeout time.Duration - WriteTimeout time.Duration - Engine ServerProvider + // DialTimeout bounds each new connection's TCP connect and, separately, + // its handshake; default 5s. See [Config].DialTimeout. + DialTimeout time.Duration + ReadTimeout time.Duration + WriteTimeout time.Duration + Engine ServerProvider } // sentinelDownGrace is how long watchFailover lets the PubSub of the sentinel