diff --git a/driver/redis/cluster.go b/driver/redis/cluster.go index e395070f4..832ba8190 100644 --- a/driver/redis/cluster.go +++ b/driver/redis/cluster.go @@ -77,10 +77,13 @@ var ErrClusterMaxRedirects = errors.New("celeris-redis: cluster max redirects ex type ClusterClient struct { cfg ClusterConfig - mu sync.RWMutex - nodes map[string]*clusterNode // addr -> node - slots [16384]*clusterNode // slot -> primary node - replicas [16384][]*clusterNode // slot -> replica nodes + mu sync.RWMutex + nodes map[string]*clusterNode // addr -> node + // topoApplied is the registration number of the read whose table is in + // slots (guarded by mu; see applyTopology). + topoApplied uint64 + slots [16384]*clusterNode // slot -> primary node + replicas [16384][]*clusterNode // slot -> replica nodes replicaIdx atomic.Uint64 // round-robin counter for replica selection @@ -538,7 +541,7 @@ func (c *ClusterClient) runRefresh(ctx context.Context, f *topoRefresh) error { // Closed on a panic too, so joiners are not left waiting for a read that // will never end (the panic goes on to the caller). defer close(f.done) - f.err = c.readTopology(ctx) + f.err = c.readTopology(ctx, f.seq) f.cut = f.err != nil && ctx.Err() != nil if !f.cut { c.refMu.Lock() @@ -558,7 +561,7 @@ const topologyProbes = 3 // not marked failing, up to topologyProbes. One failed read is shared by every // caller it serves (see refreshTopologySince), so it must not be lost to a node // the others could have come from. -func (c *ClusterClient) readTopology(ctx context.Context) error { +func (c *ClusterClient) readTopology(ctx context.Context, seq uint64) error { var tried []*clusterNode var lastErr error for range topologyProbes { @@ -580,7 +583,7 @@ func (c *ClusterClient) readTopology(ctx context.Context) error { if v.Type != protocol.TyArray { return fmt.Errorf("CLUSTER SLOTS: unexpected reply type %s", v.Type) } - return c.applyTopology(ctx, v) + return c.applyTopology(ctx, v, seq) } if lastErr != nil { return lastErr @@ -611,7 +614,7 @@ func (c *ClusterClient) probeNode(tried []*clusterNode) *clusterNode { // applyTopology rebuilds the slot-to-node mapping from a CLUSTER SLOTS reply. // It is the part of refreshTopology that runs under c.mu, split out so a test // can run it at the moment a Close has just returned. -func (c *ClusterClient) applyTopology(ctx context.Context, v *Value) error { +func (c *ClusterClient) applyTopology(ctx context.Context, v *Value, seq uint64) error { c.mu.Lock() defer c.mu.Unlock() // closeNodes leaves c.nodes nil: a refresh whose reply came before Close @@ -619,6 +622,23 @@ func (c *ClusterClient) applyTopology(ctx context.Context, v *Value) error { if c.nodes == nil { return ErrClosed } + // The slot table is never replaced by a read registered before the one whose + // table is in place, whatever order they ended in: an older read that ends last + // cannot put a slot back where it was before a move (celeris#1219). seq is the + // read's registration number (topoRefresh.seq, 1 or more). A read that failed, + // or was cut short, never gets here and so does not advance topoApplied. + // + // What this does NOT guarantee: the order is that of registration, not of the + // requests reaching the server or of the cluster's truth, which a client cannot + // tell without a server epoch (CLUSTER SLOTS has none). A read registered later + // that was sent to a lagging node, or an older read that retried a second node + // after the newer one was sent, can still install the older table, and the + // fresher answer is dropped. The next -MOVED corrects it (one extra round), as + // would the 60 s refresh. + if seq < c.topoApplied { + return nil + } + c.topoApplied = seq wantReplicas := c.cfg.ReadOnly || c.cfg.RouteByLatency diff --git a/driver/redis/cluster_close_refresh_1104_test.go b/driver/redis/cluster_close_refresh_1104_test.go index fa3b859c3..b807af155 100644 --- a/driver/redis/cluster_close_refresh_1104_test.go +++ b/driver/redis/cluster_close_refresh_1104_test.go @@ -59,7 +59,7 @@ func TestClusterRefreshAfterCloseDoesNotWriteToTheNodes1104(t *testing.T) { for range iters { cc, v, _ := closedClusterWithSlots1104(t) var err error - if p := panicOf1104(func() { err = cc.applyTopology(context.Background(), v) }); p != nil { + if p := panicOf1104(func() { err = cc.applyTopology(context.Background(), v, 1) }); p != nil { panics++ continue } diff --git a/driver/redis/sentinel.go b/driver/redis/sentinel.go index 7252b8be2..69f49f151 100644 --- a/driver/redis/sentinel.go +++ b/driver/redis/sentinel.go @@ -455,6 +455,11 @@ var ErrSentinelUnhealthy = errors.New("celeris-redis: sentinel client unhealthy, // On dial failure, it retries 3 times with exponential backoff. If all retries // fail, the client is marked unhealthy and returns errors until the next // successful +switch-master. +// +// It is called only from watchFailover, which subscribeLoop runs inline, so the +// notifications are handled one at a time in the order they were delivered: a +// slow dial of an older failover cannot install its client after a newer one (the +// ordering problem of celeris#1219 does not exist on this path). func (s *SentinelClient) handleSwitchMaster(payload string) { // watchFailover's select between a queued message and closeCh picks at // random, so this can run after Close: make no dial for a closed client. diff --git a/driver/redis/topology_order_1219_test.go b/driver/redis/topology_order_1219_test.go new file mode 100644 index 000000000..8992d3bb2 --- /dev/null +++ b/driver/redis/topology_order_1219_test.go @@ -0,0 +1,114 @@ +package redis + +// celeris#1219: an older CLUSTER SLOTS read that ended AFTER a newer one overwrote +// the slot table with its older answer (applyTopology had no ordering): a node +// that learned of a move late could put the table back to the state before the +// move, and the commands then took an extra -MOVED round or a stale route until the +// next refresh. Found by the review of #1215. + +import ( + "context" + "testing" +) + +// Read 1 takes its table before a slot moves and is held; read 2 starts after the +// move, ends and is applied; read 1 is released and ends last. The table must keep +// read 2's answer. (Main: read 1 overwrites it, the slot routes to the old owner.) +func TestAnOlderTopologyReadEndingLastDoesNotOverwriteANewerOne1219(t *testing.T) { + const key = "{order}k" + cl := newRPCluster(t, 3) + cc := newRPClusterClient(t, cl) + slot := int(Slot(key)) + from := cl.owner(slot) + to := otherNode(cl, from) + + waitHeld, release := holdNextSlotsRead(t) + r1 := make(chan error, 1) + go func() { r1 <- cc.refreshTopology(rpCtx(t)) }() // registered first, takes the pre-move table + waitHeld() + cl.moveSlot(slot, to, 0) + if err := cc.refreshTopology(rpCtx(t)); err != nil { // registered later, applied + t.Fatal(err) + } + if r := cc.route(Slot(key)); r == nil || r.addr != to.addr { + t.Fatal("the newer read did not put the slot on the new owner") + } + release() + if err := <-r1; err != nil { + t.Fatal(err) + } + if r := cc.route(Slot(key)); r == nil || r.addr != to.addr { + t.Errorf("the table went back to the pre-move owner: the older read, which ended last, overwrote the newer one") + } +} + +// The order of registration decides, not the order of ending: reads that end in +// the order they started all apply (a refresh after a move updates the table), and +// so does a newer read after an older one that failed. +func TestTopologyReadsThatEndInOrderAllApply1219(t *testing.T) { + const key = "{inorder}k" + cl := newRPCluster(t, 3) + cc := newRPClusterClient(t, cl) + slot := int(Slot(key)) + a := cl.owner(slot) + b := otherNode(cl, a) + ctx := rpCtx(t) + for i, to := range []*rpNode{b, a, b, a} { + cl.moveSlot(slot, to, 0) + if err := cc.refreshTopology(ctx); err != nil { + t.Fatal(err) + } + if r := cc.route(Slot(key)); r == nil || r.addr != to.addr { + t.Fatalf("move %d: after the refresh the slot does not route to its new owner", i) + } + } + // A read cut short by its caller's context applies nothing and does not + // advance the order: the next read still applies. + cl.moveSlot(slot, b, 0) + cctx, cancel := context.WithCancel(ctx) + waitHeld, release := holdNextSlotsRead(t) + r := make(chan error, 1) + go func() { r <- cc.refreshTopology(cctx) }() + waitHeld() + cancel() + t.Logf("the cancelled read: %v", <-r) + release() + if err := cc.refreshTopology(ctx); err != nil { + t.Fatal(err) + } + if rt := cc.route(Slot(key)); rt == nil || rt.addr != b.addr { + t.Error("a read after a cancelled one did not apply") + } +} + +// An older read that carries the move is still in flight when a newer read fails +// (here: cut short by its caller). The failed read applies nothing, so it must not +// advance the order: the older read, which ends later, applies and the table gets +// the move. (A variant that advances the order when a read fails, or one that drops +// a result whenever a newer read is merely registered, loses the update.) +func TestAnOlderReadStillAppliesWhenTheNewerOneFailed1219(t *testing.T) { + const key = "{newerfails}k" + cl := newRPCluster(t, 3) + cc := newRPClusterClient(t, cl) + slot := int(Slot(key)) + from := cl.owner(slot) + to := otherNode(cl, from) + cl.moveSlot(slot, to, 0) // the client's table still names from + + waitHeld, release := holdNextSlotsRead(t) + r1 := make(chan error, 1) + go func() { r1 <- cc.refreshTopology(rpCtx(t)) }() // takes the post-move table, held + waitHeld() + cctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := cc.refreshTopology(cctx); err == nil { // registered later, cut short + t.Fatal("the cancelled read succeeded") + } + release() + if err := <-r1; err != nil { + t.Fatal(err) + } + if r := cc.route(Slot(key)); r == nil || r.addr != to.addr { + t.Errorf("the older read, which carries the move, was dropped because a newer read failed") + } +}