Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions driver/redis/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,10 +77,13 @@ var ErrClusterMaxRedirects = errors.New("celeris-redis: cluster max redirects ex
type ClusterClient struct {
cfg ClusterConfig

mu sync.RWMutex
nodes map[string]*clusterNode // addr -> node
slots [16384]*clusterNode // slot -> primary node
replicas [16384][]*clusterNode // slot -> replica nodes
mu sync.RWMutex
nodes map[string]*clusterNode // addr -> node
// topoApplied is the registration number of the read whose table is in
// slots (guarded by mu; see applyTopology).
topoApplied uint64
slots [16384]*clusterNode // slot -> primary node
replicas [16384][]*clusterNode // slot -> replica nodes

replicaIdx atomic.Uint64 // round-robin counter for replica selection

Expand Down Expand Up @@ -538,7 +541,7 @@ func (c *ClusterClient) runRefresh(ctx context.Context, f *topoRefresh) error {
// Closed on a panic too, so joiners are not left waiting for a read that
// will never end (the panic goes on to the caller).
defer close(f.done)
f.err = c.readTopology(ctx)
f.err = c.readTopology(ctx, f.seq)
f.cut = f.err != nil && ctx.Err() != nil
if !f.cut {
c.refMu.Lock()
Expand All @@ -558,7 +561,7 @@ const topologyProbes = 3
// not marked failing, up to topologyProbes. One failed read is shared by every
// caller it serves (see refreshTopologySince), so it must not be lost to a node
// the others could have come from.
func (c *ClusterClient) readTopology(ctx context.Context) error {
func (c *ClusterClient) readTopology(ctx context.Context, seq uint64) error {
var tried []*clusterNode
var lastErr error
for range topologyProbes {
Expand All @@ -580,7 +583,7 @@ func (c *ClusterClient) readTopology(ctx context.Context) error {
if v.Type != protocol.TyArray {
return fmt.Errorf("CLUSTER SLOTS: unexpected reply type %s", v.Type)
}
return c.applyTopology(ctx, v)
return c.applyTopology(ctx, v, seq)
}
if lastErr != nil {
return lastErr
Expand Down Expand Up @@ -611,14 +614,31 @@ func (c *ClusterClient) probeNode(tried []*clusterNode) *clusterNode {
// applyTopology rebuilds the slot-to-node mapping from a CLUSTER SLOTS reply.
// It is the part of refreshTopology that runs under c.mu, split out so a test
// can run it at the moment a Close has just returned.
func (c *ClusterClient) applyTopology(ctx context.Context, v *Value) error {
func (c *ClusterClient) applyTopology(ctx context.Context, v *Value, seq uint64) error {
c.mu.Lock()
defer c.mu.Unlock()
// closeNodes leaves c.nodes nil: a refresh whose reply came before Close
// gets here after it, and must neither dial nor write into the nil map.
if c.nodes == nil {
return ErrClosed
}
// The slot table is never replaced by a read registered before the one whose
// table is in place, whatever order they ended in: an older read that ends last
// cannot put a slot back where it was before a move (celeris#1219). seq is the
// read's registration number (topoRefresh.seq, 1 or more). A read that failed,
// or was cut short, never gets here and so does not advance topoApplied.
//
// What this does NOT guarantee: the order is that of registration, not of the
// requests reaching the server or of the cluster's truth, which a client cannot
// tell without a server epoch (CLUSTER SLOTS has none). A read registered later
// that was sent to a lagging node, or an older read that retried a second node
// after the newer one was sent, can still install the older table, and the
// fresher answer is dropped. The next -MOVED corrects it (one extra round), as
// would the 60 s refresh.
if seq < c.topoApplied {
return nil
}
c.topoApplied = seq

wantReplicas := c.cfg.ReadOnly || c.cfg.RouteByLatency

Expand Down
2 changes: 1 addition & 1 deletion driver/redis/cluster_close_refresh_1104_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func TestClusterRefreshAfterCloseDoesNotWriteToTheNodes1104(t *testing.T) {
for range iters {
cc, v, _ := closedClusterWithSlots1104(t)
var err error
if p := panicOf1104(func() { err = cc.applyTopology(context.Background(), v) }); p != nil {
if p := panicOf1104(func() { err = cc.applyTopology(context.Background(), v, 1) }); p != nil {
panics++
continue
}
Expand Down
5 changes: 5 additions & 0 deletions driver/redis/sentinel.go
Original file line number Diff line number Diff line change
Expand Up @@ -455,6 +455,11 @@ var ErrSentinelUnhealthy = errors.New("celeris-redis: sentinel client unhealthy,
// On dial failure, it retries 3 times with exponential backoff. If all retries
// fail, the client is marked unhealthy and returns errors until the next
// successful +switch-master.
//
// It is called only from watchFailover, which subscribeLoop runs inline, so the
// notifications are handled one at a time in the order they were delivered: a
// slow dial of an older failover cannot install its client after a newer one (the
// ordering problem of celeris#1219 does not exist on this path).
func (s *SentinelClient) handleSwitchMaster(payload string) {
// watchFailover's select between a queued message and closeCh picks at
// random, so this can run after Close: make no dial for a closed client.
Expand Down
114 changes: 114 additions & 0 deletions driver/redis/topology_order_1219_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
package redis

// celeris#1219: an older CLUSTER SLOTS read that ended AFTER a newer one overwrote
// the slot table with its older answer (applyTopology had no ordering): a node
// that learned of a move late could put the table back to the state before the
// move, and the commands then took an extra -MOVED round or a stale route until the
// next refresh. Found by the review of #1215.

import (
"context"
"testing"
)

// Read 1 takes its table before a slot moves and is held; read 2 starts after the
// move, ends and is applied; read 1 is released and ends last. The table must keep
// read 2's answer. (Main: read 1 overwrites it, the slot routes to the old owner.)
func TestAnOlderTopologyReadEndingLastDoesNotOverwriteANewerOne1219(t *testing.T) {
const key = "{order}k"
cl := newRPCluster(t, 3)
cc := newRPClusterClient(t, cl)
slot := int(Slot(key))
from := cl.owner(slot)
to := otherNode(cl, from)

waitHeld, release := holdNextSlotsRead(t)
r1 := make(chan error, 1)
go func() { r1 <- cc.refreshTopology(rpCtx(t)) }() // registered first, takes the pre-move table
waitHeld()
cl.moveSlot(slot, to, 0)
if err := cc.refreshTopology(rpCtx(t)); err != nil { // registered later, applied
t.Fatal(err)
}
if r := cc.route(Slot(key)); r == nil || r.addr != to.addr {
t.Fatal("the newer read did not put the slot on the new owner")
}
release()
if err := <-r1; err != nil {
t.Fatal(err)
}
if r := cc.route(Slot(key)); r == nil || r.addr != to.addr {
t.Errorf("the table went back to the pre-move owner: the older read, which ended last, overwrote the newer one")
}
}

// The order of registration decides, not the order of ending: reads that end in
// the order they started all apply (a refresh after a move updates the table), and
// so does a newer read after an older one that failed.
func TestTopologyReadsThatEndInOrderAllApply1219(t *testing.T) {
const key = "{inorder}k"
cl := newRPCluster(t, 3)
cc := newRPClusterClient(t, cl)
slot := int(Slot(key))
a := cl.owner(slot)
b := otherNode(cl, a)
ctx := rpCtx(t)
for i, to := range []*rpNode{b, a, b, a} {
cl.moveSlot(slot, to, 0)
if err := cc.refreshTopology(ctx); err != nil {
t.Fatal(err)
}
if r := cc.route(Slot(key)); r == nil || r.addr != to.addr {
t.Fatalf("move %d: after the refresh the slot does not route to its new owner", i)
}
}
// A read cut short by its caller's context applies nothing and does not
// advance the order: the next read still applies.
cl.moveSlot(slot, b, 0)
cctx, cancel := context.WithCancel(ctx)
waitHeld, release := holdNextSlotsRead(t)
r := make(chan error, 1)
go func() { r <- cc.refreshTopology(cctx) }()
waitHeld()
cancel()
t.Logf("the cancelled read: %v", <-r)
release()
if err := cc.refreshTopology(ctx); err != nil {
t.Fatal(err)
}
if rt := cc.route(Slot(key)); rt == nil || rt.addr != b.addr {
t.Error("a read after a cancelled one did not apply")
}
}

// An older read that carries the move is still in flight when a newer read fails
// (here: cut short by its caller). The failed read applies nothing, so it must not
// advance the order: the older read, which ends later, applies and the table gets
// the move. (A variant that advances the order when a read fails, or one that drops
// a result whenever a newer read is merely registered, loses the update.)
func TestAnOlderReadStillAppliesWhenTheNewerOneFailed1219(t *testing.T) {
const key = "{newerfails}k"
cl := newRPCluster(t, 3)
cc := newRPClusterClient(t, cl)
slot := int(Slot(key))
from := cl.owner(slot)
to := otherNode(cl, from)
cl.moveSlot(slot, to, 0) // the client's table still names from

waitHeld, release := holdNextSlotsRead(t)
r1 := make(chan error, 1)
go func() { r1 <- cc.refreshTopology(rpCtx(t)) }() // takes the post-move table, held
waitHeld()
cctx, cancel := context.WithCancel(context.Background())
cancel()
if err := cc.refreshTopology(cctx); err == nil { // registered later, cut short
t.Fatal("the cancelled read succeeded")
}
release()
if err := <-r1; err != nil {
t.Fatal(err)
}
if r := cc.route(Slot(key)); r == nil || r.addr != to.addr {
t.Errorf("the older read, which carries the move, was dropped because a newer read failed")
}
}
Loading