diff --git a/beacon/join.go b/beacon/join.go
index 41f4f5b..7792f28 100644
--- a/beacon/join.go
+++ b/beacon/join.go
@@ -151,7 +151,7 @@ func (b *Beacon) exchangeMembership(ctx context.Context, peers []cluster.NodeId)
return fmt.Errorf("membership tip %v unavailable", tip)
}
- wire, err := effects.MarshalEffect(eff)
+ wire, err := b.engine.EffectCache().MarshalEffect(eff)
if err != nil {
return err
}
diff --git a/cluster/ancestry.go b/cluster/ancestry.go
new file mode 100644
index 0000000..df19644
--- /dev/null
+++ b/cluster/ancestry.go
@@ -0,0 +1,281 @@
+package cluster
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log/slog"
+ "time"
+
+ "github.com/quic-go/quic-go"
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/effects"
+)
+
+const streamTypeAncestry byte = 0x05
+
+type ancestryReader interface {
+ ReadAncestry(string, effects.Tip, func(*pb.AncestryResponse) error) error
+}
+
+// ReadAncestry streams resident ancestry through its missing boundaries. No
+// network loads or frontier publication happen on the serving node. QUIC
+// backpressure bounds buffered bytes; each frame retains the normal size limit.
+func (r *EngineLogReader) ReadAncestry(key string, root effects.Tip, send func(*pb.AncestryResponse) error) error {
+ if r.effectCache == nil {
+ return fmt.Errorf("effect cache not initialized")
+ }
+
+ var sendErr error
+
+ count, bytes := 0, 0
+
+ r.effectCache.VisitAncestry(key, root, func(tip effects.Tip, eff *pb.Effect, isRoot bool) bool {
+ resp := &pb.AncestryResponse{Ref: &pb.EffectRef{NodeId: tip[0], Offset: tip[1]}, IsRoot: isRoot}
+ if eff != nil {
+ resp.EffectData, sendErr = MarshalEffectWire(eff, r.effectCache)
+ if sendErr != nil {
+ return false
+ }
+ }
+
+ if len(resp.GetEffectData()) > maxFrameSize-128 {
+ sendErr = fmt.Errorf("ancestry effect exceeds frame limit")
+
+ return false
+ }
+
+ sendErr = send(resp)
+ if sendErr == nil && eff != nil {
+ RecordFetchServed()
+ }
+
+ count++
+ bytes += len(resp.GetEffectData())
+
+ return sendErr == nil
+ })
+ slog.Debug("ancestry: served stream", "key", key, "root", root, "records", count, "bytes", bytes, "error", sendErr)
+
+ return sendErr
+}
+
+func handleAncestryStream(stream *quic.Stream, reader LogReader) {
+ defer stream.CancelRead(0)
+ // Bound a stalled reader without holding any node/publication locks.
+ if err := stream.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
+ return
+ }
+
+ req := &pb.AncestryRequest{}
+ if err := readMessage(stream, req); err != nil {
+ return
+ }
+
+ var err error
+ if source, ok := reader.(ancestryReader); ok && req.GetRoot() != nil && req.GetKey() != "" {
+ err = source.ReadAncestry(req.GetKey(), effects.Tip{req.GetRoot().GetNodeId(), req.GetRoot().GetOffset()},
+ func(resp *pb.AncestryResponse) error {
+ if err := stream.SetWriteDeadline(time.Now().Add(10 * time.Second)); err != nil {
+ return err
+ }
+
+ return writeMessage(stream, resp)
+ })
+ } else {
+ err = fmt.Errorf("resident ancestry reader unavailable or invalid request")
+ }
+
+ end := &pb.AncestryResponse{Done: true}
+ if err != nil {
+ end.Error = err.Error()
+ }
+
+ if err := writeMessage(stream, end); err != nil {
+ stream.CancelWrite(0)
+ }
+}
+
+func ancestryOverConn(ctx context.Context, conn *quic.Conn, key string, root effects.Tip,
+ have func(effects.Tip) bool, receive func(effects.Tip, []byte, bool) error,
+) error {
+ if conn == nil {
+ return ErrPeerUnavailable
+ }
+
+ stream, err := conn.OpenStreamSync(ctx)
+ if err != nil {
+ return err
+ }
+ defer stream.CancelRead(0)
+ defer stream.CancelWrite(0)
+
+ stop := context.AfterFunc(ctx, func() {
+ stream.CancelRead(0)
+ stream.CancelWrite(0)
+ })
+ defer stop()
+
+ if err := stream.SetWriteDeadline(time.Now().Add(10 * time.Second)); err != nil {
+ return err
+ }
+
+ if _, err := stream.Write([]byte{streamTypeAncestry}); err != nil {
+ return err
+ }
+
+ if err := writeMessage(stream, &pb.AncestryRequest{Key: key, Root: &pb.EffectRef{NodeId: root[0], Offset: root[1]}}); err != nil {
+ return err
+ }
+
+ if err := stream.Close(); err != nil {
+ return err
+ }
+
+ found, answered := false, false
+
+ var missing []effects.Tip
+
+ for count := 0; ; count++ {
+ if err := stream.SetReadDeadline(time.Now().Add(10 * time.Second)); err != nil {
+ return err
+ }
+
+ resp := &pb.AncestryResponse{}
+ if err := readMessage(stream, resp); err != nil {
+ return err
+ }
+
+ if resp.GetError() != "" {
+ return fmt.Errorf("ancestry source: %s", resp.GetError())
+ }
+
+ if resp.GetDone() {
+ if !answered {
+ return fmt.Errorf("ancestry source omitted root")
+ }
+
+ if !found {
+ return effects.ErrEffectNotFound
+ }
+
+ // Publish boundary evidence only after the stream finishes.
+ for _, tip := range missing {
+ if err := receive(tip, nil, false); err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }
+
+ if resp.GetRef() == nil {
+ return fmt.Errorf("ancestry source omitted effect identity")
+ }
+
+ tip := effects.Tip{resp.GetRef().GetNodeId(), resp.GetRef().GetOffset()}
+ if count == 0 && tip != root {
+ return fmt.Errorf("ancestry source returned a different root")
+ }
+
+ if tip == root {
+ answered, found = true, len(resp.GetEffectData()) != 0
+ }
+
+ if len(resp.GetEffectData()) == 0 {
+ missing = append(missing, tip)
+
+ continue
+ }
+
+ if have(tip) {
+ if resp.GetIsRoot() {
+ if err := receive(tip, nil, true); err != nil {
+ return err
+ }
+ }
+ // Residency does not prove that this vertex or its siblings have closure.
+ continue
+ }
+
+ if err := receive(tip, resp.GetEffectData(), resp.GetIsRoot()); err != nil {
+ return err
+ }
+ }
+}
+
+func (pm *PeerManager) ancestryConn(peer NodeId) *quic.Conn {
+ token := pm.mu.RLock()
+
+ var conn *quic.Conn
+ if pc := pm.peers[peer]; pc != nil {
+ conn = pc.GetQuicConn()
+ }
+
+ pm.mu.RUnlock(token)
+
+ if conn == nil || conn.Context().Err() != nil {
+ pm.inboundMu.RLock()
+ conn = pm.inboundConns[peer]
+ pm.inboundMu.RUnlock()
+ }
+
+ return conn
+}
+
+// FetchAncestry streams one selected holder's available closure, with one
+// fallback peer. Subscription replication reconciles further divergence.
+func (pm *PeerManager) FetchAncestry(key string, root effects.Tip, peers []pb.NodeID,
+ have func(effects.Tip) bool, receive func(effects.Tip, []byte, bool) error,
+) error {
+ var unavailable error
+
+ for _, peer := range peers[:min(2, len(peers))] {
+ err := ancestryOverConn(pm.ctx, pm.ancestryConn(peer), key, root, have, receive)
+ if err == nil {
+ return nil
+ }
+
+ if !errors.Is(err, effects.ErrEffectNotFound) {
+ unavailable = errors.Join(unavailable, fmt.Errorf("peer %d: %w", peer, err))
+ }
+ }
+
+ token := pm.mu.RLock()
+ cdn := pm.cdnFetcher
+ pm.mu.RUnlock(token)
+
+ if cdn != nil {
+ ctx, cancel := context.WithTimeout(pm.ctx, cloudBackstopTimeout)
+ defer cancel()
+
+ data, err := cdn.FetchFromCDN(ctx, &pb.EffectRef{NodeId: root[0], Offset: root[1]})
+ if err == nil && len(data) > 0 {
+ return receive(root, data, false)
+ }
+
+ if err != nil && !errors.Is(err, effects.ErrCDNBlobMissing) {
+ unavailable = errors.Join(unavailable, err)
+ }
+
+ if unavailable != nil {
+ return unavailable
+ }
+
+ if err != nil {
+ return err
+ }
+
+ return fmt.Errorf("CDN returned empty ancestry effect without absence evidence")
+ }
+
+ if unavailable != nil {
+ return unavailable
+ }
+
+ if len(peers) == 0 {
+ return ErrPeerUnavailable
+ }
+
+ return effects.ErrEffectNotFound
+}
diff --git a/cluster/ancestry_test.go b/cluster/ancestry_test.go
new file mode 100644
index 0000000..810bfde
--- /dev/null
+++ b/cluster/ancestry_test.go
@@ -0,0 +1,273 @@
+package cluster
+
+import (
+ "context"
+ "errors"
+ "io"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/quic-go/quic-go"
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/effects"
+)
+
+type countedAncestryReader struct {
+ *EngineLogReader
+
+ calls atomic.Int32
+}
+
+func (r *countedAncestryReader) ReadAncestry(key string, root effects.Tip, send func(*pb.AncestryResponse) error) error {
+ r.calls.Add(1)
+
+ return r.EngineLogReader.ReadAncestry(key, root, send)
+}
+
+func ancestryTestEngine(t *testing.T) *effects.Engine {
+ t.Helper()
+
+ e := effects.NewEngine(effects.EngineConfig{NodeID: 42})
+
+ t.Cleanup(func() { _ = e.Close() })
+
+ return e
+}
+
+func ancestryTestEffect(offset uint64) *pb.Effect {
+ eff := &pb.Effect{
+ Key: []byte("k"), NodeId: 42,
+ Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}},
+ }
+ if offset > 1 {
+ eff.Deps = []*pb.EffectRef{{NodeId: 42, Offset: offset - 1}}
+ }
+
+ return eff
+}
+
+func TestAncestryTransferHolderFallbackStreamingAndAbsence(t *testing.T) {
+ empty, holder := ancestryTestEngine(t), ancestryTestEngine(t)
+
+ for i := uint64(1); i <= 130; i++ {
+ eff := ancestryTestEffect(i)
+ if i == 1 {
+ eff.Deps = []*pb.EffectRef{{NodeId: 42, Offset: 999}}
+ }
+
+ if i == 100 {
+ eff.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}
+ }
+
+ if err := holder.EffectCache().Put(effects.Tip{42, i}, eff); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ readers := []*countedAncestryReader{
+ {EngineLogReader: NewEngineLogReader(empty.EffectCache())},
+ {EngineLogReader: NewEngineLogReader(empty.EffectCache())},
+ {EngineLogReader: NewEngineLogReader(holder.EffectCache())},
+ {EngineLogReader: NewEngineLogReader(holder.EffectCache())},
+ }
+ pms := startNNodeCluster(t, []EffectHandler{&recordingHandler{}, &recordingHandler{}, &recordingHandler{}, &recordingHandler{}},
+ []LogReader{readers[0], readers[1], readers[2], readers[3]})
+ local := map[effects.Tip]bool{}
+
+ var boundaries []effects.Tip
+
+ have := func(tip effects.Tip) bool { return local[tip] }
+ receive := func(tip effects.Tip, data []byte, isRoot bool) error {
+ if len(data) == 0 {
+ boundaries = append(boundaries, tip)
+
+ return nil
+ }
+
+ local[tip] = true
+
+ return nil
+ }
+ // First holder misses; the next serves multiple generations, crossing a
+ // snapshot. The unrelated fourth node must never be queried.
+ for _, root := range []uint64{130} {
+ if err := pms[0].FetchAncestry("k", effects.Tip{42, root}, []pb.NodeID{1, 2}, have, receive); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if len(local) != 130 || readers[1].calls.Load() != 1 || readers[2].calls.Load() != 1 || readers[3].calls.Load() != 0 {
+ t.Fatalf("effects=%d calls=%d/%d/%d", len(local), readers[1].calls.Load(), readers[2].calls.Load(), readers[3].calls.Load())
+ }
+
+ if len(boundaries) != 1 || boundaries[0] != (effects.Tip{42, 999}) {
+ t.Fatalf("boundaries = %v", boundaries)
+ }
+ // A resident vertex does not stop the stream: its dependencies may still
+ // be missing locally. All 130 records must arrive in one request.
+ clear(local)
+ local[effects.Tip{42, 125}] = true
+
+ if err := pms[0].FetchAncestry("k", effects.Tip{42, 130}, []pb.NodeID{2}, have, receive); err != nil {
+ t.Fatal(err)
+ }
+
+ if len(local) != 130 {
+ t.Fatalf("transfer stopped at local join: %d effects", len(local))
+ }
+
+ missing := effects.Tip{42, 999}
+ if err := pms[0].FetchAncestry("k", missing, []pb.NodeID{1, 2}, have, receive); !errors.Is(err, effects.ErrEffectNotFound) {
+ t.Fatal(err)
+ }
+ // A missing response cannot hide an unavailable holder, including when
+ // the CDN also responds with a missing-blob error.
+ pms[0].SetCDNFetcher(&recordingCDN{err: effects.ErrCDNBlobMissing})
+
+ err := pms[0].FetchAncestry("k", missing, []pb.NodeID{99, 1}, have, receive)
+ if err == nil || errors.Is(err, effects.ErrEffectNotFound) || errors.Is(err, effects.ErrCDNBlobMissing) {
+ t.Fatalf("unavailable holder masked: %v", err)
+ }
+}
+
+func TestAncestryResidentReaderScopesBindAndReportsBoundary(t *testing.T) {
+ e := ancestryTestEngine(t)
+ root := effects.Tip{42, 3}
+
+ bind := &pb.Effect{Key: []byte("k"), NodeId: 42, Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{
+ Keys: []*pb.TransactionalBindEffect_KeyBind{
+ {Key: []byte("k"), NewTip: &pb.EffectRef{NodeId: 42, Offset: 2}},
+ {Key: []byte("other"), NewTip: &pb.EffectRef{NodeId: 42, Offset: 50}},
+ },
+ }}}
+ if err := e.EffectCache().Put(root, bind); err != nil {
+ t.Fatal(err)
+ }
+
+ var records []*pb.AncestryResponse
+
+ err := NewEngineLogReader(e.EffectCache()).ReadAncestry("k", root, func(r *pb.AncestryResponse) error {
+ records = append(records, r)
+
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(records) != 2 || records[1].GetRef().GetOffset() != 2 || len(records[1].GetEffectData()) != 0 {
+ t.Fatalf("unexpected bind ancestry: %v", records)
+ }
+}
+
+func TestAncestryStreamCancellationAndTruncation(t *testing.T) {
+ tls := generateTestTLSConfig(t)
+ tls.NextProtos = []string{"swytch-cluster"}
+
+ listener, err := quic.ListenAddr("127.0.0.1:0", tls, clusterQUICConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Cleanup(func() { _ = listener.Close() })
+
+ ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second)
+ defer cancel()
+
+ client, err := quic.DialAddr(ctx, listener.Addr().String(), tls, clusterQUICConfig())
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Cleanup(func() { _ = client.CloseWithError(0, "done") })
+
+ server, err := listener.Accept(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ t.Cleanup(func() { _ = server.CloseWithError(0, "done") })
+
+ for _, mode := range []string{"cancel", "truncate", "complete", "resident-root"} {
+ requestCtx, cancelRequest := context.WithCancel(ctx)
+
+ var boundaries, roots atomic.Int32
+
+ done := make(chan error, 1)
+ go func() {
+ done <- ancestryOverConn(requestCtx, client, "k", effects.Tip{42, 1}, func(effects.Tip) bool { return mode == "resident-root" }, func(_ effects.Tip, data []byte, isRoot bool) error {
+ if isRoot {
+ roots.Add(1)
+ } else if len(data) == 0 {
+ boundaries.Add(1)
+ }
+
+ return nil
+ })
+ }()
+
+ stream, err := server.AcceptStream(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var prefix [1]byte
+ if _, err := io.ReadFull(stream, prefix[:]); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := readMessage(stream, &pb.AncestryRequest{}); err != nil {
+ t.Fatal(err)
+ }
+
+ if mode == "cancel" {
+ cancelRequest()
+ } else {
+ for _, resp := range []*pb.AncestryResponse{
+ {Ref: &pb.EffectRef{NodeId: 42, Offset: 1}, EffectData: []byte{1}},
+ {Ref: &pb.EffectRef{NodeId: 42, Offset: 2}, EffectData: []byte{1}, IsRoot: true},
+ {Ref: &pb.EffectRef{NodeId: 42, Offset: 3}},
+ } {
+ if err := writeMessage(stream, resp); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if mode == "complete" || mode == "resident-root" {
+ if err := writeMessage(stream, &pb.AncestryResponse{Done: true}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ // EOF without an explicit terminal record is not definitive absence.
+ if err := stream.Close(); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ select {
+ case err := <-done:
+ if mode == "complete" || mode == "resident-root" {
+ if err != nil {
+ t.Fatal(err)
+ }
+ } else if err == nil || errors.Is(err, effects.ErrEffectNotFound) {
+ t.Fatalf("incomplete stream classified as absence: %v", err)
+ }
+ case <-ctx.Done():
+ t.Fatal("stream did not cancel")
+ }
+
+ if mode == "complete" || mode == "resident-root" {
+ if boundaries.Load() != 1 || roots.Load() != 1 {
+ t.Fatalf("boundaries=%d roots=%d", boundaries.Load(), roots.Load())
+ }
+ } else if boundaries.Load() != 0 {
+ t.Fatal("incomplete stream published missing boundaries")
+ }
+
+ cancelRequest()
+ stream.CancelRead(0)
+ stream.CancelWrite(0)
+ }
+}
diff --git a/cluster/cloud_read.go b/cluster/cloud_read.go
index d56d2a8..2e2f664 100644
--- a/cluster/cloud_read.go
+++ b/cluster/cloud_read.go
@@ -48,7 +48,12 @@ func (cs *CloudSync) FetchFromCDN(ctx context.Context, ref *pb.EffectRef) ([]byt
return nil, err
}
- return MarshalEffectWire(eff)
+ var pool *effects.VertexPool
+ if cs.engine != nil {
+ pool = cs.engine.EffectCache()
+ }
+
+ return MarshalEffectWire(eff, pool)
}
// MayHold implements effects.CloudReader: the free, no-RPC filter gate the
@@ -124,10 +129,8 @@ func (cs *CloudSync) CloudTips(ctx context.Context, key string) ([]effects.Tip,
return tips, sidecar, missing, nil
}
-// MarkPending implements effects.CloudReader: it records a key whose Cloud
-// frontier has a durable hole (or whose read was served outbox-only during a
-// cloud outage), pending a re-consult by reconcileLoop. The caller logs the
-// reason; this just tracks it.
+// MarkPending schedules a re-consult when the outbox was served without a
+// complete GetTips response. It does not classify the graph's reconstructability.
func (cs *CloudSync) MarkPending(key string) {
cs.mu.Lock()
if cs.reconcile == nil {
@@ -197,12 +200,9 @@ func (cs *CloudSync) reconcileLoop() {
}
}
-// reconcileBatch re-consults GetTips for a batch of keys in one RPC and merges
-// each returned frontier into the DAG. A key is cleared from the pending set
-// only when the cloud gave a complete answer for it and its frontier installed
-// with nothing left pending; anything else (install error, or a durable hole
-// InstallCloudTips reports back as still-pending) leaves it pending for the
-// next tick. The whole RPC failing (cloud still unreachable) clears nothing.
+// reconcileBatch re-consults GetTips and ingests the returned closure. Transport
+// failures leave the key scheduled for retry. Definitive absence is an exhausted
+// closure boundary; whether a read needs that history belongs to reconstruction.
func (cs *CloudSync) reconcileBatch(keys []string) {
names := make([][]byte, len(keys))
@@ -235,19 +235,12 @@ func (cs *CloudSync) reconcileBatch(keys []string) {
continue
}
- pending, err := cs.engine.InstallCloudTips(key, tips, cs.appendSidecar(nil, kt), cloudTips(kt.GetMissing()))
- if err != nil {
+ if err := cs.engine.InstallCloudTips(key, tips, cs.appendSidecar(nil, kt)); err != nil {
slog.Warn("cloud reconcile: install failed; will retry", "key", key, "error", err)
continue
}
- if len(pending) > 0 {
- slog.Warn("cloud reconcile: frontier partially holed; leaving pending", "key", key, "pending", pending)
-
- continue
- }
-
slog.Info("cloud reconcile: merged cloud frontier", "key", key, "tips", len(tips))
cs.clearReconcile(key)
}
diff --git a/cluster/cloud_subscription_walk_test.go b/cluster/cloud_subscription_walk_test.go
new file mode 100644
index 0000000..7391784
--- /dev/null
+++ b/cluster/cloud_subscription_walk_test.go
@@ -0,0 +1,70 @@
+package cluster
+
+import (
+ "fmt"
+ "slices"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/effects"
+)
+
+func TestDiscoverySubscriptionObservationSnapshotBoundary(t *testing.T) {
+ for _, initialSubscription := range []bool{false, true} {
+ for _, reverseDeps := range []bool{false, true} {
+ for _, unsubscribe := range []bool{false, true} {
+ t.Run(fmt.Sprintf("initial-%t/reverse-%t/unsubscribe-%t", initialSubscription, reverseDeps, unsubscribe), func(t *testing.T) {
+ u, s, d, pruned := effects.Tip{7, 1}, effects.Tip{7, 2}, effects.Tip{7, 3}, effects.Tip{7, 99}
+ ref := func(tip effects.Tip) *pb.EffectRef { return &pb.EffectRef{NodeId: tip[0], Offset: tip[1]} }
+ data := func(n int64) *pb.DataEffect {
+ return &pb.DataEffect{
+ Op: pb.EffectOp_INSERT_OP, Merge: pb.MergeRule_ADDITIVE_INT,
+ Collection: pb.CollectionKind_SCALAR, Value: &pb.DataEffect_IntVal{IntVal: n},
+ }
+ }
+
+ sub := &pb.Effect{Kind: &pb.Effect_Subscription{Subscription: &pb.SubscriptionEffect{SubscriberNodeId: 7, Unsubscribe: unsubscribe}}}
+ if unsubscribe {
+ sub.Deps = []*pb.EffectRef{ref(s)}
+ }
+
+ child := &pb.Effect{Deps: []*pb.EffectRef{ref(s), ref(u)}, Kind: &pb.Effect_Data{Data: data(1)}}
+ if unsubscribe {
+ child.Deps = []*pb.EffectRef{ref(u)}
+ }
+
+ if reverseDeps {
+ slices.Reverse(child.GetDeps())
+ }
+
+ graph := map[effects.Tip]*pb.Effect{
+ u: sub,
+ s: {Deps: []*pb.EffectRef{ref(pruned)}, Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{
+ Op: pb.EffectOp_INSERT_OP, Merge: pb.MergeRule_ADDITIVE_INT,
+ Collection: pb.CollectionKind_SCALAR, Commutative: true, Scalar: data(10),
+ }}}},
+ d: child,
+ }
+
+ tips := []effects.Tip{d}
+ if initialSubscription {
+ tips = append(tips, u)
+ }
+
+ got, err := reduceDiscoveredTips(tips, func(tip effects.Tip) (*pb.Effect, error) {
+ if eff, ok := graph[tip]; ok {
+ return eff, nil
+ }
+
+ t.Errorf("requested discarded history %v", tip)
+
+ return nil, fmt.Errorf("history %v no longer exists", tip)
+ })
+ if err != nil || got.GetScalar().GetIntVal() != 11 {
+ t.Fatalf("state = %v, error = %v; want 11, nil", got, err)
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/cluster/cloud_sync.go b/cluster/cloud_sync.go
index 24daca7..d59ea18 100644
--- a/cluster/cloud_sync.go
+++ b/cluster/cloud_sync.go
@@ -777,7 +777,12 @@ func (f *ownFilter) has(h uint64) bool {
// — so the cloud can bill the customer's data rather than the sealed blob it
// stores or the writer's compression choices.
func (cs *CloudSync) buildEnvelope(tip effects.Tip, eff *pb.Effect) (*dp.Effect, error) {
- raw, err := effects.MarshalEffect(eff)
+ var pool *effects.VertexPool
+ if cs.engine != nil {
+ pool = cs.engine.EffectCache()
+ }
+
+ raw, err := pool.MarshalEffect(eff)
if err != nil {
return nil, fmt.Errorf("marshal: %w", err)
}
diff --git a/cluster/cloud_sync_test.go b/cluster/cloud_sync_test.go
index 633b40c..3b30b7b 100644
--- a/cluster/cloud_sync_test.go
+++ b/cluster/cloud_sync_test.go
@@ -725,12 +725,9 @@ func (c *sequencedTipsClient) GetTips(context.Context, *dp.GetTipsRequest, ...gr
return resp, nil
}
-// TestReconcileKeepsKeyPendingUntilHoleHeals: a key with a durable hole in its
-// Cloud frontier must stay in the reconcile set across a tick that still
-// reports the hole, and only clear once Cloud's own billing sweep prunes the
-// orphaned marker and a later tick sees nothing left to reconcile — reconcile
-// never repairs the hole itself, it only waits it out.
-func TestReconcileKeepsKeyPendingUntilHoleHeals(t *testing.T) {
+// With no fetch source, closure cannot establish availability. Keep the key
+// scheduled until GetTips supplies a complete response that requires no fetch.
+func TestReconcileRetriesUnavailableClosure(t *testing.T) {
engine := effects.NewEngine(effects.EngineConfig{NodeID: 1})
defer func() {
if err := engine.Close(); err != nil {
@@ -757,13 +754,13 @@ func TestReconcileKeepsKeyPendingUntilHoleHeals(t *testing.T) {
cs.reconcileBatch([]string{"k"})
if got := cs.reconcileKeys(); !slices.Equal(got, []string{"k"}) {
- t.Fatalf("reconcile set after a still-holed tick = %v, want [k]", got)
+ t.Fatalf("reconcile set after an unavailable closure = %v, want [k]", got)
}
cs.reconcileBatch([]string{"k"})
if got := cs.reconcileKeys(); got != nil {
- t.Fatalf("reconcile set after the hole healed = %v, want empty", got)
+ t.Fatalf("reconcile set after a complete empty response = %v, want empty", got)
}
}
diff --git a/cluster/engine_glue.go b/cluster/engine_glue.go
index 66551b4..d12a36f 100644
--- a/cluster/engine_glue.go
+++ b/cluster/engine_glue.go
@@ -48,6 +48,14 @@ func NewEngineEffectHandler(engine *effects.Engine) *EngineEffectHandler {
return &EngineEffectHandler{engine: engine}
}
+func (h *EngineEffectHandler) ConstrainHistoryProgress(peer NodeId, my, your uint64) (uint64, uint64) {
+ if h.engine == nil {
+ return 0, 0
+ }
+
+ return h.engine.ConstrainHistoryProgress(peer, my, your)
+}
+
// HandleOffsetNotify delegates to the engine's HandleRemote. The
// engine handles log storage, keytrie update, cache invalidation, NACK
// generation, and notification callbacks.
@@ -110,15 +118,20 @@ func (r *EngineLogReader) ReadEffect(ref *pb.EffectRef) ([]byte, error) {
return nil, fmt.Errorf("effect not found at %v", ref)
}
- return MarshalEffectWire(eff)
+ return MarshalEffectWire(eff, r.effectCache)
}
// MarshalEffectWire serializes an Effect to the peer-fetch wire format:
// [4 bytes LE keyLen][key][proto-marshalled effect]. Exported so external
// durability layers (the cloud dataplane's internal store) can produce blobs
// the engine's fetch path parses directly.
-func MarshalEffectWire(eff *pb.Effect) ([]byte, error) {
- protoData, err := effects.MarshalEffect(eff)
+func MarshalEffectWire(eff *pb.Effect, pools ...*effects.VertexPool) ([]byte, error) {
+ var pool *effects.VertexPool
+ if len(pools) != 0 {
+ pool = pools[0]
+ }
+
+ protoData, err := pool.MarshalEffect(eff)
if err != nil {
return nil, err
}
diff --git a/cluster/evidence_test.go b/cluster/evidence_test.go
new file mode 100644
index 0000000..bafdeb3
--- /dev/null
+++ b/cluster/evidence_test.go
@@ -0,0 +1,110 @@
+package cluster
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "io"
+ "net/http"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/effects"
+ "google.golang.org/protobuf/proto"
+)
+
+// The proof's ordinary ancestry is deliberately unavailable. Neither peer
+// streaming nor cloud transport may try to reconstruct that foreign key.
+func TestSnapshotEvidencePeerAndCloudTransport(t *testing.T) {
+ source := ancestryTestEngine(t)
+ member, owner := effects.Tip{42, 1}, effects.Tip{42, 2}
+ ref := func(t effects.Tip) *pb.EffectRef { return &pb.EffectRef{NodeId: t[0], Offset: t[1]} }
+
+ proof := &pb.Effect{Key: []byte("foreign"), Deps: []*pb.EffectRef{{NodeId: 99, Offset: 99}}, Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}}}
+ if err := source.EffectCache().Put(member, proof); err != nil {
+ t.Fatal(err)
+ }
+
+ snapshot := &pb.Effect{Key: []byte("k"), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{Evidence: &pb.BindEvidence{Deps: []*pb.EffectRef{ref(member)}}}}}
+ if err := source.EffectCache().Put(owner, snapshot); err != nil {
+ t.Fatal(err)
+ }
+
+ verify := func(t *testing.T, data []byte) {
+ t.Helper()
+
+ keyLen := int(binary.LittleEndian.Uint32(data))
+
+ wire := &pb.Effect{}
+ if err := effects.UnmarshalEffect(data[4+keyLen:], wire); err != nil {
+ t.Fatal(err)
+ }
+
+ if len(wire.GetEvidencePackage()) != 1 {
+ t.Fatal("proof omitted from wire package")
+ }
+
+ target := ancestryTestEngine(t)
+ if err := target.EffectCache().Put(owner, wire); err != nil {
+ t.Fatal(err)
+ }
+
+ stored, ok := target.EffectCache().Get(member)
+ if !ok || !proto.Equal(stored, proof) {
+ t.Fatal("proof payload changed in transit")
+ }
+ }
+ reader := NewEngineLogReader(source.EffectCache())
+
+ t.Run("fetch", func(t *testing.T) {
+ data, err := reader.ReadEffect(ref(owner))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ verify(t, data)
+ })
+ t.Run("ancestry", func(t *testing.T) {
+ count := 0
+
+ err := reader.ReadAncestry("k", owner, func(record *pb.AncestryResponse) error {
+ count++
+
+ verify(t, record.GetEffectData())
+
+ return nil
+ })
+ if err != nil || count != 1 {
+ t.Fatalf("stream walked proof ancestry: count=%d error=%v", count, err)
+ }
+ })
+ t.Run("cloud", func(t *testing.T) {
+ enc, err := effects.NewEncryptorFromIKM([]byte("snapshot-proof-test"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ uploader := &CloudSync{engine: source, enc: enc, keyNameKey: DeriveKeyNameKey("snapshot-proof-test")}
+
+ env, err := uploader.buildEnvelope(owner, snapshot)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ data, err := proto.Marshal(env)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ downloader := &CloudSync{enc: enc, httpClient: &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
+ return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(data))}, nil
+ })}}
+
+ wire, err := downloader.FetchFromCDN(context.Background(), ref(owner))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ verify(t, wire)
+ })
+}
diff --git a/cluster/fetch_exhaustion_test.go b/cluster/fetch_exhaustion_test.go
new file mode 100644
index 0000000..814941f
--- /dev/null
+++ b/cluster/fetch_exhaustion_test.go
@@ -0,0 +1,40 @@
+package cluster
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/effects"
+)
+
+func TestFetchExhaustionPreservesUnreachableHolder(t *testing.T) {
+ pms := startNNodeCluster(t,
+ []EffectHandler{&recordingHandler{}, &recordingHandler{}, &recordingHandler{}},
+ []LogReader{newStaticLogReader(), newStaticLogReader(), newStaticLogReader()},
+ )
+
+ conns := pms[0].fetchConns()
+ if len(conns) < 2 {
+ t.Fatal("test requires multiple peer connections")
+ }
+
+ ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+ defer cancel()
+
+ ref := &pb.EffectRef{NodeId: 42, Offset: 99}
+ if _, err := pms[0].fetchFromConns(ctx, ref, conns); !errors.Is(err, effects.ErrEffectNotFound) {
+ t.Fatalf("reachable holders with no effect: %v", err)
+ }
+
+ if err := conns[0].CloseWithError(0, "test unavailable holder"); err != nil {
+ t.Fatal(err)
+ }
+ // The closed connection fails immediately; the surviving peers reply
+ // missing afterward. Arrival order must not turn that into exhaustion.
+ if _, err := pms[0].fetchFromConns(ctx, ref, conns); err == nil || errors.Is(err, effects.ErrEffectNotFound) {
+ t.Fatalf("unreachable holder classified as missing: %v", err)
+ }
+}
diff --git a/cluster/fetch_order_test.go b/cluster/fetch_order_test.go
index 866cbe6..2f873cb 100644
--- a/cluster/fetch_order_test.go
+++ b/cluster/fetch_order_test.go
@@ -174,7 +174,7 @@ func TestFetchFromAnyPreferPeersMissStillTriesCDN(t *testing.T) {
// TestFetchFromAnyCDN404NotMaskedByPeerError: under PreferPeers, when both
// legs fail, a durable CDN 404 must still surface as ErrCDNBlobMissing —
// not be swallowed by whichever peer error happened to run first — since
-// classifyCloudTips keys its pending-vs-install split on errors.Is against
+// Closure distinguishes definitive absence using errors.Is against
// the sentinel. The peer error's text must still be present for diagnosis.
func TestFetchFromAnyCDN404NotMaskedByPeerError(t *testing.T) {
pm := newFetchTestPM(t)
diff --git a/cluster/heartbeat.go b/cluster/heartbeat.go
index b46b62a..c0ff8cc 100644
--- a/cluster/heartbeat.go
+++ b/cluster/heartbeat.go
@@ -54,12 +54,14 @@ const (
livenessCheckFloor = 10 * time.Millisecond
livenessCheckCeil = 500 * time.Millisecond
- clockTickVersion byte = 1
+ clockTickVersion byte = 2
+
+ legacyTickHeaderSize = 36
// tickHeaderSize = type(1) + version(1) + nodeID(8) + seq(8) +
- // wallClock(8) + jumpNs(8) + count(2) = 36 bytes.
+ // wallClock(8) + jumpNs(8) + count(2) + effect counters(16) = 52 bytes.
// (Noise AEAD tag is added externally by the transport layer.)
- tickHeaderSize = 36
+ tickHeaderSize = 52
// observedTickSize = node(8) + seq(8) + observedAt(8) = 24 bytes.
observedTickSize = 24
@@ -92,6 +94,10 @@ type ObservedTick struct {
// event. The implicit edge to the node's previous tick is Seq−1; the
// cross-node edges are the Observed entries. See docs/design/clock.md.
type ClockTick struct {
+ // Received wire version, used to select a compatible reply format.
+ // Zero denotes a locally constructed tick using the current version.
+ version byte
+
NodeID NodeId
Seq uint64
WallClock int64 // sender's wall clock, ns since epoch
@@ -101,6 +107,12 @@ type ClockTick struct {
// origin: its wall clock stepped, so accumulated skew samples are stale.
JumpNs int64
+ // Effect counters are per recipient and independent of clock measurements.
+ // MyCount includes relevant local publications while the peer was away.
+ // YourCount names an effect from the recipient whose ancestry we processed;
+ // it is a witness, not a promise of delivery on unrelated keys below it.
+ MyCount, YourCount uint64
+
Observed []ObservedTick
}
@@ -127,6 +139,9 @@ type HeartbeatManager struct {
// sendFunc transmits a tick packet to a peer via the QUIC transport.
sendFunc func(peerID NodeId, data []byte) error
+ // Installed before Start. The engine applies transaction overlays without
+ // changing the authoritative counters held by peer health.
+ constrainProgress func(NodeId, uint64, uint64) (uint64, uint64)
cancel chan struct{}
wg sync.WaitGroup
@@ -204,6 +219,15 @@ func (hm *HeartbeatManager) SendTickTo(peerID NodeId) {
hm.ownTicks.put(tick.Seq, tick.WallClock)
hm.mu.Unlock()
+ if ph := hm.healthTable.Get(peerID); ph != nil {
+ tick.MyCount, tick.YourCount = ph.sent.Load(), ph.processed.Load()
+ tick.version = byte(ph.tickVersion.Load())
+ }
+
+ if hm.constrainProgress != nil {
+ tick.MyCount, tick.YourCount = hm.constrainProgress(peerID, tick.MyCount, tick.YourCount)
+ }
+
if err := hm.sendFunc(peerID, MarshalClockTick(tick)); err != nil {
slog.Debug("immediate clock tick send failed", "peer", peerID, "error", err)
}
@@ -229,7 +253,16 @@ func (hm *HeartbeatManager) ProcessInboundTick(peerID NodeId, tick *ClockTick) {
}
now := time.Now()
+
ph := hm.healthTable.GetOrCreate(peerID)
+ for old := ph.tickVersion.Load(); uint32(tick.version) > old; old = ph.tickVersion.Load() {
+ if ph.tickVersion.CompareAndSwap(old, uint32(tick.version)) {
+ break
+ }
+ }
+
+ advanceEffectCounter(&ph.peerSent, tick.MyCount)
+ advanceEffectCounter(&ph.peerProcessed, tick.YourCount)
RecordClockTickReceived(peerID)
if tick.JumpNs != 0 {
@@ -368,7 +401,8 @@ func (hm *HeartbeatManager) livenessCadence() time.Duration {
// sendTick emits the next clock tick to every registered peer: one sequence
// number, one wall-clock reading, the drained observation set, marshaled
-// once and fanned out — every peer sees the same clock-DAG entry.
+// once, then extended with each recipient's effect counters. Every peer sees
+// the same clock-DAG entry, but its own replication evidence.
func (hm *HeartbeatManager) sendTick() {
now := time.Now()
wall := now.UnixNano()
@@ -413,7 +447,23 @@ func (hm *HeartbeatManager) sendTick() {
}
for _, peerID := range peerIDs {
- if err := hm.sendFunc(peerID, packet); err != nil {
+ peerPacket := slices.Clone(packet)
+
+ if ph := hm.healthTable.Get(peerID); ph != nil {
+ my, your := ph.sent.Load(), ph.processed.Load()
+ if hm.constrainProgress != nil {
+ my, your = hm.constrainProgress(peerID, my, your)
+ }
+
+ binary.LittleEndian.PutUint64(peerPacket[36:44], my)
+ binary.LittleEndian.PutUint64(peerPacket[44:52], your)
+
+ if ph.tickVersion.Load() == 1 {
+ peerPacket = legacyClockTick(peerPacket)
+ }
+ }
+
+ if err := hm.sendFunc(peerID, peerPacket); err != nil {
slog.Debug("clock tick send failed", "peer", peerID, "error", err)
continue
@@ -520,7 +570,7 @@ func (r *tickRing) get(seq uint64) (int64, bool) {
// MarshalClockTick builds a clock tick plaintext body:
// [1:type][1:version][8:nodeID LE][8:seq LE][8:wallClock LE][8:jumpNs LE]
-// [2:count LE] + count × [8:node LE][8:seq LE][8:observedAt LE]
+// [2:count LE][8:myCount LE][8:yourCount LE] + count × [8:node LE][8:seq LE][8:observedAt LE]
// Encryption (Noise AEAD) is applied by the transport layer.
func MarshalClockTick(tick *ClockTick) []byte {
buf := make([]byte, tickHeaderSize+len(tick.Observed)*observedTickSize)
@@ -531,6 +581,8 @@ func MarshalClockTick(tick *ClockTick) []byte {
binary.LittleEndian.PutUint64(buf[18:26], uint64(tick.WallClock))
binary.LittleEndian.PutUint64(buf[26:34], uint64(tick.JumpNs))
binary.LittleEndian.PutUint16(buf[34:36], uint16(len(tick.Observed)))
+ binary.LittleEndian.PutUint64(buf[36:44], tick.MyCount)
+ binary.LittleEndian.PutUint64(buf[44:52], tick.YourCount)
off := tickHeaderSize
for _, obs := range tick.Observed {
@@ -540,35 +592,61 @@ func MarshalClockTick(tick *ClockTick) []byte {
off += observedTickSize
}
+ if tick.version == 1 {
+ return legacyClockTick(buf)
+ }
+
return buf
}
+// legacyClockTick removes the v2 counters from an exclusively owned packet.
+// Unknown peers initially receive v2. A legacy peer always emits v1, so its
+// first inbound tick selects v1 for subsequent sends in both directions.
+// Receiving v1 carries no reclamation evidence; zero counters remain zero.
+func legacyClockTick(packet []byte) []byte {
+ packet[1] = 1
+ copy(packet[legacyTickHeaderSize:], packet[tickHeaderSize:])
+
+ return packet[:len(packet)-(tickHeaderSize-legacyTickHeaderSize)]
+}
+
// ParseClockTick parses a clock tick plaintext body (already Noise-decrypted).
func ParseClockTick(body []byte) (*ClockTick, error) {
- if len(body) < tickHeaderSize {
- return nil, fmt.Errorf("clock tick too short: %d < %d", len(body), tickHeaderSize)
+ if len(body) < legacyTickHeaderSize {
+ return nil, fmt.Errorf("clock tick too short: %d < %d", len(body), legacyTickHeaderSize)
}
- if body[1] != clockTickVersion {
+ if body[1] != clockTickVersion && body[1] != 1 {
return nil, fmt.Errorf("unsupported clock tick version: %d", body[1])
}
+ headerSize := tickHeaderSize
+ if body[1] == 1 {
+ headerSize = legacyTickHeaderSize // Legacy peers supply no reclamation evidence.
+ }
+
count := int(binary.LittleEndian.Uint16(body[34:36]))
- if want := tickHeaderSize + count*observedTickSize; len(body) != want {
+ if want := headerSize + count*observedTickSize; len(body) != want {
return nil, fmt.Errorf("clock tick length %d does not match %d observations (want %d)",
len(body), count, want)
}
tick := &ClockTick{
+ version: body[1],
NodeID: NodeId(binary.LittleEndian.Uint64(body[2:10])),
Seq: binary.LittleEndian.Uint64(body[10:18]),
WallClock: int64(binary.LittleEndian.Uint64(body[18:26])),
JumpNs: int64(binary.LittleEndian.Uint64(body[26:34])),
}
+ if body[1] == clockTickVersion {
+ tick.MyCount = binary.LittleEndian.Uint64(body[36:44])
+ tick.YourCount = binary.LittleEndian.Uint64(body[44:52])
+ }
+
if count > 0 {
tick.Observed = make([]ObservedTick, count)
- off := tickHeaderSize
+ off := headerSize
for i := range tick.Observed {
tick.Observed[i] = ObservedTick{
NodeID: NodeId(binary.LittleEndian.Uint64(body[off : off+8])),
diff --git a/cluster/heartbeat_compat_test.go b/cluster/heartbeat_compat_test.go
new file mode 100644
index 0000000..08ce2ee
--- /dev/null
+++ b/cluster/heartbeat_compat_test.go
@@ -0,0 +1,140 @@
+package cluster
+
+import (
+ "encoding/binary"
+ "fmt"
+ "slices"
+ "testing"
+ "time"
+)
+
+// Keep this decoder independent of ParseClockTick: the old receiver rejects
+// v2 before recording liveness, and requires the exact v1 observation layout.
+func parseV1TickForTest(body []byte) (*ClockTick, error) {
+ if len(body) < 36 || body[1] != 1 {
+ return nil, fmt.Errorf("not a version-1 tick")
+ }
+
+ count := int(binary.LittleEndian.Uint16(body[34:36]))
+ if len(body) != 36+24*count {
+ return nil, fmt.Errorf("invalid version-1 tick length")
+ }
+
+ tick := &ClockTick{
+ NodeID: NodeId(binary.LittleEndian.Uint64(body[2:10])),
+ Seq: binary.LittleEndian.Uint64(body[10:18]),
+ WallClock: int64(binary.LittleEndian.Uint64(body[18:26])),
+ JumpNs: int64(binary.LittleEndian.Uint64(body[26:34])),
+ }
+ for off := 36; off < len(body); off += 24 {
+ tick.Observed = append(tick.Observed, ObservedTick{
+ NodeID: NodeId(binary.LittleEndian.Uint64(body[off : off+8])),
+ Seq: binary.LittleEndian.Uint64(body[off+8 : off+16]),
+ ObservedAt: int64(binary.LittleEndian.Uint64(body[off+16 : off+24])),
+ })
+ }
+
+ return tick, nil
+}
+
+func TestHeartbeatMixedVersionRollout(t *testing.T) {
+ health := NewPeerHealthTable(ClockConfig{})
+ hm := NewHeartbeatManager(1, health, ClockConfig{})
+ legacyHealth := NewPeerHealthTable(ClockConfig{})
+ legacy := NewHeartbeatManager(2, legacyHealth, ClockConfig{})
+
+ hm.AddPeer(2, "")
+ hm.AddPeer(3, "")
+ legacy.AddPeer(1, "")
+
+ // A real v1 packet, constructed without the upgraded marshaler.
+ v1 := make([]byte, 36)
+ v1[0], v1[1] = PacketTypeClockTick, 1
+ binary.LittleEndian.PutUint64(v1[2:10], 2)
+ binary.LittleEndian.PutUint64(v1[10:18], 7)
+ binary.LittleEndian.PutUint64(v1[18:26], uint64(time.Now().UnixNano()))
+
+ tick, err := ParseClockTick(v1)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ hm.ProcessInboundTick(2, tick)
+
+ v2, err := ParseClockTick(MarshalClockTick(&ClockTick{NodeID: 3, Seq: 9, WallClock: time.Now().UnixNano(), MyCount: 30, YourCount: 10}))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ hm.ProcessInboundTick(3, v2)
+
+ for _, id := range []NodeId{2, 3} {
+ health.Get(id).sent.Store(12)
+ health.Get(id).processed.Store(24)
+ }
+
+ var (
+ retained [][]byte
+ legacyTicks []*ClockTick
+ )
+
+ hm.sendFunc = func(id NodeId, packet []byte) error {
+ retained = append(retained, packet)
+ if id == 2 {
+ got, err := parseV1TickForTest(packet)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ legacyTicks = append(legacyTicks, got)
+ legacy.ProcessInboundTick(1, got)
+ } else {
+ got, err := ParseClockTick(packet)
+ if err != nil || packet[1] != 2 || got.MyCount != 12 || got.YourCount != 24 {
+ t.Fatalf("v2 peer lost progress: %+v, %v", got, err)
+ }
+ }
+
+ return nil
+ }
+ hm.sendTick()
+ hm.SendTickTo(2)
+ hm.SendTickTo(3)
+ hm.sendTick()
+
+ if len(legacyTicks) != 3 || len(legacyTicks[0].Observed) != 2 || legacyTicks[0].Observed[0].Seq != 7 || legacyTicks[0].Observed[1].Seq != 9 {
+ t.Fatalf("legacy observations changed: %+v", legacyTicks)
+ }
+
+ if len(legacyTicks[1].Observed) != 0 || len(legacyTicks[2].Observed) != 0 {
+ t.Fatal("immediate or drained tick included stale observations")
+ }
+
+ legacyHealth.CheckLiveness()
+
+ if got := legacyHealth.AliveSymmetricPeers("", map[NodeId]string{1: ""}); !slices.Equal(got, []NodeId{1}) {
+ t.Fatalf("legacy peer excluded upgraded replication target: %v", got)
+ }
+
+ if health.Get(2).peerSent.Load() != 0 || health.Get(2).peerProcessed.Load() != 0 {
+ t.Fatal("legacy ticks supplied reclamation evidence")
+ }
+
+ if _, err := parseV1TickForTest(retained[0]); err != nil {
+ t.Fatalf("subsequent sends mutated a retained legacy packet: %v", err)
+ }
+
+ // Capability must not regress on delayed v1 packets after learning v2.
+ hm.ProcessInboundTick(3, &ClockTick{version: 1, NodeID: 3, Seq: 1})
+
+ if health.Get(3).tickVersion.Load() != 2 {
+ t.Fatal("delayed legacy tick regressed peer capability")
+ }
+
+ hm.RemovePeer(2)
+ hm.AddPeer(2, "")
+
+ if health.Get(2).tickVersion.Load() != 0 {
+ t.Fatal("removed peer retained wire version")
+ }
+}
diff --git a/cluster/heartbeat_progress_test.go b/cluster/heartbeat_progress_test.go
new file mode 100644
index 0000000..88fe9f0
--- /dev/null
+++ b/cluster/heartbeat_progress_test.go
@@ -0,0 +1,104 @@
+package cluster
+
+import (
+ "testing"
+)
+
+func TestHeartbeatEffectCountersArePerRecipient(t *testing.T) {
+ table := NewPeerHealthTable(ClockConfig{})
+ hm := NewHeartbeatManager(1, table, ClockConfig{})
+ hm.AddPeer(2, "")
+ hm.AddPeer(3, "")
+ table.Get(2).sent.Store(7)
+ table.Get(2).processed.Store(22)
+ table.Get(3).sent.Store(8)
+ table.Get(3).processed.Store(17)
+
+ packets := make(map[NodeId][]byte)
+ hm.sendFunc = func(peer NodeId, packet []byte) error {
+ packets[peer] = packet // The transport may retain each packet.
+
+ return nil
+ }
+
+ for _, immediate := range []bool{false, true} {
+ if immediate {
+ hm.SendTickTo(2)
+ hm.SendTickTo(3)
+ } else {
+ hm.sendTick()
+ }
+
+ for peer, want := range map[NodeId][2]uint64{2: {7, 22}, 3: {8, 17}} {
+ tick, err := ParseClockTick(packets[peer])
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if tick.MyCount != want[0] || tick.YourCount != want[1] {
+ t.Fatalf("peer %d: counts=(%d,%d), want %v", peer, tick.MyCount, tick.YourCount, want)
+ }
+ }
+ }
+}
+
+func TestHeartbeatEffectEvidenceDoesNotRegress(t *testing.T) {
+ table := NewPeerHealthTable(ClockConfig{})
+ hm := NewHeartbeatManager(1, table, ClockConfig{})
+ hm.AddPeer(2, "")
+ hm.ProcessInboundTick(2, &ClockTick{NodeID: 2, Seq: 2, MyCount: 22, YourCount: 7})
+ hm.ProcessInboundTick(2, &ClockTick{NodeID: 2, Seq: 1, MyCount: 17, YourCount: 5})
+
+ peer := table.Get(2)
+ if peer.peerSent.Load() != 22 || peer.peerProcessed.Load() != 7 {
+ t.Fatal("a delayed heartbeat regressed effect evidence")
+ }
+
+ hm.RemovePeer(2)
+ hm.AddPeer(3, "")
+
+ if table.Get(2) != nil || table.Get(3).peerSent.Load() != 0 || table.Get(3).peerProcessed.Load() != 0 {
+ t.Fatal("a new incarnation inherited the departed peer's evidence")
+ }
+}
+
+func TestHeartbeatAppliesOverlayWithoutChangingTruth(t *testing.T) {
+ table := NewPeerHealthTable(ClockConfig{})
+ hm := NewHeartbeatManager(1, table, ClockConfig{})
+ hm.AddPeer(2, "")
+
+ peer := table.Get(2)
+ peer.sent.Store(12)
+ peer.processed.Store(30)
+
+ hm.constrainProgress = func(id NodeId, my, your uint64) (uint64, uint64) {
+ if id != 2 || my != 12 || your != 30 {
+ t.Fatal("overlay did not receive the authoritative peer counters")
+ }
+
+ return 7, 17
+ }
+
+ var sent int
+
+ hm.sendFunc = func(_ NodeId, data []byte) error {
+ tick, err := ParseClockTick(data)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if tick.MyCount != 7 || tick.YourCount != 17 {
+ t.Fatalf("heartbeat ignored overlay: %+v", tick)
+ }
+
+ sent++
+
+ return nil
+ }
+ hm.sendTick()
+ hm.SendTickTo(2)
+
+ if sent != 2 || peer.sent.Load() != 12 || peer.processed.Load() != 30 {
+ t.Fatal("overlay changed authoritative progress")
+ }
+}
diff --git a/cluster/peer_health.go b/cluster/peer_health.go
index 4fa0373..3219af3 100644
--- a/cluster/peer_health.go
+++ b/cluster/peer_health.go
@@ -42,6 +42,24 @@ type PeerHealth struct {
// clock holds the per-peer measurement windows (offsets, inter-arrival)
// and derives RTT_min / skew / inter-arrival statistics from them.
clock *peerClockStats
+
+ // Wire version learned from inbound ticks. Unknown peers are probed with
+ // the current version; v1 peers select the legacy layout on first contact.
+ // Membership owns this state so a new incarnation negotiates independently.
+ tickVersion atomic.Uint32
+
+ // Each direction has the two counters carried by a heartbeat. Membership
+ // owns their lifetime; a new node identity starts without prior evidence.
+ sent, processed atomic.Uint64
+ peerSent, peerProcessed atomic.Uint64
+}
+
+func advanceEffectCounter(counter *atomic.Uint64, value uint64) {
+ for old := counter.Load(); value > old; old = counter.Load() {
+ if counter.CompareAndSwap(old, value) {
+ return
+ }
+ }
}
// IsReplicationTarget returns true if this peer is both alive and has a
diff --git a/cluster/peer_manager.go b/cluster/peer_manager.go
index 6a5ee09..285cfce 100644
--- a/cluster/peer_manager.go
+++ b/cluster/peer_manager.go
@@ -252,6 +252,12 @@ func (pm *PeerManager) Start(ctx context.Context) error {
// Start heartbeat manager
pm.heartbeat = NewHeartbeatManager(pm.config.NodeID, pm.healthTable, pm.config.Clock)
+ if handler, ok := pm.handler.(interface {
+ ConstrainHistoryProgress(NodeId, uint64, uint64) (uint64, uint64)
+ }); ok {
+ pm.heartbeat.constrainProgress = handler.ConstrainHistoryProgress
+ }
+
pm.heartbeat.Start(func(peerID NodeId, data []byte) error {
return pm.quicTransport.SendDirect(peerID, data)
})
@@ -523,6 +529,35 @@ func (pm *PeerManager) PeerIDs() []NodeId {
return ids
}
+func (pm *PeerManager) HistoryProgress() []effects.HistoryProgress {
+ peers := pm.PeerIDs()
+
+ result := make([]effects.HistoryProgress, 0, len(peers))
+ for _, peer := range peers {
+ p := effects.HistoryProgress{Node: uint64(peer)}
+ if ph := pm.healthTable.Get(peer); ph != nil {
+ p.Minted, p.Processed = ph.peerSent.Load(), ph.peerProcessed.Load()
+ p.Received = ph.processed.Load()
+ }
+
+ result = append(result, p)
+ }
+
+ return result
+}
+
+func (pm *PeerManager) RecordHistorySent(peer NodeId, counter uint64) {
+ if ph := pm.healthTable.Get(peer); ph != nil {
+ advanceEffectCounter(&ph.sent, counter)
+ }
+}
+
+func (pm *PeerManager) RecordHistoryProcessed(peer NodeId, counter uint64) {
+ if ph := pm.healthTable.Get(peer); ph != nil {
+ advanceEffectCounter(&ph.processed, counter)
+ }
+}
+
// SetForwardHandler sets the handler for forwarded transactions (adaptive serialization).
// Must be called before any forwarded transactions are received.
func (pm *PeerManager) SetForwardHandler(h ForwardHandler) {
@@ -707,7 +742,11 @@ func (pm *PeerManager) fetchFromConns(ctx context.Context, offset *pb.EffectRef,
for range conns {
r := <-ch
if r.err != nil {
- lastErr = r.err
+ // Absence is definitive only when every responding connection was
+ // missing. Never let a later empty reply mask a transport failure.
+ if lastErr == nil || errors.Is(lastErr, errFetchNotFound) || !errors.Is(r.err, errFetchNotFound) {
+ lastErr = r.err
+ }
continue
}
diff --git a/cluster/proto/ancestry.pb.go b/cluster/proto/ancestry.pb.go
new file mode 100644
index 0000000..e13b71e
--- /dev/null
+++ b/cluster/proto/ancestry.pb.go
@@ -0,0 +1,270 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// versions:
+// protoc-gen-go v1.32.0
+// protoc v3.21.12
+// source: cluster/proto/ancestry.proto
+
+package proto
+
+import (
+ protoreflect "google.golang.org/protobuf/reflect/protoreflect"
+ protoimpl "google.golang.org/protobuf/runtime/protoimpl"
+ reflect "reflect"
+ sync "sync"
+)
+
+const (
+ // Verify that this generated code is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
+ // Verify that runtime/protoimpl is sufficiently up-to-date.
+ _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
+)
+
+// A streaming resident ancestry transfer. Dependencies are scoped to key using
+// the same rules as ingestion. Snapshots do not terminate the transfer.
+type AncestryRequest struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+ Root *EffectRef `protobuf:"bytes,2,opt,name=root,proto3" json:"root,omitempty"`
+}
+
+func (x *AncestryRequest) Reset() {
+ *x = AncestryRequest{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_cluster_proto_ancestry_proto_msgTypes[0]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *AncestryRequest) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AncestryRequest) ProtoMessage() {}
+
+func (x *AncestryRequest) ProtoReflect() protoreflect.Message {
+ mi := &file_cluster_proto_ancestry_proto_msgTypes[0]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use AncestryRequest.ProtoReflect.Descriptor instead.
+func (*AncestryRequest) Descriptor() ([]byte, []int) {
+ return file_cluster_proto_ancestry_proto_rawDescGZIP(), []int{0}
+}
+
+func (x *AncestryRequest) GetKey() string {
+ if x != nil {
+ return x.Key
+ }
+ return ""
+}
+
+func (x *AncestryRequest) GetRoot() *EffectRef {
+ if x != nil {
+ return x.Root
+ }
+ return nil
+}
+
+// One effect or peer-local missing boundary. A clean transfer ends with done;
+// EOF/reset without done is a transport failure, never evidence of absence.
+type AncestryResponse struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Ref *EffectRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"`
+ EffectData []byte `protobuf:"bytes,2,opt,name=effect_data,json=effectData,proto3" json:"effect_data,omitempty"`
+ Done bool `protobuf:"varint,3,opt,name=done,proto3" json:"done,omitempty"`
+ Error string `protobuf:"bytes,4,opt,name=error,proto3" json:"error,omitempty"`
+ // History beyond this vertex has been culled, regardless of its effect deps.
+ IsRoot bool `protobuf:"varint,5,opt,name=is_root,json=isRoot,proto3" json:"is_root,omitempty"`
+}
+
+func (x *AncestryResponse) Reset() {
+ *x = AncestryResponse{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_cluster_proto_ancestry_proto_msgTypes[1]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *AncestryResponse) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*AncestryResponse) ProtoMessage() {}
+
+func (x *AncestryResponse) ProtoReflect() protoreflect.Message {
+ mi := &file_cluster_proto_ancestry_proto_msgTypes[1]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use AncestryResponse.ProtoReflect.Descriptor instead.
+func (*AncestryResponse) Descriptor() ([]byte, []int) {
+ return file_cluster_proto_ancestry_proto_rawDescGZIP(), []int{1}
+}
+
+func (x *AncestryResponse) GetRef() *EffectRef {
+ if x != nil {
+ return x.Ref
+ }
+ return nil
+}
+
+func (x *AncestryResponse) GetEffectData() []byte {
+ if x != nil {
+ return x.EffectData
+ }
+ return nil
+}
+
+func (x *AncestryResponse) GetDone() bool {
+ if x != nil {
+ return x.Done
+ }
+ return false
+}
+
+func (x *AncestryResponse) GetError() string {
+ if x != nil {
+ return x.Error
+ }
+ return ""
+}
+
+func (x *AncestryResponse) GetIsRoot() bool {
+ if x != nil {
+ return x.IsRoot
+ }
+ return false
+}
+
+var File_cluster_proto_ancestry_proto protoreflect.FileDescriptor
+
+var file_cluster_proto_ancestry_proto_rawDesc = []byte{
+ 0x0a, 0x1c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f,
+ 0x61, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x72, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x11,
+ 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76,
+ 0x31, 0x1a, 0x1f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
+ 0x2f, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f,
+ 0x74, 0x6f, 0x22, 0x55, 0x0a, 0x0f, 0x41, 0x6e, 0x63, 0x65, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65,
+ 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x18,
+ 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63,
+ 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74,
+ 0x52, 0x65, 0x66, 0x52, 0x04, 0x72, 0x6f, 0x6f, 0x74, 0x22, 0xa6, 0x01, 0x0a, 0x10, 0x41, 0x6e,
+ 0x63, 0x65, 0x73, 0x74, 0x72, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e,
+ 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77,
+ 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
+ 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x1f,
+ 0x0a, 0x0b, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20,
+ 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12,
+ 0x12, 0x0a, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64,
+ 0x6f, 0x6e, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x04, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f,
+ 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x52, 0x6f,
+ 0x6f, 0x74, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d,
+ 0x2f, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x64, 0x62, 0x2f, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65,
+ 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06,
+ 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+}
+
+var (
+ file_cluster_proto_ancestry_proto_rawDescOnce sync.Once
+ file_cluster_proto_ancestry_proto_rawDescData = file_cluster_proto_ancestry_proto_rawDesc
+)
+
+func file_cluster_proto_ancestry_proto_rawDescGZIP() []byte {
+ file_cluster_proto_ancestry_proto_rawDescOnce.Do(func() {
+ file_cluster_proto_ancestry_proto_rawDescData = protoimpl.X.CompressGZIP(file_cluster_proto_ancestry_proto_rawDescData)
+ })
+ return file_cluster_proto_ancestry_proto_rawDescData
+}
+
+var file_cluster_proto_ancestry_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
+var file_cluster_proto_ancestry_proto_goTypes = []interface{}{
+ (*AncestryRequest)(nil), // 0: swytch.cluster.v1.AncestryRequest
+ (*AncestryResponse)(nil), // 1: swytch.cluster.v1.AncestryResponse
+ (*EffectRef)(nil), // 2: swytch.cluster.v1.EffectRef
+}
+var file_cluster_proto_ancestry_proto_depIdxs = []int32{
+ 2, // 0: swytch.cluster.v1.AncestryRequest.root:type_name -> swytch.cluster.v1.EffectRef
+ 2, // 1: swytch.cluster.v1.AncestryResponse.ref:type_name -> swytch.cluster.v1.EffectRef
+ 2, // [2:2] is the sub-list for method output_type
+ 2, // [2:2] is the sub-list for method input_type
+ 2, // [2:2] is the sub-list for extension type_name
+ 2, // [2:2] is the sub-list for extension extendee
+ 0, // [0:2] is the sub-list for field type_name
+}
+
+func init() { file_cluster_proto_ancestry_proto_init() }
+func file_cluster_proto_ancestry_proto_init() {
+ if File_cluster_proto_ancestry_proto != nil {
+ return
+ }
+ file_cluster_proto_replication_proto_init()
+ if !protoimpl.UnsafeEnabled {
+ file_cluster_proto_ancestry_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*AncestryRequest); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_cluster_proto_ancestry_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*AncestryResponse); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ }
+ type x struct{}
+ out := protoimpl.TypeBuilder{
+ File: protoimpl.DescBuilder{
+ GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
+ RawDescriptor: file_cluster_proto_ancestry_proto_rawDesc,
+ NumEnums: 0,
+ NumMessages: 2,
+ NumExtensions: 0,
+ NumServices: 0,
+ },
+ GoTypes: file_cluster_proto_ancestry_proto_goTypes,
+ DependencyIndexes: file_cluster_proto_ancestry_proto_depIdxs,
+ MessageInfos: file_cluster_proto_ancestry_proto_msgTypes,
+ }.Build()
+ File_cluster_proto_ancestry_proto = out.File
+ file_cluster_proto_ancestry_proto_rawDesc = nil
+ file_cluster_proto_ancestry_proto_goTypes = nil
+ file_cluster_proto_ancestry_proto_depIdxs = nil
+}
diff --git a/cluster/proto/ancestry.proto b/cluster/proto/ancestry.proto
new file mode 100644
index 0000000..e06c5be
--- /dev/null
+++ b/cluster/proto/ancestry.proto
@@ -0,0 +1,24 @@
+syntax = "proto3";
+package swytch.cluster.v1;
+
+option go_package = "github.com/swytchdb/engine/cluster/proto";
+
+import "cluster/proto/replication.proto";
+
+// A streaming resident ancestry transfer. Dependencies are scoped to key using
+// the same rules as ingestion. Snapshots do not terminate the transfer.
+message AncestryRequest {
+ string key = 1;
+ EffectRef root = 2;
+}
+
+// One effect or peer-local missing boundary. A clean transfer ends with done;
+// EOF/reset without done is a transport failure, never evidence of absence.
+message AncestryResponse {
+ EffectRef ref = 1;
+ bytes effect_data = 2;
+ bool done = 3;
+ string error = 4;
+ // History beyond this vertex has been culled, regardless of its effect deps.
+ bool is_root = 5;
+}
diff --git a/cluster/proto/replication.pb.go b/cluster/proto/replication.pb.go
index d1f545b..ffd8ae9 100644
--- a/cluster/proto/replication.pb.go
+++ b/cluster/proto/replication.pb.go
@@ -525,7 +525,7 @@ func (x TypedValue_Kind) Number() protoreflect.EnumNumber {
// Deprecated: Use TypedValue_Kind.Descriptor instead.
func (TypedValue_Kind) EnumDescriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{9, 0}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{10, 0}
}
type Predicate_Kind int32
@@ -586,7 +586,7 @@ func (x Predicate_Kind) Number() protoreflect.EnumNumber {
// Deprecated: Use Predicate_Kind.Descriptor instead.
func (Predicate_Kind) EnumDescriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{10, 0}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{11, 0}
}
type RowWriteEffect_Kind int32
@@ -635,7 +635,7 @@ func (x RowWriteEffect_Kind) Number() protoreflect.EnumNumber {
// Deprecated: Use RowWriteEffect_Kind.Descriptor instead.
func (RowWriteEffect_Kind) EnumDescriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{12, 0}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{13, 0}
}
// CompressedValue is a raw value stored under a compression codec: data is
@@ -1392,6 +1392,9 @@ type SnapshotEffect struct {
// traverse historical adjudications via snapshots without consulting tips.
// Zero-valued if no prior snapshot exists for this key.
PrevSnapshot *EffectRef `protobuf:"bytes,4,opt,name=prev_snapshot,json=prevSnapshot,proto3" json:"prev_snapshot,omitempty"`
+ // Owned proof references for the recorded bind outcomes. These are not
+ // value dependencies and do not participate in frontier or LCA selection.
+ Evidence *BindEvidence `protobuf:"bytes,5,opt,name=evidence,proto3" json:"evidence,omitempty"`
}
func (x *SnapshotEffect) Reset() {
@@ -1454,6 +1457,70 @@ func (x *SnapshotEffect) GetPrevSnapshot() *EffectRef {
return nil
}
+func (x *SnapshotEffect) GetEvidence() *BindEvidence {
+ if x != nil {
+ return x.Evidence
+ }
+ return nil
+}
+
+// The evaluated binds are distinct from supporting ancestry. All references
+// own their vertices; neither kind is a value-DAG edge.
+type BindEvidence struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Binds []*EffectRef `protobuf:"bytes,1,rep,name=binds,proto3" json:"binds,omitempty"` // includes subjects from predecessor proofs
+ Deps []*EffectRef `protobuf:"bytes,2,rep,name=deps,proto3" json:"deps,omitempty"`
+}
+
+func (x *BindEvidence) Reset() {
+ *x = BindEvidence{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_cluster_proto_replication_proto_msgTypes[9]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *BindEvidence) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*BindEvidence) ProtoMessage() {}
+
+func (x *BindEvidence) ProtoReflect() protoreflect.Message {
+ mi := &file_cluster_proto_replication_proto_msgTypes[9]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use BindEvidence.ProtoReflect.Descriptor instead.
+func (*BindEvidence) Descriptor() ([]byte, []int) {
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{9}
+}
+
+func (x *BindEvidence) GetBinds() []*EffectRef {
+ if x != nil {
+ return x.Binds
+ }
+ return nil
+}
+
+func (x *BindEvidence) GetDeps() []*EffectRef {
+ if x != nil {
+ return x.Deps
+ }
+ return nil
+}
+
// TypedValue is a tagged literal used in Predicate comparisons and
// in the column tuple of a RowWriteEffect. The Kind tag selects
// which payload field is meaningful.
@@ -1472,7 +1539,7 @@ type TypedValue struct {
func (x *TypedValue) Reset() {
*x = TypedValue{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[9]
+ mi := &file_cluster_proto_replication_proto_msgTypes[10]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1485,7 +1552,7 @@ func (x *TypedValue) String() string {
func (*TypedValue) ProtoMessage() {}
func (x *TypedValue) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[9]
+ mi := &file_cluster_proto_replication_proto_msgTypes[10]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1498,7 +1565,7 @@ func (x *TypedValue) ProtoReflect() protoreflect.Message {
// Deprecated: Use TypedValue.ProtoReflect.Descriptor instead.
func (*TypedValue) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{9}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{10}
}
func (x *TypedValue) GetKind() TypedValue_Kind {
@@ -1564,7 +1631,7 @@ type Predicate struct {
func (x *Predicate) Reset() {
*x = Predicate{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[10]
+ mi := &file_cluster_proto_replication_proto_msgTypes[11]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1577,7 +1644,7 @@ func (x *Predicate) String() string {
func (*Predicate) ProtoMessage() {}
func (x *Predicate) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[10]
+ mi := &file_cluster_proto_replication_proto_msgTypes[11]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1590,7 +1657,7 @@ func (x *Predicate) ProtoReflect() protoreflect.Message {
// Deprecated: Use Predicate.ProtoReflect.Descriptor instead.
func (*Predicate) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{10}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{11}
}
func (x *Predicate) GetKind() Predicate_Kind {
@@ -1670,7 +1737,7 @@ type ObservationEffect struct {
func (x *ObservationEffect) Reset() {
*x = ObservationEffect{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[11]
+ mi := &file_cluster_proto_replication_proto_msgTypes[12]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1683,7 +1750,7 @@ func (x *ObservationEffect) String() string {
func (*ObservationEffect) ProtoMessage() {}
func (x *ObservationEffect) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[11]
+ mi := &file_cluster_proto_replication_proto_msgTypes[12]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1696,7 +1763,7 @@ func (x *ObservationEffect) ProtoReflect() protoreflect.Message {
// Deprecated: Use ObservationEffect.ProtoReflect.Descriptor instead.
func (*ObservationEffect) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{11}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{12}
}
func (x *ObservationEffect) GetPredicate() *Predicate {
@@ -1729,7 +1796,7 @@ type RowWriteEffect struct {
func (x *RowWriteEffect) Reset() {
*x = RowWriteEffect{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[12]
+ mi := &file_cluster_proto_replication_proto_msgTypes[13]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1742,7 +1809,7 @@ func (x *RowWriteEffect) String() string {
func (*RowWriteEffect) ProtoMessage() {}
func (x *RowWriteEffect) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[12]
+ mi := &file_cluster_proto_replication_proto_msgTypes[13]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1755,7 +1822,7 @@ func (x *RowWriteEffect) ProtoReflect() protoreflect.Message {
// Deprecated: Use RowWriteEffect.ProtoReflect.Descriptor instead.
func (*RowWriteEffect) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{12}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{13}
}
func (x *RowWriteEffect) GetPk() []byte {
@@ -1797,7 +1864,7 @@ type SubscriptionEffect struct {
func (x *SubscriptionEffect) Reset() {
*x = SubscriptionEffect{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[13]
+ mi := &file_cluster_proto_replication_proto_msgTypes[14]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1810,7 +1877,7 @@ func (x *SubscriptionEffect) String() string {
func (*SubscriptionEffect) ProtoMessage() {}
func (x *SubscriptionEffect) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[13]
+ mi := &file_cluster_proto_replication_proto_msgTypes[14]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1823,7 +1890,7 @@ func (x *SubscriptionEffect) ProtoReflect() protoreflect.Message {
// Deprecated: Use SubscriptionEffect.ProtoReflect.Descriptor instead.
func (*SubscriptionEffect) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{13}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{14}
}
func (x *SubscriptionEffect) GetSubscriberNodeId() uint64 {
@@ -1869,7 +1936,7 @@ type PubSubMessage struct {
func (x *PubSubMessage) Reset() {
*x = PubSubMessage{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[14]
+ mi := &file_cluster_proto_replication_proto_msgTypes[15]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1882,7 +1949,7 @@ func (x *PubSubMessage) String() string {
func (*PubSubMessage) ProtoMessage() {}
func (x *PubSubMessage) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[14]
+ mi := &file_cluster_proto_replication_proto_msgTypes[15]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1895,7 +1962,7 @@ func (x *PubSubMessage) ProtoReflect() protoreflect.Message {
// Deprecated: Use PubSubMessage.ProtoReflect.Descriptor instead.
func (*PubSubMessage) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{14}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{15}
}
func (x *PubSubMessage) GetChannel() []byte {
@@ -1926,7 +1993,7 @@ type SerializationEffect struct {
func (x *SerializationEffect) Reset() {
*x = SerializationEffect{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[15]
+ mi := &file_cluster_proto_replication_proto_msgTypes[16]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -1939,7 +2006,7 @@ func (x *SerializationEffect) String() string {
func (*SerializationEffect) ProtoMessage() {}
func (x *SerializationEffect) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[15]
+ mi := &file_cluster_proto_replication_proto_msgTypes[16]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -1952,7 +2019,7 @@ func (x *SerializationEffect) ProtoReflect() protoreflect.Message {
// Deprecated: Use SerializationEffect.ProtoReflect.Descriptor instead.
func (*SerializationEffect) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{15}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{16}
}
func (x *SerializationEffect) GetLeaderNodeId() uint64 {
@@ -1969,6 +2036,64 @@ func (x *SerializationEffect) GetRelease() bool {
return false
}
+// Inline proof payload, encoded with the normal Effect wire encoding. Packages
+// are dependency-first and deduplicated; only explicit evidence edges are
+// followed, never the payloads' ordinary value ancestry.
+type EvidenceRecord struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Ref *EffectRef `protobuf:"bytes,1,opt,name=ref,proto3" json:"ref,omitempty"`
+ EffectData []byte `protobuf:"bytes,2,opt,name=effect_data,json=effectData,proto3" json:"effect_data,omitempty"`
+}
+
+func (x *EvidenceRecord) Reset() {
+ *x = EvidenceRecord{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_cluster_proto_replication_proto_msgTypes[17]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *EvidenceRecord) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*EvidenceRecord) ProtoMessage() {}
+
+func (x *EvidenceRecord) ProtoReflect() protoreflect.Message {
+ mi := &file_cluster_proto_replication_proto_msgTypes[17]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use EvidenceRecord.ProtoReflect.Descriptor instead.
+func (*EvidenceRecord) Descriptor() ([]byte, []int) {
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{17}
+}
+
+func (x *EvidenceRecord) GetRef() *EffectRef {
+ if x != nil {
+ return x.Ref
+ }
+ return nil
+}
+
+func (x *EvidenceRecord) GetEffectData() []byte {
+ if x != nil {
+ return x.EffectData
+ }
+ return nil
+}
+
// Effect is the top-level envelope stored in the log and sent over the wire.
type Effect struct {
state protoimpl.MessageState
@@ -1983,6 +2108,9 @@ type Effect struct {
TxnId string `protobuf:"bytes,5,opt,name=txn_id,json=txnId,proto3" json:"txn_id,omitempty"` // transaction ID (empty = non-transactional)
ForkChoiceHash []byte `protobuf:"bytes,6,opt,name=fork_choice_hash,json=forkChoiceHash,proto3" json:"fork_choice_hash,omitempty"` // SHA-256(node_id_LE8 || hlc_LE8), computed at emit time
Virtual []byte `protobuf:"bytes,7,opt,name=virtual,proto3" json:"virtual,omitempty"` // partition key WITHIN the key (empty = root partition); enables nested values
+ // Transport-only payloads for snapshot.evidence (including nested proofs).
+ // Receivers install these before the owner and strip this field in the pool.
+ EvidencePackage []*EvidenceRecord `protobuf:"bytes,8,rep,name=evidence_package,json=evidencePackage,proto3" json:"evidence_package,omitempty"`
// The effect payload — exactly one of these is set.
//
// Types that are assignable to Kind:
@@ -2003,7 +2131,7 @@ type Effect struct {
func (x *Effect) Reset() {
*x = Effect{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[16]
+ mi := &file_cluster_proto_replication_proto_msgTypes[18]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2016,7 +2144,7 @@ func (x *Effect) String() string {
func (*Effect) ProtoMessage() {}
func (x *Effect) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[16]
+ mi := &file_cluster_proto_replication_proto_msgTypes[18]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2029,7 +2157,7 @@ func (x *Effect) ProtoReflect() protoreflect.Message {
// Deprecated: Use Effect.ProtoReflect.Descriptor instead.
func (*Effect) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{16}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{18}
}
func (x *Effect) GetKey() []byte {
@@ -2081,6 +2209,13 @@ func (x *Effect) GetVirtual() []byte {
return nil
}
+func (x *Effect) GetEvidencePackage() []*EvidenceRecord {
+ if x != nil {
+ return x.EvidencePackage
+ }
+ return nil
+}
+
func (m *Effect) GetKind() isEffect_Kind {
if m != nil {
return m.Kind
@@ -2240,7 +2375,7 @@ type OffsetNotify struct {
func (x *OffsetNotify) Reset() {
*x = OffsetNotify{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[17]
+ mi := &file_cluster_proto_replication_proto_msgTypes[19]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2253,7 +2388,7 @@ func (x *OffsetNotify) String() string {
func (*OffsetNotify) ProtoMessage() {}
func (x *OffsetNotify) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[17]
+ mi := &file_cluster_proto_replication_proto_msgTypes[19]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2266,7 +2401,7 @@ func (x *OffsetNotify) ProtoReflect() protoreflect.Message {
// Deprecated: Use OffsetNotify.ProtoReflect.Descriptor instead.
func (*OffsetNotify) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{17}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{19}
}
func (x *OffsetNotify) GetOrigin() *EffectRef {
@@ -2323,7 +2458,7 @@ type FetchRequest struct {
func (x *FetchRequest) Reset() {
*x = FetchRequest{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[18]
+ mi := &file_cluster_proto_replication_proto_msgTypes[20]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2336,7 +2471,7 @@ func (x *FetchRequest) String() string {
func (*FetchRequest) ProtoMessage() {}
func (x *FetchRequest) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[18]
+ mi := &file_cluster_proto_replication_proto_msgTypes[20]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2349,7 +2484,7 @@ func (x *FetchRequest) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchRequest.ProtoReflect.Descriptor instead.
func (*FetchRequest) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{18}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{20}
}
func (x *FetchRequest) GetRef() *EffectRef {
@@ -2372,7 +2507,7 @@ type FetchResponse struct {
func (x *FetchResponse) Reset() {
*x = FetchResponse{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[19]
+ mi := &file_cluster_proto_replication_proto_msgTypes[21]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2385,7 +2520,7 @@ func (x *FetchResponse) String() string {
func (*FetchResponse) ProtoMessage() {}
func (x *FetchResponse) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[19]
+ mi := &file_cluster_proto_replication_proto_msgTypes[21]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2398,7 +2533,7 @@ func (x *FetchResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use FetchResponse.ProtoReflect.Descriptor instead.
func (*FetchResponse) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{19}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{21}
}
func (x *FetchResponse) GetRef() *EffectRef {
@@ -2439,7 +2574,7 @@ type NackTipDetail struct {
func (x *NackTipDetail) Reset() {
*x = NackTipDetail{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[20]
+ mi := &file_cluster_proto_replication_proto_msgTypes[22]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2452,7 +2587,7 @@ func (x *NackTipDetail) String() string {
func (*NackTipDetail) ProtoMessage() {}
func (x *NackTipDetail) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[20]
+ mi := &file_cluster_proto_replication_proto_msgTypes[22]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2465,7 +2600,7 @@ func (x *NackTipDetail) ProtoReflect() protoreflect.Message {
// Deprecated: Use NackTipDetail.ProtoReflect.Descriptor instead.
func (*NackTipDetail) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{20}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{22}
}
func (x *NackTipDetail) GetRef() *EffectRef {
@@ -2572,7 +2707,7 @@ type KeyConsumedTips struct {
func (x *KeyConsumedTips) Reset() {
*x = KeyConsumedTips{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[21]
+ mi := &file_cluster_proto_replication_proto_msgTypes[23]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2585,7 +2720,7 @@ func (x *KeyConsumedTips) String() string {
func (*KeyConsumedTips) ProtoMessage() {}
func (x *KeyConsumedTips) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[21]
+ mi := &file_cluster_proto_replication_proto_msgTypes[23]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2598,7 +2733,7 @@ func (x *KeyConsumedTips) ProtoReflect() protoreflect.Message {
// Deprecated: Use KeyConsumedTips.ProtoReflect.Descriptor instead.
func (*KeyConsumedTips) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{21}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{23}
}
func (x *KeyConsumedTips) GetKey() []byte {
@@ -2635,7 +2770,7 @@ type NackNotify struct {
func (x *NackNotify) Reset() {
*x = NackNotify{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[22]
+ mi := &file_cluster_proto_replication_proto_msgTypes[24]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2648,7 +2783,7 @@ func (x *NackNotify) String() string {
func (*NackNotify) ProtoMessage() {}
func (x *NackNotify) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[22]
+ mi := &file_cluster_proto_replication_proto_msgTypes[24]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2661,7 +2796,7 @@ func (x *NackNotify) ProtoReflect() protoreflect.Message {
// Deprecated: Use NackNotify.ProtoReflect.Descriptor instead.
func (*NackNotify) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{22}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{24}
}
func (x *NackNotify) GetKey() []byte {
@@ -2737,7 +2872,7 @@ type ForwardedTransaction struct {
func (x *ForwardedTransaction) Reset() {
*x = ForwardedTransaction{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[23]
+ mi := &file_cluster_proto_replication_proto_msgTypes[25]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2750,7 +2885,7 @@ func (x *ForwardedTransaction) String() string {
func (*ForwardedTransaction) ProtoMessage() {}
func (x *ForwardedTransaction) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[23]
+ mi := &file_cluster_proto_replication_proto_msgTypes[25]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2763,7 +2898,7 @@ func (x *ForwardedTransaction) ProtoReflect() protoreflect.Message {
// Deprecated: Use ForwardedTransaction.ProtoReflect.Descriptor instead.
func (*ForwardedTransaction) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{23}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{25}
}
func (x *ForwardedTransaction) GetCommands() []*ForwardedCommand {
@@ -2806,7 +2941,7 @@ type ForwardedCommand struct {
func (x *ForwardedCommand) Reset() {
*x = ForwardedCommand{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[24]
+ mi := &file_cluster_proto_replication_proto_msgTypes[26]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2819,7 +2954,7 @@ func (x *ForwardedCommand) String() string {
func (*ForwardedCommand) ProtoMessage() {}
func (x *ForwardedCommand) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[24]
+ mi := &file_cluster_proto_replication_proto_msgTypes[26]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2832,7 +2967,7 @@ func (x *ForwardedCommand) ProtoReflect() protoreflect.Message {
// Deprecated: Use ForwardedCommand.ProtoReflect.Descriptor instead.
func (*ForwardedCommand) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{24}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{26}
}
func (x *ForwardedCommand) GetCommandName() []byte {
@@ -2862,7 +2997,7 @@ type ForwardedResponse struct {
func (x *ForwardedResponse) Reset() {
*x = ForwardedResponse{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[25]
+ mi := &file_cluster_proto_replication_proto_msgTypes[27]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2875,7 +3010,7 @@ func (x *ForwardedResponse) String() string {
func (*ForwardedResponse) ProtoMessage() {}
func (x *ForwardedResponse) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[25]
+ mi := &file_cluster_proto_replication_proto_msgTypes[27]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -2888,7 +3023,7 @@ func (x *ForwardedResponse) ProtoReflect() protoreflect.Message {
// Deprecated: Use ForwardedResponse.ProtoReflect.Descriptor instead.
func (*ForwardedResponse) Descriptor() ([]byte, []int) {
- return file_cluster_proto_replication_proto_rawDescGZIP(), []int{25}
+ return file_cluster_proto_replication_proto_rawDescGZIP(), []int{27}
}
func (x *ForwardedResponse) GetRespData() []byte {
@@ -2926,7 +3061,7 @@ type TransactionalBindEffect_KeyBind struct {
func (x *TransactionalBindEffect_KeyBind) Reset() {
*x = TransactionalBindEffect_KeyBind{}
if protoimpl.UnsafeEnabled {
- mi := &file_cluster_proto_replication_proto_msgTypes[26]
+ mi := &file_cluster_proto_replication_proto_msgTypes[28]
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
ms.StoreMessageInfo(mi)
}
@@ -2939,7 +3074,7 @@ func (x *TransactionalBindEffect_KeyBind) String() string {
func (*TransactionalBindEffect_KeyBind) ProtoMessage() {}
func (x *TransactionalBindEffect_KeyBind) ProtoReflect() protoreflect.Message {
- mi := &file_cluster_proto_replication_proto_msgTypes[26]
+ mi := &file_cluster_proto_replication_proto_msgTypes[28]
if protoimpl.UnsafeEnabled && x != nil {
ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
if ms.LoadMessageInfo() == nil {
@@ -3166,7 +3301,7 @@ var file_cluster_proto_replication_proto_rawDesc = []byte{
0x76, 0x31, 0x2e, 0x52, 0x65, 0x64, 0x75, 0x63, 0x65, 0x64, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74,
0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x17, 0x0a, 0x15, 0x5f,
0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6c, 0x65,
- 0x61, 0x64, 0x65, 0x72, 0x22, 0x81, 0x03, 0x0a, 0x0e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
+ 0x61, 0x64, 0x65, 0x72, 0x22, 0xbe, 0x03, 0x0a, 0x0e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
0x74, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12, 0x41, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65,
0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x73, 0x77,
0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
@@ -3184,333 +3319,355 @@ var file_cluster_proto_replication_proto_rawDesc = []byte{
0x76, 0x5f, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0c,
- 0x70, 0x72, 0x65, 0x76, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x1a, 0x5a, 0x0a, 0x10,
- 0x54, 0x78, 0x6e, 0x56, 0x65, 0x72, 0x64, 0x69, 0x63, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
- 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
- 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
- 0x0e, 0x32, 0x1a, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74,
- 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x64, 0x69, 0x63, 0x74, 0x52, 0x05, 0x76,
- 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xf0, 0x01, 0x0a, 0x0a, 0x54, 0x79, 0x70,
- 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18,
- 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63,
- 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64, 0x56,
- 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12,
- 0x17, 0x0a, 0x07, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03,
- 0x52, 0x06, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x6c, 0x6f, 0x61,
- 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x66, 0x6c, 0x6f,
- 0x61, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x78, 0x74, 0x5f, 0x76, 0x61,
- 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x78, 0x74, 0x56, 0x61, 0x6c,
- 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6c, 0x6f, 0x62, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x05, 0x20, 0x01,
- 0x28, 0x0c, 0x52, 0x07, 0x62, 0x6c, 0x6f, 0x62, 0x56, 0x61, 0x6c, 0x22, 0x3e, 0x0a, 0x04, 0x4b,
- 0x69, 0x6e, 0x64, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56, 0x41, 0x4c, 0x55,
- 0x45, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05,
- 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x54, 0x45, 0x58, 0x54, 0x10,
- 0x03, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4c, 0x4f, 0x42, 0x10, 0x04, 0x22, 0xac, 0x03, 0x0a, 0x09,
- 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x04, 0x6b, 0x69, 0x6e,
- 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
+ 0x70, 0x72, 0x65, 0x76, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x3b, 0x0a, 0x08,
+ 0x65, 0x76, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f,
+ 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
+ 0x76, 0x31, 0x2e, 0x42, 0x69, 0x6e, 0x64, 0x45, 0x76, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x52,
+ 0x08, 0x65, 0x76, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x1a, 0x5a, 0x0a, 0x10, 0x54, 0x78, 0x6e,
+ 0x56, 0x65, 0x72, 0x64, 0x69, 0x63, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a,
+ 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12,
+ 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a,
+ 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
+ 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x64, 0x69, 0x63, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75,
+ 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x74, 0x0a, 0x0c, 0x42, 0x69, 0x6e, 0x64, 0x45, 0x76, 0x69,
+ 0x64, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x62, 0x69, 0x6e, 0x64, 0x73, 0x18, 0x01,
+ 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c,
+ 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52,
+ 0x65, 0x66, 0x52, 0x05, 0x62, 0x69, 0x6e, 0x64, 0x73, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x65, 0x70,
+ 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
+ 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65,
+ 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x04, 0x64, 0x65, 0x70, 0x73, 0x22, 0xf0, 0x01, 0x0a, 0x0a,
+ 0x54, 0x79, 0x70, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x36, 0x0a, 0x04, 0x6b, 0x69,
+ 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63,
+ 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70,
+ 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69,
+ 0x6e, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20,
+ 0x01, 0x28, 0x03, 0x52, 0x06, 0x69, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x1b, 0x0a, 0x09, 0x66,
+ 0x6c, 0x6f, 0x61, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08,
+ 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x56, 0x61, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x65, 0x78, 0x74,
+ 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x65, 0x78, 0x74,
+ 0x56, 0x61, 0x6c, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6c, 0x6f, 0x62, 0x5f, 0x76, 0x61, 0x6c, 0x18,
+ 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x62, 0x6c, 0x6f, 0x62, 0x56, 0x61, 0x6c, 0x22, 0x3e,
+ 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0e, 0x0a, 0x0a, 0x4e, 0x55, 0x4c, 0x4c, 0x5f, 0x56,
+ 0x41, 0x4c, 0x55, 0x45, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12,
+ 0x09, 0x0a, 0x05, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x02, 0x12, 0x08, 0x0a, 0x04, 0x54, 0x45,
+ 0x58, 0x54, 0x10, 0x03, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4c, 0x4f, 0x42, 0x10, 0x04, 0x22, 0xac,
+ 0x03, 0x0a, 0x09, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x35, 0x0a, 0x04,
+ 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79,
+ 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50,
+ 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b,
+ 0x69, 0x6e, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x18,
+ 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x12, 0x38,
+ 0x0a, 0x08, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b,
+ 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
+ 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x08,
+ 0x63, 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x63, 0x68, 0x69, 0x6c,
+ 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x65, 0x64,
- 0x69, 0x63, 0x61, 0x74, 0x65, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64,
- 0x12, 0x19, 0x0a, 0x08, 0x62, 0x6f, 0x6f, 0x6c, 0x5f, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x08, 0x52, 0x07, 0x62, 0x6f, 0x6f, 0x6c, 0x56, 0x61, 0x6c, 0x12, 0x38, 0x0a, 0x08, 0x63,
- 0x68, 0x69, 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e,
- 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76,
- 0x31, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x08, 0x63, 0x68, 0x69,
- 0x6c, 0x64, 0x72, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x18, 0x04,
- 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c,
- 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61,
- 0x74, 0x65, 0x52, 0x05, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x6f, 0x6c,
- 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x63, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63,
- 0x6f, 0x6c, 0x32, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f, 0x6c, 0x32, 0x12,
- 0x31, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x73, 0x77,
- 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
- 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x43, 0x6d, 0x70, 0x4f, 0x70, 0x52, 0x02,
- 0x6f, 0x70, 0x12, 0x37, 0x0a, 0x07, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x18, 0x08, 0x20,
- 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75,
- 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64, 0x56, 0x61, 0x6c,
- 0x75, 0x65, 0x52, 0x07, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x22, 0x4d, 0x0a, 0x04, 0x4b,
- 0x69, 0x6e, 0x64, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00, 0x12, 0x07, 0x0a,
- 0x03, 0x41, 0x4e, 0x44, 0x10, 0x01, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x52, 0x10, 0x02, 0x12, 0x07,
- 0x0a, 0x03, 0x4e, 0x4f, 0x54, 0x10, 0x03, 0x12, 0x07, 0x0a, 0x03, 0x43, 0x4d, 0x50, 0x10, 0x04,
- 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4c, 0x5f, 0x43, 0x4d, 0x50, 0x10, 0x05, 0x12, 0x0b, 0x0a,
- 0x07, 0x49, 0x53, 0x5f, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x06, 0x22, 0x4f, 0x0a, 0x11, 0x4f, 0x62,
- 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12,
- 0x3a, 0x0a, 0x09, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01,
- 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73,
- 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65,
- 0x52, 0x09, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, 0xc1, 0x01, 0x0a, 0x0e,
- 0x52, 0x6f, 0x77, 0x57, 0x72, 0x69, 0x74, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12, 0x0e,
- 0x0a, 0x02, 0x70, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x70, 0x6b, 0x12, 0x3a,
- 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x26, 0x2e, 0x73,
- 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31,
- 0x2e, 0x52, 0x6f, 0x77, 0x57, 0x72, 0x69, 0x74, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x2e,
- 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x37, 0x0a, 0x07, 0x63, 0x6f,
- 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x77,
- 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
- 0x54, 0x79, 0x70, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x63, 0x6f, 0x6c, 0x75,
- 0x6d, 0x6e, 0x73, 0x22, 0x2a, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0a, 0x0a, 0x06, 0x49,
- 0x4e, 0x53, 0x45, 0x52, 0x54, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50, 0x44, 0x41, 0x54,
- 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x02, 0x22,
- 0xa0, 0x01, 0x0a, 0x12, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
- 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72,
- 0x69, 0x62, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01,
- 0x28, 0x04, 0x52, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x4e, 0x6f,
- 0x64, 0x65, 0x49, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x75, 0x6e, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72,
- 0x69, 0x62, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x75, 0x6e, 0x73, 0x75, 0x62,
- 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76,
- 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x64, 0x69, 0x73, 0x63, 0x6f,
- 0x76, 0x65, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72, 0x61,
- 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x65, 0x70, 0x68, 0x65, 0x6d, 0x65, 0x72,
- 0x61, 0x6c, 0x22, 0x43, 0x0a, 0x0d, 0x50, 0x75, 0x62, 0x53, 0x75, 0x62, 0x4d, 0x65, 0x73, 0x73,
- 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x18, 0x01,
- 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x18, 0x0a,
- 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07,
- 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x55, 0x0a, 0x13, 0x53, 0x65, 0x72, 0x69, 0x61,
- 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12, 0x24,
- 0x0a, 0x0e, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64,
- 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4e, 0x6f,
- 0x64, 0x65, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x18,
- 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x22, 0x93,
- 0x07, 0x0a, 0x06, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
- 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x03, 0x68,
- 0x6c, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
- 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
- 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x68, 0x6c, 0x63, 0x12, 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64,
- 0x65, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65,
- 0x49, 0x64, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x65, 0x70, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b,
+ 0x69, 0x63, 0x61, 0x74, 0x65, 0x52, 0x05, 0x63, 0x68, 0x69, 0x6c, 0x64, 0x12, 0x10, 0x0a, 0x03,
+ 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x63, 0x6f, 0x6c, 0x12, 0x12,
+ 0x0a, 0x04, 0x63, 0x6f, 0x6c, 0x32, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x63, 0x6f,
+ 0x6c, 0x32, 0x12, 0x31, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21,
+ 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
+ 0x76, 0x31, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x43, 0x6d, 0x70, 0x4f,
+ 0x70, 0x52, 0x02, 0x6f, 0x70, 0x12, 0x37, 0x0a, 0x07, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c,
+ 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e,
+ 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64,
+ 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x22, 0x4d,
+ 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x08, 0x0a, 0x04, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x00,
+ 0x12, 0x07, 0x0a, 0x03, 0x41, 0x4e, 0x44, 0x10, 0x01, 0x12, 0x06, 0x0a, 0x02, 0x4f, 0x52, 0x10,
+ 0x02, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x4f, 0x54, 0x10, 0x03, 0x12, 0x07, 0x0a, 0x03, 0x43, 0x4d,
+ 0x50, 0x10, 0x04, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x4f, 0x4c, 0x5f, 0x43, 0x4d, 0x50, 0x10, 0x05,
+ 0x12, 0x0b, 0x0a, 0x07, 0x49, 0x53, 0x5f, 0x4e, 0x55, 0x4c, 0x4c, 0x10, 0x06, 0x22, 0x4f, 0x0a,
+ 0x11, 0x4f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65,
+ 0x63, 0x74, 0x12, 0x3a, 0x0a, 0x09, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x18,
+ 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63,
+ 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63,
+ 0x61, 0x74, 0x65, 0x52, 0x09, 0x70, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x22, 0xc1,
+ 0x01, 0x0a, 0x0e, 0x52, 0x6f, 0x77, 0x57, 0x72, 0x69, 0x74, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63,
+ 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x70, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x02, 0x70,
+ 0x6b, 0x12, 0x3a, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32,
+ 0x26, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
+ 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x77, 0x57, 0x72, 0x69, 0x74, 0x65, 0x45, 0x66, 0x66, 0x65,
+ 0x63, 0x74, 0x2e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x12, 0x37, 0x0a,
+ 0x07, 0x63, 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d,
+ 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
+ 0x76, 0x31, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x64, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x63,
+ 0x6f, 0x6c, 0x75, 0x6d, 0x6e, 0x73, 0x22, 0x2a, 0x0a, 0x04, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0a,
+ 0x0a, 0x06, 0x49, 0x4e, 0x53, 0x45, 0x52, 0x54, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x55, 0x50,
+ 0x44, 0x41, 0x54, 0x45, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45,
+ 0x10, 0x02, 0x22, 0xa0, 0x01, 0x0a, 0x12, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74,
+ 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x75, 0x62,
+ 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18,
+ 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65,
+ 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x20, 0x0a, 0x0b, 0x75, 0x6e, 0x73, 0x75, 0x62,
+ 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x75, 0x6e,
+ 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x64, 0x69, 0x73,
+ 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x64, 0x69,
+ 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x70, 0x68, 0x65, 0x6d,
+ 0x65, 0x72, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x65, 0x70, 0x68, 0x65,
+ 0x6d, 0x65, 0x72, 0x61, 0x6c, 0x22, 0x43, 0x0a, 0x0d, 0x50, 0x75, 0x62, 0x53, 0x75, 0x62, 0x4d,
+ 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65,
+ 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c,
+ 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28,
+ 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x22, 0x55, 0x0a, 0x13, 0x53, 0x65,
+ 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65, 0x63,
+ 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x6c, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x6e, 0x6f, 0x64, 0x65,
+ 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0c, 0x6c, 0x65, 0x61, 0x64, 0x65,
+ 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61,
+ 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73,
+ 0x65, 0x22, 0x61, 0x0a, 0x0e, 0x45, 0x76, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x63,
+ 0x6f, 0x72, 0x64, 0x12, 0x2e, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b,
0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
- 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x04,
- 0x64, 0x65, 0x70, 0x73, 0x12, 0x15, 0x0a, 0x06, 0x74, 0x78, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05,
- 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x78, 0x6e, 0x49, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x66,
- 0x6f, 0x72, 0x6b, 0x5f, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18,
- 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x66, 0x6f, 0x72, 0x6b, 0x43, 0x68, 0x6f, 0x69, 0x63,
- 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c,
- 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x12,
- 0x33, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e,
+ 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03,
+ 0x72, 0x65, 0x66, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x61,
+ 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74,
+ 0x44, 0x61, 0x74, 0x61, 0x22, 0xe1, 0x07, 0x0a, 0x06, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x12,
+ 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65,
+ 0x79, 0x12, 0x2c, 0x0a, 0x03, 0x68, 0x6c, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
+ 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+ 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x68, 0x6c, 0x63, 0x12,
+ 0x17, 0x0a, 0x07, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04,
+ 0x52, 0x06, 0x6e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x65, 0x70, 0x73,
+ 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e,
+ 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63,
+ 0x74, 0x52, 0x65, 0x66, 0x52, 0x04, 0x64, 0x65, 0x70, 0x73, 0x12, 0x15, 0x0a, 0x06, 0x74, 0x78,
+ 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x78, 0x6e, 0x49,
+ 0x64, 0x12, 0x28, 0x0a, 0x10, 0x66, 0x6f, 0x72, 0x6b, 0x5f, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65,
+ 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0e, 0x66, 0x6f, 0x72,
+ 0x6b, 0x43, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x18, 0x0a, 0x07, 0x76,
+ 0x69, 0x72, 0x74, 0x75, 0x61, 0x6c, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x69,
+ 0x72, 0x74, 0x75, 0x61, 0x6c, 0x12, 0x4c, 0x0a, 0x10, 0x65, 0x76, 0x69, 0x64, 0x65, 0x6e, 0x63,
+ 0x65, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32,
+ 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
+ 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x76, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x63, 0x6f,
+ 0x72, 0x64, 0x52, 0x0f, 0x65, 0x76, 0x69, 0x64, 0x65, 0x6e, 0x63, 0x65, 0x50, 0x61, 0x63, 0x6b,
+ 0x61, 0x67, 0x65, 0x12, 0x33, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, 0x0a, 0x20, 0x01, 0x28,
+ 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74,
+ 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74,
+ 0x48, 0x00, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x33, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61,
+ 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e,
+ 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45,
+ 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x47, 0x0a,
+ 0x08, 0x74, 0x78, 0x6e, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32,
+ 0x2a, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72,
+ 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61,
+ 0x6c, 0x42, 0x69, 0x6e, 0x64, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x07, 0x74,
+ 0x78, 0x6e, 0x42, 0x69, 0x6e, 0x64, 0x12, 0x3f, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68,
+ 0x6f, 0x74, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63,
+ 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6e, 0x61,
+ 0x70, 0x73, 0x68, 0x6f, 0x74, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x08, 0x73,
+ 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x12, 0x4b, 0x0a, 0x0c, 0x73, 0x75, 0x62, 0x73, 0x63,
+ 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e,
0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76,
- 0x31, 0x2e, 0x44, 0x61, 0x74, 0x61, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x04,
- 0x64, 0x61, 0x74, 0x61, 0x12, 0x33, 0x0a, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x18, 0x0b, 0x20, 0x01,
- 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73,
- 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x65, 0x74, 0x61, 0x45, 0x66, 0x66, 0x65, 0x63,
- 0x74, 0x48, 0x00, 0x52, 0x04, 0x6d, 0x65, 0x74, 0x61, 0x12, 0x47, 0x0a, 0x08, 0x74, 0x78, 0x6e,
- 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x73, 0x77,
+ 0x31, 0x2e, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66,
+ 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70,
+ 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x4e, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a,
+ 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x73, 0x77,
0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
- 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x42, 0x69, 0x6e,
- 0x64, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x07, 0x74, 0x78, 0x6e, 0x42, 0x69,
- 0x6e, 0x64, 0x12, 0x3f, 0x0a, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f, 0x74, 0x18, 0x0e,
- 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c,
- 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6e, 0x61, 0x70, 0x73, 0x68, 0x6f,
- 0x74, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x08, 0x73, 0x6e, 0x61, 0x70, 0x73,
- 0x68, 0x6f, 0x74, 0x12, 0x4b, 0x0a, 0x0c, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74,
- 0x69, 0x6f, 0x6e, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x73, 0x77, 0x79, 0x74,
- 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x75,
- 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74,
- 0x48, 0x00, 0x52, 0x0c, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e,
- 0x12, 0x4e, 0x0a, 0x0d, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f,
- 0x6e, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
- 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x65, 0x72, 0x69,
- 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48,
- 0x00, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e,
- 0x12, 0x33, 0x0a, 0x04, 0x6e, 0x6f, 0x6f, 0x70, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d,
+ 0x53, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66,
+ 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61,
+ 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x33, 0x0a, 0x04, 0x6e, 0x6f, 0x6f, 0x70, 0x18, 0x11, 0x20, 0x01,
+ 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73,
+ 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x6f, 0x70, 0x45, 0x66, 0x66, 0x65, 0x63,
+ 0x74, 0x48, 0x00, 0x52, 0x04, 0x6e, 0x6f, 0x6f, 0x70, 0x12, 0x48, 0x0a, 0x0b, 0x6f, 0x62, 0x73,
+ 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24,
0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
- 0x76, 0x31, 0x2e, 0x4e, 0x6f, 0x6f, 0x70, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52,
- 0x04, 0x6e, 0x6f, 0x6f, 0x70, 0x12, 0x48, 0x0a, 0x0b, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61,
- 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x73, 0x77, 0x79,
- 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4f,
- 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74,
- 0x48, 0x00, 0x52, 0x0b, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12,
- 0x40, 0x0a, 0x09, 0x72, 0x6f, 0x77, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65, 0x18, 0x13, 0x20, 0x01,
- 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73,
- 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x77, 0x57, 0x72, 0x69, 0x74, 0x65, 0x45,
- 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x08, 0x72, 0x6f, 0x77, 0x57, 0x72, 0x69, 0x74,
- 0x65, 0x12, 0x49, 0x0a, 0x0e, 0x70, 0x75, 0x62, 0x73, 0x75, 0x62, 0x5f, 0x6d, 0x65, 0x73, 0x73,
- 0x61, 0x67, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x73, 0x77, 0x79, 0x74,
- 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x75,
- 0x62, 0x53, 0x75, 0x62, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x48, 0x00, 0x52, 0x0d, 0x70,
- 0x75, 0x62, 0x73, 0x75, 0x62, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x42, 0x06, 0x0a, 0x04,
- 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x82, 0x02, 0x0a, 0x0c, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x4e,
- 0x6f, 0x74, 0x69, 0x66, 0x79, 0x12, 0x34, 0x0a, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x18,
- 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63,
- 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74,
- 0x52, 0x65, 0x66, 0x52, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x12, 0x2c, 0x0a, 0x03, 0x68,
- 0x6c, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c,
- 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73,
- 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x68, 0x6c, 0x63, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x65, 0x70,
- 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
- 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65,
- 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x04, 0x64, 0x65, 0x70, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6b,
- 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x1f, 0x0a,
- 0x0b, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01,
- 0x28, 0x0c, 0x52, 0x0a, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, 0x23,
- 0x0a, 0x0d, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18,
- 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x74, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74,
- 0x65, 0x78, 0x74, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, 0x3e, 0x0a, 0x0c, 0x46, 0x65, 0x74,
- 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a, 0x03, 0x72, 0x65, 0x66,
- 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e,
- 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63,
- 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x60, 0x0a, 0x0d, 0x46, 0x65, 0x74,
- 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e, 0x0a, 0x03, 0x72, 0x65,
- 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
- 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65,
- 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x66,
- 0x66, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52,
- 0x0a, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x44, 0x61, 0x74, 0x61, 0x22, 0xe9, 0x04, 0x0a, 0x0d,
- 0x4e, 0x61, 0x63, 0x6b, 0x54, 0x69, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x2e, 0x0a,
+ 0x76, 0x31, 0x2e, 0x4f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x66,
+ 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x6f, 0x62, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74,
+ 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, 0x09, 0x72, 0x6f, 0x77, 0x5f, 0x77, 0x72, 0x69, 0x74, 0x65,
+ 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e,
+ 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x6f, 0x77, 0x57, 0x72,
+ 0x69, 0x74, 0x65, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x08, 0x72, 0x6f, 0x77,
+ 0x57, 0x72, 0x69, 0x74, 0x65, 0x12, 0x49, 0x0a, 0x0e, 0x70, 0x75, 0x62, 0x73, 0x75, 0x62, 0x5f,
+ 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e,
+ 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76,
+ 0x31, 0x2e, 0x50, 0x75, 0x62, 0x53, 0x75, 0x62, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x48,
+ 0x00, 0x52, 0x0d, 0x70, 0x75, 0x62, 0x73, 0x75, 0x62, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65,
+ 0x42, 0x06, 0x0a, 0x04, 0x6b, 0x69, 0x6e, 0x64, 0x22, 0x82, 0x02, 0x0a, 0x0c, 0x4f, 0x66, 0x66,
+ 0x73, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x12, 0x34, 0x0a, 0x06, 0x6f, 0x72, 0x69,
+ 0x67, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74,
+ 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66,
+ 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x06, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x12,
+ 0x2c, 0x0a, 0x03, 0x68, 0x6c, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67,
+ 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54,
+ 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x68, 0x6c, 0x63, 0x12, 0x30, 0x0a,
+ 0x04, 0x64, 0x65, 0x70, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77,
+ 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
+ 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x04, 0x64, 0x65, 0x70, 0x73, 0x12,
+ 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65,
+ 0x79, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61,
+ 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x44, 0x61,
+ 0x74, 0x61, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74,
+ 0x65, 0x78, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x74, 0x72, 0x61, 0x63, 0x65,
+ 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x22, 0x3e, 0x0a,
+ 0x0c, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2e, 0x0a,
0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79,
0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45,
- 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x2c, 0x0a,
- 0x03, 0x68, 0x6c, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f,
- 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d,
- 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x68, 0x6c, 0x63, 0x12, 0x29, 0x0a, 0x10, 0x69,
- 0x73, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x18,
- 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x73, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63,
- 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x64, 0x61, 0x74,
- 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x44, 0x61, 0x74, 0x61, 0x12,
- 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08,
- 0x52, 0x06, 0x69, 0x73, 0x42, 0x69, 0x6e, 0x64, 0x12, 0x35, 0x0a, 0x08, 0x62, 0x69, 0x6e, 0x64,
- 0x5f, 0x68, 0x6c, 0x63, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f,
- 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d,
- 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x62, 0x69, 0x6e, 0x64, 0x48, 0x6c, 0x63, 0x12,
- 0x41, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20,
- 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75,
- 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
- 0x6f, 0x6e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
- 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64,
- 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x49,
- 0x64, 0x12, 0x2b, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1b, 0x2e,
- 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76,
- 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x4f, 0x70, 0x52, 0x02, 0x6f, 0x70, 0x12, 0x30,
- 0x0a, 0x04, 0x64, 0x65, 0x70, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73,
- 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31,
- 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x04, 0x64, 0x65, 0x70, 0x73,
- 0x12, 0x20, 0x0a, 0x0c, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x69, 0x64,
- 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x4e, 0x6f, 0x64, 0x65,
- 0x49, 0x64, 0x12, 0x31, 0x0a, 0x15, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x66, 0x6f, 0x72, 0x6b, 0x5f,
- 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x0d, 0x20, 0x01, 0x28,
- 0x0c, 0x52, 0x12, 0x62, 0x69, 0x6e, 0x64, 0x46, 0x6f, 0x72, 0x6b, 0x43, 0x68, 0x6f, 0x69, 0x63,
- 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x50, 0x0a, 0x12, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x63, 0x6f,
- 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x70, 0x73, 0x18, 0x0e, 0x20, 0x03, 0x28,
- 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74,
- 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65,
- 0x64, 0x54, 0x69, 0x70, 0x73, 0x52, 0x10, 0x62, 0x69, 0x6e, 0x64, 0x43, 0x6f, 0x6e, 0x73, 0x75,
- 0x6d, 0x65, 0x64, 0x54, 0x69, 0x70, 0x73, 0x22, 0x66, 0x0a, 0x0f, 0x4b, 0x65, 0x79, 0x43, 0x6f,
- 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x54, 0x69, 0x70, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65,
- 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x41, 0x0a, 0x0d,
- 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x70, 0x73, 0x18, 0x02, 0x20,
- 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75,
- 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65,
- 0x66, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x54, 0x69, 0x70, 0x73, 0x22,
- 0x8a, 0x03, 0x0a, 0x0a, 0x4e, 0x61, 0x63, 0x6b, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x12, 0x10,
- 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79,
- 0x12, 0x3e, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67, 0x18,
- 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63,
- 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74,
- 0x52, 0x65, 0x66, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x69, 0x6e, 0x67,
- 0x12, 0x30, 0x0a, 0x04, 0x74, 0x69, 0x70, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c,
+ 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x60, 0x0a,
+ 0x0d, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2e,
+ 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77,
+ 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
+ 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65, 0x66, 0x12, 0x1f,
+ 0x0a, 0x0b, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x02, 0x20,
+ 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x65, 0x66, 0x66, 0x65, 0x63, 0x74, 0x44, 0x61, 0x74, 0x61, 0x22,
+ 0xe9, 0x04, 0x0a, 0x0d, 0x4e, 0x61, 0x63, 0x6b, 0x54, 0x69, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69,
+ 0x6c, 0x12, 0x2e, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c,
0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
- 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x04, 0x74, 0x69,
- 0x70, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x74, 0x69, 0x70, 0x5f, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c,
- 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
- 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4e, 0x61, 0x63, 0x6b,
- 0x54, 0x69, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x0a, 0x74, 0x69, 0x70, 0x44, 0x65,
- 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x6f, 0x74, 0x5f, 0x73, 0x75, 0x62,
- 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0d, 0x6e,
- 0x6f, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x64, 0x12, 0x3f, 0x0a, 0x0c,
- 0x63, 0x61, 0x75, 0x73, 0x61, 0x6c, 0x5f, 0x63, 0x68, 0x61, 0x69, 0x6e, 0x18, 0x06, 0x20, 0x03,
+ 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x03, 0x72, 0x65,
+ 0x66, 0x12, 0x2c, 0x0a, 0x03, 0x68, 0x6c, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
+ 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+ 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x03, 0x68, 0x6c, 0x63, 0x12,
+ 0x29, 0x0a, 0x10, 0x69, 0x73, 0x5f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f,
+ 0x6e, 0x61, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x69, 0x73, 0x54, 0x72, 0x61,
+ 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73,
+ 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x44,
+ 0x61, 0x74, 0x61, 0x12, 0x17, 0x0a, 0x07, 0x69, 0x73, 0x5f, 0x62, 0x69, 0x6e, 0x64, 0x18, 0x06,
+ 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x69, 0x73, 0x42, 0x69, 0x6e, 0x64, 0x12, 0x35, 0x0a, 0x08,
+ 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x68, 0x6c, 0x63, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a,
+ 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66,
+ 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x07, 0x62, 0x69, 0x6e, 0x64,
+ 0x48, 0x6c, 0x63, 0x12, 0x41, 0x0a, 0x0a, 0x63, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+ 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
+ 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6c, 0x6c,
+ 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x69, 0x6e, 0x64, 0x52, 0x0a, 0x63, 0x6f, 0x6c, 0x6c,
+ 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x6c, 0x65, 0x6d, 0x65, 0x6e,
+ 0x74, 0x5f, 0x69, 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x65, 0x6c, 0x65, 0x6d,
+ 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x2b, 0x0a, 0x02, 0x6f, 0x70, 0x18, 0x0a, 0x20, 0x01, 0x28,
+ 0x0e, 0x32, 0x1b, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74,
+ 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x4f, 0x70, 0x52, 0x02,
+ 0x6f, 0x70, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x65, 0x70, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x0b,
+ 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
+ 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x04,
+ 0x64, 0x65, 0x70, 0x73, 0x12, 0x20, 0x0a, 0x0c, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x64,
+ 0x65, 0x5f, 0x69, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x62, 0x69, 0x6e, 0x64,
+ 0x4e, 0x6f, 0x64, 0x65, 0x49, 0x64, 0x12, 0x31, 0x0a, 0x15, 0x62, 0x69, 0x6e, 0x64, 0x5f, 0x66,
+ 0x6f, 0x72, 0x6b, 0x5f, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18,
+ 0x0d, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x12, 0x62, 0x69, 0x6e, 0x64, 0x46, 0x6f, 0x72, 0x6b, 0x43,
+ 0x68, 0x6f, 0x69, 0x63, 0x65, 0x48, 0x61, 0x73, 0x68, 0x12, 0x50, 0x0a, 0x12, 0x62, 0x69, 0x6e,
+ 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x70, 0x73, 0x18,
+ 0x0e, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x22, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63,
+ 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4b, 0x65, 0x79, 0x43, 0x6f, 0x6e,
+ 0x73, 0x75, 0x6d, 0x65, 0x64, 0x54, 0x69, 0x70, 0x73, 0x52, 0x10, 0x62, 0x69, 0x6e, 0x64, 0x43,
+ 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x54, 0x69, 0x70, 0x73, 0x22, 0x66, 0x0a, 0x0f, 0x4b,
+ 0x65, 0x79, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x54, 0x69, 0x70, 0x73, 0x12, 0x10,
+ 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79,
+ 0x12, 0x41, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x5f, 0x74, 0x69, 0x70,
+ 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
+ 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65,
+ 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x54,
+ 0x69, 0x70, 0x73, 0x22, 0x8a, 0x03, 0x0a, 0x0a, 0x4e, 0x61, 0x63, 0x6b, 0x4e, 0x6f, 0x74, 0x69,
+ 0x66, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
+ 0x03, 0x6b, 0x65, 0x79, 0x12, 0x3e, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74,
+ 0x69, 0x6e, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74,
+ 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66,
+ 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63,
+ 0x74, 0x69, 0x6e, 0x67, 0x12, 0x30, 0x0a, 0x04, 0x74, 0x69, 0x70, 0x73, 0x18, 0x03, 0x20, 0x03,
0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73,
0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x52, 0x65, 0x66,
- 0x52, 0x0b, 0x63, 0x61, 0x75, 0x73, 0x61, 0x6c, 0x43, 0x68, 0x61, 0x69, 0x6e, 0x12, 0x26, 0x0a,
- 0x0f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72,
- 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x6e, 0x6f, 0x64, 0x65, 0x4b, 0x65, 0x79, 0x46,
- 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x5f,
- 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0d, 0x66,
- 0x69, 0x6c, 0x74, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xc4, 0x01, 0x0a,
- 0x14, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x61,
- 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64,
- 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68,
- 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x6f, 0x72, 0x77,
- 0x61, 0x72, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x08, 0x63, 0x6f,
- 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x77, 0x61, 0x74, 0x63, 0x68, 0x65,
- 0x64, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x0b, 0x77, 0x61,
- 0x74, 0x63, 0x68, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x72, 0x69,
- 0x67, 0x69, 0x6e, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a,
- 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x27, 0x0a, 0x0f, 0x61, 0x75,
- 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x75, 0x73, 0x65, 0x72, 0x18, 0x05, 0x20,
- 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x55,
- 0x73, 0x65, 0x72, 0x22, 0x49, 0x0a, 0x10, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64,
- 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6d, 0x6d, 0x61,
- 0x6e, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0b, 0x63,
- 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x61, 0x72,
- 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x61, 0x72, 0x67, 0x73, 0x22, 0x6b,
- 0x0a, 0x11, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f,
- 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f, 0x64, 0x61, 0x74, 0x61,
- 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70, 0x44, 0x61, 0x74, 0x61,
- 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52,
- 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f,
- 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65,
- 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2a, 0x38, 0x0a, 0x08, 0x45,
- 0x66, 0x66, 0x65, 0x63, 0x74, 0x4f, 0x70, 0x12, 0x0e, 0x0a, 0x0a, 0x55, 0x4e, 0x4b, 0x4e, 0x4f,
- 0x57, 0x4e, 0x5f, 0x4f, 0x50, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x4e, 0x53, 0x45, 0x52,
- 0x54, 0x5f, 0x4f, 0x50, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45,
- 0x5f, 0x4f, 0x50, 0x10, 0x02, 0x2a, 0x78, 0x0a, 0x09, 0x4d, 0x65, 0x72, 0x67, 0x65, 0x52, 0x75,
- 0x6c, 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x4c, 0x41, 0x53, 0x54, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45,
- 0x5f, 0x57, 0x49, 0x4e, 0x53, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41, 0x44, 0x44, 0x49, 0x54,
- 0x49, 0x56, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x41, 0x44, 0x44,
- 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x02, 0x12, 0x0b, 0x0a,
- 0x07, 0x4d, 0x41, 0x58, 0x5f, 0x49, 0x4e, 0x54, 0x10, 0x03, 0x12, 0x14, 0x0a, 0x10, 0x46, 0x49,
- 0x52, 0x53, 0x54, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x57, 0x49, 0x4e, 0x53, 0x10, 0x04,
- 0x12, 0x0d, 0x0a, 0x09, 0x4d, 0x41, 0x58, 0x5f, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x05, 0x2a,
- 0x34, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4b, 0x69, 0x6e,
- 0x64, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x43, 0x41, 0x4c, 0x41, 0x52, 0x10, 0x00, 0x12, 0x09, 0x0a,
- 0x05, 0x4b, 0x45, 0x59, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x4f, 0x52, 0x44, 0x45,
- 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x6e, 0x0a, 0x09, 0x50, 0x6c, 0x61, 0x63, 0x65, 0x6d, 0x65,
- 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x4e, 0x4f, 0x4e, 0x45,
- 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x48, 0x45, 0x41, 0x44,
- 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x54, 0x41, 0x49, 0x4c,
- 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x41, 0x46, 0x54, 0x45,
- 0x52, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x42, 0x45, 0x46,
- 0x4f, 0x52, 0x45, 0x10, 0x04, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f, 0x53,
- 0x45, 0x4c, 0x46, 0x10, 0x05, 0x2a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73,
- 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53,
- 0x49, 0x4f, 0x4e, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x4f,
- 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x5a, 0x53, 0x54, 0x44, 0x10, 0x01,
- 0x2a, 0xe5, 0x01, 0x0a, 0x09, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x14,
- 0x0a, 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49,
- 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x52,
- 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4c, 0x49,
- 0x53, 0x54, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x41, 0x53,
- 0x48, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x45, 0x54, 0x10,
- 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5a, 0x53, 0x45, 0x54, 0x10, 0x05,
- 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x45, 0x41, 0x4d, 0x10,
- 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x4c, 0x4c, 0x10, 0x07, 0x12,
- 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x4d, 0x41, 0x50, 0x10, 0x08,
- 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x4f, 0x10, 0x09, 0x12, 0x0d,
- 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x10, 0x0a, 0x12, 0x14, 0x0a,
- 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x4f, 0x42, 0x4a, 0x45, 0x43,
- 0x54, 0x10, 0x0b, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e,
- 0x5f, 0x41, 0x52, 0x52, 0x41, 0x59, 0x10, 0x0c, 0x2a, 0x35, 0x0a, 0x07, 0x56, 0x65, 0x72, 0x64,
- 0x69, 0x63, 0x74, 0x12, 0x17, 0x0a, 0x13, 0x56, 0x45, 0x52, 0x44, 0x49, 0x43, 0x54, 0x5f, 0x55,
- 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x07, 0x0a, 0x03,
- 0x57, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x4f, 0x53, 0x54, 0x10, 0x02, 0x2a,
- 0x4c, 0x0a, 0x0e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65, 0x43, 0x6d, 0x70, 0x4f,
- 0x70, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x45, 0x51, 0x10, 0x00, 0x12, 0x0a, 0x0a,
- 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x4c, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50,
- 0x5f, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x47, 0x54, 0x10,
- 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x47, 0x45, 0x10, 0x04, 0x32, 0xaf, 0x01,
- 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x54, 0x0a,
- 0x0c, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x1f, 0x2e,
- 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76,
- 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x1a, 0x1f,
- 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e,
- 0x76, 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x28,
- 0x01, 0x30, 0x01, 0x12, 0x4a, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68, 0x12, 0x1f, 0x2e, 0x73,
- 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31,
- 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e,
- 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76,
- 0x31, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42,
- 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x77,
- 0x79, 0x74, 0x63, 0x68, 0x64, 0x62, 0x2f, 0x65, 0x6e, 0x67, 0x69, 0x6e, 0x65, 0x2f, 0x63, 0x6c,
- 0x75, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f,
- 0x74, 0x6f, 0x33,
+ 0x52, 0x04, 0x74, 0x69, 0x70, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x74, 0x69, 0x70, 0x5f, 0x64, 0x65,
+ 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x73, 0x77,
+ 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
+ 0x4e, 0x61, 0x63, 0x6b, 0x54, 0x69, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x52, 0x0a, 0x74,
+ 0x69, 0x70, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x6f, 0x74,
+ 0x5f, 0x73, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28,
+ 0x08, 0x52, 0x0d, 0x6e, 0x6f, 0x74, 0x53, 0x75, 0x62, 0x73, 0x63, 0x72, 0x69, 0x62, 0x65, 0x64,
+ 0x12, 0x3f, 0x0a, 0x0c, 0x63, 0x61, 0x75, 0x73, 0x61, 0x6c, 0x5f, 0x63, 0x68, 0x61, 0x69, 0x6e,
+ 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e,
+ 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x66, 0x66, 0x65, 0x63,
+ 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x63, 0x61, 0x75, 0x73, 0x61, 0x6c, 0x43, 0x68, 0x61, 0x69,
+ 0x6e, 0x12, 0x26, 0x0a, 0x0f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69,
+ 0x6c, 0x74, 0x65, 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0d, 0x6e, 0x6f, 0x64, 0x65,
+ 0x4b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x25, 0x0a, 0x0e, 0x66, 0x69, 0x6c,
+ 0x74, 0x65, 0x72, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28,
+ 0x04, 0x52, 0x0d, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e,
+ 0x22, 0xc4, 0x01, 0x0a, 0x14, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x54, 0x72,
+ 0x61, 0x6e, 0x73, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x3f, 0x0a, 0x08, 0x63, 0x6f, 0x6d,
+ 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x73, 0x77,
+ 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e,
+ 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64,
+ 0x52, 0x08, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x77, 0x61,
+ 0x74, 0x63, 0x68, 0x65, 0x64, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c,
+ 0x52, 0x0b, 0x77, 0x61, 0x74, 0x63, 0x68, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x1f, 0x0a,
+ 0x0b, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01,
+ 0x28, 0x04, 0x52, 0x0a, 0x6f, 0x72, 0x69, 0x67, 0x69, 0x6e, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x27,
+ 0x0a, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x5f, 0x75, 0x73, 0x65,
+ 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69,
+ 0x7a, 0x65, 0x64, 0x55, 0x73, 0x65, 0x72, 0x22, 0x49, 0x0a, 0x10, 0x46, 0x6f, 0x72, 0x77, 0x61,
+ 0x72, 0x64, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63,
+ 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28,
+ 0x0c, 0x52, 0x0b, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x12,
+ 0x0a, 0x04, 0x61, 0x72, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x61, 0x72,
+ 0x67, 0x73, 0x22, 0x6b, 0x0a, 0x11, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x65, 0x64, 0x52,
+ 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x65, 0x73, 0x70, 0x5f,
+ 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x72, 0x65, 0x73, 0x70,
+ 0x44, 0x61, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20,
+ 0x01, 0x28, 0x08, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x23, 0x0a, 0x0d, 0x65, 0x72,
+ 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28,
+ 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x2a,
+ 0x38, 0x0a, 0x08, 0x45, 0x66, 0x66, 0x65, 0x63, 0x74, 0x4f, 0x70, 0x12, 0x0e, 0x0a, 0x0a, 0x55,
+ 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x5f, 0x4f, 0x50, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x49,
+ 0x4e, 0x53, 0x45, 0x52, 0x54, 0x5f, 0x4f, 0x50, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45,
+ 0x4d, 0x4f, 0x56, 0x45, 0x5f, 0x4f, 0x50, 0x10, 0x02, 0x2a, 0x78, 0x0a, 0x09, 0x4d, 0x65, 0x72,
+ 0x67, 0x65, 0x52, 0x75, 0x6c, 0x65, 0x12, 0x13, 0x0a, 0x0f, 0x4c, 0x41, 0x53, 0x54, 0x5f, 0x57,
+ 0x52, 0x49, 0x54, 0x45, 0x5f, 0x57, 0x49, 0x4e, 0x53, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x41,
+ 0x44, 0x44, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x10, 0x01, 0x12, 0x12, 0x0a,
+ 0x0e, 0x41, 0x44, 0x44, 0x49, 0x54, 0x49, 0x56, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10,
+ 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x4d, 0x41, 0x58, 0x5f, 0x49, 0x4e, 0x54, 0x10, 0x03, 0x12, 0x14,
+ 0x0a, 0x10, 0x46, 0x49, 0x52, 0x53, 0x54, 0x5f, 0x57, 0x52, 0x49, 0x54, 0x45, 0x5f, 0x57, 0x49,
+ 0x4e, 0x53, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x4d, 0x41, 0x58, 0x5f, 0x42, 0x59, 0x54, 0x45,
+ 0x53, 0x10, 0x05, 0x2a, 0x34, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
+ 0x6e, 0x4b, 0x69, 0x6e, 0x64, 0x12, 0x0a, 0x0a, 0x06, 0x53, 0x43, 0x41, 0x4c, 0x41, 0x52, 0x10,
+ 0x00, 0x12, 0x09, 0x0a, 0x05, 0x4b, 0x45, 0x59, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07,
+ 0x4f, 0x52, 0x44, 0x45, 0x52, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x6e, 0x0a, 0x09, 0x50, 0x6c, 0x61,
+ 0x63, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f,
+ 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f,
+ 0x48, 0x45, 0x41, 0x44, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f,
+ 0x54, 0x41, 0x49, 0x4c, 0x10, 0x02, 0x12, 0x0f, 0x0a, 0x0b, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x5f,
+ 0x41, 0x46, 0x54, 0x45, 0x52, 0x10, 0x03, 0x12, 0x10, 0x0a, 0x0c, 0x50, 0x4c, 0x41, 0x43, 0x45,
+ 0x5f, 0x42, 0x45, 0x46, 0x4f, 0x52, 0x45, 0x10, 0x04, 0x12, 0x0e, 0x0a, 0x0a, 0x50, 0x4c, 0x41,
+ 0x43, 0x45, 0x5f, 0x53, 0x45, 0x4c, 0x46, 0x10, 0x05, 0x2a, 0x39, 0x0a, 0x0b, 0x43, 0x6f, 0x6d,
+ 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x10, 0x43, 0x4f, 0x4d, 0x50,
+ 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x14,
+ 0x0a, 0x10, 0x43, 0x4f, 0x4d, 0x50, 0x52, 0x45, 0x53, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x5a, 0x53,
+ 0x54, 0x44, 0x10, 0x01, 0x2a, 0xe5, 0x01, 0x0a, 0x09, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x54, 0x79,
+ 0x70, 0x65, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45,
+ 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45,
+ 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50,
+ 0x45, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45,
+ 0x5f, 0x48, 0x41, 0x53, 0x48, 0x10, 0x03, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x59, 0x50, 0x45, 0x5f,
+ 0x53, 0x45, 0x54, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x5a, 0x53,
+ 0x45, 0x54, 0x10, 0x05, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x52,
+ 0x45, 0x41, 0x4d, 0x10, 0x06, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x48, 0x4c,
+ 0x4c, 0x10, 0x07, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x49, 0x54, 0x4d,
+ 0x41, 0x50, 0x10, 0x08, 0x12, 0x0c, 0x0a, 0x08, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x45, 0x4f,
+ 0x10, 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x10,
+ 0x0a, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x4f,
+ 0x42, 0x4a, 0x45, 0x43, 0x54, 0x10, 0x0b, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x59, 0x50, 0x45, 0x5f,
+ 0x4a, 0x53, 0x4f, 0x4e, 0x5f, 0x41, 0x52, 0x52, 0x41, 0x59, 0x10, 0x0c, 0x2a, 0x35, 0x0a, 0x07,
+ 0x56, 0x65, 0x72, 0x64, 0x69, 0x63, 0x74, 0x12, 0x17, 0x0a, 0x13, 0x56, 0x45, 0x52, 0x44, 0x49,
+ 0x43, 0x54, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00,
+ 0x12, 0x07, 0x0a, 0x03, 0x57, 0x4f, 0x4e, 0x10, 0x01, 0x12, 0x08, 0x0a, 0x04, 0x4c, 0x4f, 0x53,
+ 0x54, 0x10, 0x02, 0x2a, 0x4c, 0x0a, 0x0e, 0x50, 0x72, 0x65, 0x64, 0x69, 0x63, 0x61, 0x74, 0x65,
+ 0x43, 0x6d, 0x70, 0x4f, 0x70, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x45, 0x51, 0x10,
+ 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x4c, 0x54, 0x10, 0x01, 0x12, 0x0a, 0x0a,
+ 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x4c, 0x45, 0x10, 0x02, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50,
+ 0x5f, 0x47, 0x54, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x43, 0x4d, 0x50, 0x5f, 0x47, 0x45, 0x10,
+ 0x04, 0x32, 0xaf, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f,
+ 0x6e, 0x12, 0x54, 0x0a, 0x0c, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x53, 0x74, 0x72, 0x65, 0x61,
+ 0x6d, 0x12, 0x1f, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74,
+ 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x4e, 0x6f, 0x74, 0x69,
+ 0x66, 0x79, 0x1a, 0x1f, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73,
+ 0x74, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x4e, 0x6f, 0x74,
+ 0x69, 0x66, 0x79, 0x28, 0x01, 0x30, 0x01, 0x12, 0x4a, 0x0a, 0x05, 0x46, 0x65, 0x74, 0x63, 0x68,
+ 0x12, 0x1f, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
+ 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
+ 0x74, 0x1a, 0x20, 0x2e, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x2e, 0x63, 0x6c, 0x75, 0x73, 0x74,
+ 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x65, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f,
+ 0x6e, 0x73, 0x65, 0x42, 0x2a, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
+ 0x6d, 0x2f, 0x73, 0x77, 0x79, 0x74, 0x63, 0x68, 0x64, 0x62, 0x2f, 0x65, 0x6e, 0x67, 0x69, 0x6e,
+ 0x65, 0x2f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62,
+ 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -3526,7 +3683,7 @@ func file_cluster_proto_replication_proto_rawDescGZIP() []byte {
}
var file_cluster_proto_replication_proto_enumTypes = make([]protoimpl.EnumInfo, 11)
-var file_cluster_proto_replication_proto_msgTypes = make([]protoimpl.MessageInfo, 32)
+var file_cluster_proto_replication_proto_msgTypes = make([]protoimpl.MessageInfo, 34)
var file_cluster_proto_replication_proto_goTypes = []interface{}{
(EffectOp)(0), // 0: swytch.cluster.v1.EffectOp
(MergeRule)(0), // 1: swytch.cluster.v1.MergeRule
@@ -3548,30 +3705,32 @@ var file_cluster_proto_replication_proto_goTypes = []interface{}{
(*ReducedElement)(nil), // 17: swytch.cluster.v1.ReducedElement
(*ReducedEffect)(nil), // 18: swytch.cluster.v1.ReducedEffect
(*SnapshotEffect)(nil), // 19: swytch.cluster.v1.SnapshotEffect
- (*TypedValue)(nil), // 20: swytch.cluster.v1.TypedValue
- (*Predicate)(nil), // 21: swytch.cluster.v1.Predicate
- (*ObservationEffect)(nil), // 22: swytch.cluster.v1.ObservationEffect
- (*RowWriteEffect)(nil), // 23: swytch.cluster.v1.RowWriteEffect
- (*SubscriptionEffect)(nil), // 24: swytch.cluster.v1.SubscriptionEffect
- (*PubSubMessage)(nil), // 25: swytch.cluster.v1.PubSubMessage
- (*SerializationEffect)(nil), // 26: swytch.cluster.v1.SerializationEffect
- (*Effect)(nil), // 27: swytch.cluster.v1.Effect
- (*OffsetNotify)(nil), // 28: swytch.cluster.v1.OffsetNotify
- (*FetchRequest)(nil), // 29: swytch.cluster.v1.FetchRequest
- (*FetchResponse)(nil), // 30: swytch.cluster.v1.FetchResponse
- (*NackTipDetail)(nil), // 31: swytch.cluster.v1.NackTipDetail
- (*KeyConsumedTips)(nil), // 32: swytch.cluster.v1.KeyConsumedTips
- (*NackNotify)(nil), // 33: swytch.cluster.v1.NackNotify
- (*ForwardedTransaction)(nil), // 34: swytch.cluster.v1.ForwardedTransaction
- (*ForwardedCommand)(nil), // 35: swytch.cluster.v1.ForwardedCommand
- (*ForwardedResponse)(nil), // 36: swytch.cluster.v1.ForwardedResponse
- (*TransactionalBindEffect_KeyBind)(nil), // 37: swytch.cluster.v1.TransactionalBindEffect.KeyBind
- nil, // 38: swytch.cluster.v1.ReducedEffect.NetAddsEntry
- nil, // 39: swytch.cluster.v1.ReducedEffect.NetRemovesEntry
- nil, // 40: swytch.cluster.v1.ReducedEffect.SubscribersEntry
- nil, // 41: swytch.cluster.v1.ReducedEffect.PartitionsEntry
- nil, // 42: swytch.cluster.v1.SnapshotEffect.TxnVerdictsEntry
- (*timestamppb.Timestamp)(nil), // 43: google.protobuf.Timestamp
+ (*BindEvidence)(nil), // 20: swytch.cluster.v1.BindEvidence
+ (*TypedValue)(nil), // 21: swytch.cluster.v1.TypedValue
+ (*Predicate)(nil), // 22: swytch.cluster.v1.Predicate
+ (*ObservationEffect)(nil), // 23: swytch.cluster.v1.ObservationEffect
+ (*RowWriteEffect)(nil), // 24: swytch.cluster.v1.RowWriteEffect
+ (*SubscriptionEffect)(nil), // 25: swytch.cluster.v1.SubscriptionEffect
+ (*PubSubMessage)(nil), // 26: swytch.cluster.v1.PubSubMessage
+ (*SerializationEffect)(nil), // 27: swytch.cluster.v1.SerializationEffect
+ (*EvidenceRecord)(nil), // 28: swytch.cluster.v1.EvidenceRecord
+ (*Effect)(nil), // 29: swytch.cluster.v1.Effect
+ (*OffsetNotify)(nil), // 30: swytch.cluster.v1.OffsetNotify
+ (*FetchRequest)(nil), // 31: swytch.cluster.v1.FetchRequest
+ (*FetchResponse)(nil), // 32: swytch.cluster.v1.FetchResponse
+ (*NackTipDetail)(nil), // 33: swytch.cluster.v1.NackTipDetail
+ (*KeyConsumedTips)(nil), // 34: swytch.cluster.v1.KeyConsumedTips
+ (*NackNotify)(nil), // 35: swytch.cluster.v1.NackNotify
+ (*ForwardedTransaction)(nil), // 36: swytch.cluster.v1.ForwardedTransaction
+ (*ForwardedCommand)(nil), // 37: swytch.cluster.v1.ForwardedCommand
+ (*ForwardedResponse)(nil), // 38: swytch.cluster.v1.ForwardedResponse
+ (*TransactionalBindEffect_KeyBind)(nil), // 39: swytch.cluster.v1.TransactionalBindEffect.KeyBind
+ nil, // 40: swytch.cluster.v1.ReducedEffect.NetAddsEntry
+ nil, // 41: swytch.cluster.v1.ReducedEffect.NetRemovesEntry
+ nil, // 42: swytch.cluster.v1.ReducedEffect.SubscribersEntry
+ nil, // 43: swytch.cluster.v1.ReducedEffect.PartitionsEntry
+ nil, // 44: swytch.cluster.v1.SnapshotEffect.TxnVerdictsEntry
+ (*timestamppb.Timestamp)(nil), // 45: google.protobuf.Timestamp
}
var file_cluster_proto_replication_proto_depIdxs = []int32{
4, // 0: swytch.cluster.v1.CompressedValue.codec:type_name -> swytch.cluster.v1.Compression
@@ -3580,83 +3739,88 @@ var file_cluster_proto_replication_proto_depIdxs = []int32{
2, // 3: swytch.cluster.v1.DataEffect.collection:type_name -> swytch.cluster.v1.CollectionKind
3, // 4: swytch.cluster.v1.DataEffect.placement:type_name -> swytch.cluster.v1.Placement
11, // 5: swytch.cluster.v1.DataEffect.compressed:type_name -> swytch.cluster.v1.CompressedValue
- 43, // 6: swytch.cluster.v1.MetaEffect.expires_at:type_name -> google.protobuf.Timestamp
+ 45, // 6: swytch.cluster.v1.MetaEffect.expires_at:type_name -> google.protobuf.Timestamp
5, // 7: swytch.cluster.v1.MetaEffect.type_tag:type_name -> swytch.cluster.v1.ValueType
- 37, // 8: swytch.cluster.v1.TransactionalBindEffect.keys:type_name -> swytch.cluster.v1.TransactionalBindEffect.KeyBind
- 43, // 9: swytch.cluster.v1.TransactionalBindEffect.txn_hlc:type_name -> google.protobuf.Timestamp
+ 39, // 8: swytch.cluster.v1.TransactionalBindEffect.keys:type_name -> swytch.cluster.v1.TransactionalBindEffect.KeyBind
+ 45, // 9: swytch.cluster.v1.TransactionalBindEffect.txn_hlc:type_name -> google.protobuf.Timestamp
15, // 10: swytch.cluster.v1.TransactionalBindEffect.abort_deps:type_name -> swytch.cluster.v1.EffectRef
12, // 11: swytch.cluster.v1.ReducedElement.data:type_name -> swytch.cluster.v1.DataEffect
- 43, // 12: swytch.cluster.v1.ReducedElement.hlc:type_name -> google.protobuf.Timestamp
- 43, // 13: swytch.cluster.v1.ReducedElement.expires_at:type_name -> google.protobuf.Timestamp
+ 45, // 12: swytch.cluster.v1.ReducedElement.hlc:type_name -> google.protobuf.Timestamp
+ 45, // 13: swytch.cluster.v1.ReducedElement.expires_at:type_name -> google.protobuf.Timestamp
0, // 14: swytch.cluster.v1.ReducedEffect.op:type_name -> swytch.cluster.v1.EffectOp
1, // 15: swytch.cluster.v1.ReducedEffect.merge:type_name -> swytch.cluster.v1.MergeRule
2, // 16: swytch.cluster.v1.ReducedEffect.collection:type_name -> swytch.cluster.v1.CollectionKind
- 43, // 17: swytch.cluster.v1.ReducedEffect.hlc:type_name -> google.protobuf.Timestamp
+ 45, // 17: swytch.cluster.v1.ReducedEffect.hlc:type_name -> google.protobuf.Timestamp
12, // 18: swytch.cluster.v1.ReducedEffect.scalar:type_name -> swytch.cluster.v1.DataEffect
- 38, // 19: swytch.cluster.v1.ReducedEffect.net_adds:type_name -> swytch.cluster.v1.ReducedEffect.NetAddsEntry
- 39, // 20: swytch.cluster.v1.ReducedEffect.net_removes:type_name -> swytch.cluster.v1.ReducedEffect.NetRemovesEntry
+ 40, // 19: swytch.cluster.v1.ReducedEffect.net_adds:type_name -> swytch.cluster.v1.ReducedEffect.NetAddsEntry
+ 41, // 20: swytch.cluster.v1.ReducedEffect.net_removes:type_name -> swytch.cluster.v1.ReducedEffect.NetRemovesEntry
17, // 21: swytch.cluster.v1.ReducedEffect.ordered_elements:type_name -> swytch.cluster.v1.ReducedElement
5, // 22: swytch.cluster.v1.ReducedEffect.type_tag:type_name -> swytch.cluster.v1.ValueType
- 43, // 23: swytch.cluster.v1.ReducedEffect.expires_at:type_name -> google.protobuf.Timestamp
- 40, // 24: swytch.cluster.v1.ReducedEffect.subscribers:type_name -> swytch.cluster.v1.ReducedEffect.SubscribersEntry
- 41, // 25: swytch.cluster.v1.ReducedEffect.partitions:type_name -> swytch.cluster.v1.ReducedEffect.PartitionsEntry
+ 45, // 23: swytch.cluster.v1.ReducedEffect.expires_at:type_name -> google.protobuf.Timestamp
+ 42, // 24: swytch.cluster.v1.ReducedEffect.subscribers:type_name -> swytch.cluster.v1.ReducedEffect.SubscribersEntry
+ 43, // 25: swytch.cluster.v1.ReducedEffect.partitions:type_name -> swytch.cluster.v1.ReducedEffect.PartitionsEntry
2, // 26: swytch.cluster.v1.SnapshotEffect.collection:type_name -> swytch.cluster.v1.CollectionKind
18, // 27: swytch.cluster.v1.SnapshotEffect.state:type_name -> swytch.cluster.v1.ReducedEffect
- 42, // 28: swytch.cluster.v1.SnapshotEffect.txn_verdicts:type_name -> swytch.cluster.v1.SnapshotEffect.TxnVerdictsEntry
+ 44, // 28: swytch.cluster.v1.SnapshotEffect.txn_verdicts:type_name -> swytch.cluster.v1.SnapshotEffect.TxnVerdictsEntry
15, // 29: swytch.cluster.v1.SnapshotEffect.prev_snapshot:type_name -> swytch.cluster.v1.EffectRef
- 8, // 30: swytch.cluster.v1.TypedValue.kind:type_name -> swytch.cluster.v1.TypedValue.Kind
- 9, // 31: swytch.cluster.v1.Predicate.kind:type_name -> swytch.cluster.v1.Predicate.Kind
- 21, // 32: swytch.cluster.v1.Predicate.children:type_name -> swytch.cluster.v1.Predicate
- 21, // 33: swytch.cluster.v1.Predicate.child:type_name -> swytch.cluster.v1.Predicate
- 7, // 34: swytch.cluster.v1.Predicate.op:type_name -> swytch.cluster.v1.PredicateCmpOp
- 20, // 35: swytch.cluster.v1.Predicate.literal:type_name -> swytch.cluster.v1.TypedValue
- 21, // 36: swytch.cluster.v1.ObservationEffect.predicate:type_name -> swytch.cluster.v1.Predicate
- 10, // 37: swytch.cluster.v1.RowWriteEffect.kind:type_name -> swytch.cluster.v1.RowWriteEffect.Kind
- 20, // 38: swytch.cluster.v1.RowWriteEffect.columns:type_name -> swytch.cluster.v1.TypedValue
- 43, // 39: swytch.cluster.v1.Effect.hlc:type_name -> google.protobuf.Timestamp
- 15, // 40: swytch.cluster.v1.Effect.deps:type_name -> swytch.cluster.v1.EffectRef
- 12, // 41: swytch.cluster.v1.Effect.data:type_name -> swytch.cluster.v1.DataEffect
- 13, // 42: swytch.cluster.v1.Effect.meta:type_name -> swytch.cluster.v1.MetaEffect
- 16, // 43: swytch.cluster.v1.Effect.txn_bind:type_name -> swytch.cluster.v1.TransactionalBindEffect
- 19, // 44: swytch.cluster.v1.Effect.snapshot:type_name -> swytch.cluster.v1.SnapshotEffect
- 24, // 45: swytch.cluster.v1.Effect.subscription:type_name -> swytch.cluster.v1.SubscriptionEffect
- 26, // 46: swytch.cluster.v1.Effect.serialization:type_name -> swytch.cluster.v1.SerializationEffect
- 14, // 47: swytch.cluster.v1.Effect.noop:type_name -> swytch.cluster.v1.NoopEffect
- 22, // 48: swytch.cluster.v1.Effect.observation:type_name -> swytch.cluster.v1.ObservationEffect
- 23, // 49: swytch.cluster.v1.Effect.row_write:type_name -> swytch.cluster.v1.RowWriteEffect
- 25, // 50: swytch.cluster.v1.Effect.pubsub_message:type_name -> swytch.cluster.v1.PubSubMessage
- 15, // 51: swytch.cluster.v1.OffsetNotify.origin:type_name -> swytch.cluster.v1.EffectRef
- 43, // 52: swytch.cluster.v1.OffsetNotify.hlc:type_name -> google.protobuf.Timestamp
- 15, // 53: swytch.cluster.v1.OffsetNotify.deps:type_name -> swytch.cluster.v1.EffectRef
- 15, // 54: swytch.cluster.v1.FetchRequest.ref:type_name -> swytch.cluster.v1.EffectRef
- 15, // 55: swytch.cluster.v1.FetchResponse.ref:type_name -> swytch.cluster.v1.EffectRef
- 15, // 56: swytch.cluster.v1.NackTipDetail.ref:type_name -> swytch.cluster.v1.EffectRef
- 43, // 57: swytch.cluster.v1.NackTipDetail.hlc:type_name -> google.protobuf.Timestamp
- 43, // 58: swytch.cluster.v1.NackTipDetail.bind_hlc:type_name -> google.protobuf.Timestamp
- 2, // 59: swytch.cluster.v1.NackTipDetail.collection:type_name -> swytch.cluster.v1.CollectionKind
- 0, // 60: swytch.cluster.v1.NackTipDetail.op:type_name -> swytch.cluster.v1.EffectOp
- 15, // 61: swytch.cluster.v1.NackTipDetail.deps:type_name -> swytch.cluster.v1.EffectRef
- 32, // 62: swytch.cluster.v1.NackTipDetail.bind_consumed_tips:type_name -> swytch.cluster.v1.KeyConsumedTips
- 15, // 63: swytch.cluster.v1.KeyConsumedTips.consumed_tips:type_name -> swytch.cluster.v1.EffectRef
- 15, // 64: swytch.cluster.v1.NackNotify.conflicting:type_name -> swytch.cluster.v1.EffectRef
- 15, // 65: swytch.cluster.v1.NackNotify.tips:type_name -> swytch.cluster.v1.EffectRef
- 31, // 66: swytch.cluster.v1.NackNotify.tip_details:type_name -> swytch.cluster.v1.NackTipDetail
- 15, // 67: swytch.cluster.v1.NackNotify.causal_chain:type_name -> swytch.cluster.v1.EffectRef
- 35, // 68: swytch.cluster.v1.ForwardedTransaction.commands:type_name -> swytch.cluster.v1.ForwardedCommand
- 15, // 69: swytch.cluster.v1.TransactionalBindEffect.KeyBind.consumed_tips:type_name -> swytch.cluster.v1.EffectRef
- 15, // 70: swytch.cluster.v1.TransactionalBindEffect.KeyBind.new_tip:type_name -> swytch.cluster.v1.EffectRef
- 17, // 71: swytch.cluster.v1.ReducedEffect.NetAddsEntry.value:type_name -> swytch.cluster.v1.ReducedElement
- 18, // 72: swytch.cluster.v1.ReducedEffect.PartitionsEntry.value:type_name -> swytch.cluster.v1.ReducedEffect
- 6, // 73: swytch.cluster.v1.SnapshotEffect.TxnVerdictsEntry.value:type_name -> swytch.cluster.v1.Verdict
- 28, // 74: swytch.cluster.v1.Replication.NotifyStream:input_type -> swytch.cluster.v1.OffsetNotify
- 29, // 75: swytch.cluster.v1.Replication.Fetch:input_type -> swytch.cluster.v1.FetchRequest
- 28, // 76: swytch.cluster.v1.Replication.NotifyStream:output_type -> swytch.cluster.v1.OffsetNotify
- 30, // 77: swytch.cluster.v1.Replication.Fetch:output_type -> swytch.cluster.v1.FetchResponse
- 76, // [76:78] is the sub-list for method output_type
- 74, // [74:76] is the sub-list for method input_type
- 74, // [74:74] is the sub-list for extension type_name
- 74, // [74:74] is the sub-list for extension extendee
- 0, // [0:74] is the sub-list for field type_name
+ 20, // 30: swytch.cluster.v1.SnapshotEffect.evidence:type_name -> swytch.cluster.v1.BindEvidence
+ 15, // 31: swytch.cluster.v1.BindEvidence.binds:type_name -> swytch.cluster.v1.EffectRef
+ 15, // 32: swytch.cluster.v1.BindEvidence.deps:type_name -> swytch.cluster.v1.EffectRef
+ 8, // 33: swytch.cluster.v1.TypedValue.kind:type_name -> swytch.cluster.v1.TypedValue.Kind
+ 9, // 34: swytch.cluster.v1.Predicate.kind:type_name -> swytch.cluster.v1.Predicate.Kind
+ 22, // 35: swytch.cluster.v1.Predicate.children:type_name -> swytch.cluster.v1.Predicate
+ 22, // 36: swytch.cluster.v1.Predicate.child:type_name -> swytch.cluster.v1.Predicate
+ 7, // 37: swytch.cluster.v1.Predicate.op:type_name -> swytch.cluster.v1.PredicateCmpOp
+ 21, // 38: swytch.cluster.v1.Predicate.literal:type_name -> swytch.cluster.v1.TypedValue
+ 22, // 39: swytch.cluster.v1.ObservationEffect.predicate:type_name -> swytch.cluster.v1.Predicate
+ 10, // 40: swytch.cluster.v1.RowWriteEffect.kind:type_name -> swytch.cluster.v1.RowWriteEffect.Kind
+ 21, // 41: swytch.cluster.v1.RowWriteEffect.columns:type_name -> swytch.cluster.v1.TypedValue
+ 15, // 42: swytch.cluster.v1.EvidenceRecord.ref:type_name -> swytch.cluster.v1.EffectRef
+ 45, // 43: swytch.cluster.v1.Effect.hlc:type_name -> google.protobuf.Timestamp
+ 15, // 44: swytch.cluster.v1.Effect.deps:type_name -> swytch.cluster.v1.EffectRef
+ 28, // 45: swytch.cluster.v1.Effect.evidence_package:type_name -> swytch.cluster.v1.EvidenceRecord
+ 12, // 46: swytch.cluster.v1.Effect.data:type_name -> swytch.cluster.v1.DataEffect
+ 13, // 47: swytch.cluster.v1.Effect.meta:type_name -> swytch.cluster.v1.MetaEffect
+ 16, // 48: swytch.cluster.v1.Effect.txn_bind:type_name -> swytch.cluster.v1.TransactionalBindEffect
+ 19, // 49: swytch.cluster.v1.Effect.snapshot:type_name -> swytch.cluster.v1.SnapshotEffect
+ 25, // 50: swytch.cluster.v1.Effect.subscription:type_name -> swytch.cluster.v1.SubscriptionEffect
+ 27, // 51: swytch.cluster.v1.Effect.serialization:type_name -> swytch.cluster.v1.SerializationEffect
+ 14, // 52: swytch.cluster.v1.Effect.noop:type_name -> swytch.cluster.v1.NoopEffect
+ 23, // 53: swytch.cluster.v1.Effect.observation:type_name -> swytch.cluster.v1.ObservationEffect
+ 24, // 54: swytch.cluster.v1.Effect.row_write:type_name -> swytch.cluster.v1.RowWriteEffect
+ 26, // 55: swytch.cluster.v1.Effect.pubsub_message:type_name -> swytch.cluster.v1.PubSubMessage
+ 15, // 56: swytch.cluster.v1.OffsetNotify.origin:type_name -> swytch.cluster.v1.EffectRef
+ 45, // 57: swytch.cluster.v1.OffsetNotify.hlc:type_name -> google.protobuf.Timestamp
+ 15, // 58: swytch.cluster.v1.OffsetNotify.deps:type_name -> swytch.cluster.v1.EffectRef
+ 15, // 59: swytch.cluster.v1.FetchRequest.ref:type_name -> swytch.cluster.v1.EffectRef
+ 15, // 60: swytch.cluster.v1.FetchResponse.ref:type_name -> swytch.cluster.v1.EffectRef
+ 15, // 61: swytch.cluster.v1.NackTipDetail.ref:type_name -> swytch.cluster.v1.EffectRef
+ 45, // 62: swytch.cluster.v1.NackTipDetail.hlc:type_name -> google.protobuf.Timestamp
+ 45, // 63: swytch.cluster.v1.NackTipDetail.bind_hlc:type_name -> google.protobuf.Timestamp
+ 2, // 64: swytch.cluster.v1.NackTipDetail.collection:type_name -> swytch.cluster.v1.CollectionKind
+ 0, // 65: swytch.cluster.v1.NackTipDetail.op:type_name -> swytch.cluster.v1.EffectOp
+ 15, // 66: swytch.cluster.v1.NackTipDetail.deps:type_name -> swytch.cluster.v1.EffectRef
+ 34, // 67: swytch.cluster.v1.NackTipDetail.bind_consumed_tips:type_name -> swytch.cluster.v1.KeyConsumedTips
+ 15, // 68: swytch.cluster.v1.KeyConsumedTips.consumed_tips:type_name -> swytch.cluster.v1.EffectRef
+ 15, // 69: swytch.cluster.v1.NackNotify.conflicting:type_name -> swytch.cluster.v1.EffectRef
+ 15, // 70: swytch.cluster.v1.NackNotify.tips:type_name -> swytch.cluster.v1.EffectRef
+ 33, // 71: swytch.cluster.v1.NackNotify.tip_details:type_name -> swytch.cluster.v1.NackTipDetail
+ 15, // 72: swytch.cluster.v1.NackNotify.causal_chain:type_name -> swytch.cluster.v1.EffectRef
+ 37, // 73: swytch.cluster.v1.ForwardedTransaction.commands:type_name -> swytch.cluster.v1.ForwardedCommand
+ 15, // 74: swytch.cluster.v1.TransactionalBindEffect.KeyBind.consumed_tips:type_name -> swytch.cluster.v1.EffectRef
+ 15, // 75: swytch.cluster.v1.TransactionalBindEffect.KeyBind.new_tip:type_name -> swytch.cluster.v1.EffectRef
+ 17, // 76: swytch.cluster.v1.ReducedEffect.NetAddsEntry.value:type_name -> swytch.cluster.v1.ReducedElement
+ 18, // 77: swytch.cluster.v1.ReducedEffect.PartitionsEntry.value:type_name -> swytch.cluster.v1.ReducedEffect
+ 6, // 78: swytch.cluster.v1.SnapshotEffect.TxnVerdictsEntry.value:type_name -> swytch.cluster.v1.Verdict
+ 30, // 79: swytch.cluster.v1.Replication.NotifyStream:input_type -> swytch.cluster.v1.OffsetNotify
+ 31, // 80: swytch.cluster.v1.Replication.Fetch:input_type -> swytch.cluster.v1.FetchRequest
+ 30, // 81: swytch.cluster.v1.Replication.NotifyStream:output_type -> swytch.cluster.v1.OffsetNotify
+ 32, // 82: swytch.cluster.v1.Replication.Fetch:output_type -> swytch.cluster.v1.FetchResponse
+ 81, // [81:83] is the sub-list for method output_type
+ 79, // [79:81] is the sub-list for method input_type
+ 79, // [79:79] is the sub-list for extension type_name
+ 79, // [79:79] is the sub-list for extension extendee
+ 0, // [0:79] is the sub-list for field type_name
}
func init() { file_cluster_proto_replication_proto_init() }
@@ -3774,7 +3938,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*TypedValue); i {
+ switch v := v.(*BindEvidence); i {
case 0:
return &v.state
case 1:
@@ -3786,7 +3950,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*Predicate); i {
+ switch v := v.(*TypedValue); i {
case 0:
return &v.state
case 1:
@@ -3798,7 +3962,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*ObservationEffect); i {
+ switch v := v.(*Predicate); i {
case 0:
return &v.state
case 1:
@@ -3810,7 +3974,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*RowWriteEffect); i {
+ switch v := v.(*ObservationEffect); i {
case 0:
return &v.state
case 1:
@@ -3822,7 +3986,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*SubscriptionEffect); i {
+ switch v := v.(*RowWriteEffect); i {
case 0:
return &v.state
case 1:
@@ -3834,7 +3998,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*PubSubMessage); i {
+ switch v := v.(*SubscriptionEffect); i {
case 0:
return &v.state
case 1:
@@ -3846,7 +4010,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*SerializationEffect); i {
+ switch v := v.(*PubSubMessage); i {
case 0:
return &v.state
case 1:
@@ -3858,7 +4022,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*Effect); i {
+ switch v := v.(*SerializationEffect); i {
case 0:
return &v.state
case 1:
@@ -3870,7 +4034,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*OffsetNotify); i {
+ switch v := v.(*EvidenceRecord); i {
case 0:
return &v.state
case 1:
@@ -3882,7 +4046,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*FetchRequest); i {
+ switch v := v.(*Effect); i {
case 0:
return &v.state
case 1:
@@ -3894,7 +4058,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*FetchResponse); i {
+ switch v := v.(*OffsetNotify); i {
case 0:
return &v.state
case 1:
@@ -3906,7 +4070,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*NackTipDetail); i {
+ switch v := v.(*FetchRequest); i {
case 0:
return &v.state
case 1:
@@ -3918,7 +4082,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*KeyConsumedTips); i {
+ switch v := v.(*FetchResponse); i {
case 0:
return &v.state
case 1:
@@ -3930,7 +4094,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*NackNotify); i {
+ switch v := v.(*NackTipDetail); i {
case 0:
return &v.state
case 1:
@@ -3942,7 +4106,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*ForwardedTransaction); i {
+ switch v := v.(*KeyConsumedTips); i {
case 0:
return &v.state
case 1:
@@ -3954,7 +4118,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*ForwardedCommand); i {
+ switch v := v.(*NackNotify); i {
case 0:
return &v.state
case 1:
@@ -3966,7 +4130,7 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} {
- switch v := v.(*ForwardedResponse); i {
+ switch v := v.(*ForwardedTransaction); i {
case 0:
return &v.state
case 1:
@@ -3978,6 +4142,30 @@ func file_cluster_proto_replication_proto_init() {
}
}
file_cluster_proto_replication_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*ForwardedCommand); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_cluster_proto_replication_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*ForwardedResponse); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
+ file_cluster_proto_replication_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} {
switch v := v.(*TransactionalBindEffect_KeyBind); i {
case 0:
return &v.state
@@ -3998,7 +4186,7 @@ func file_cluster_proto_replication_proto_init() {
(*DataEffect_Child)(nil),
}
file_cluster_proto_replication_proto_msgTypes[7].OneofWrappers = []interface{}{}
- file_cluster_proto_replication_proto_msgTypes[16].OneofWrappers = []interface{}{
+ file_cluster_proto_replication_proto_msgTypes[18].OneofWrappers = []interface{}{
(*Effect_Data)(nil),
(*Effect_Meta)(nil),
(*Effect_TxnBind)(nil),
@@ -4016,7 +4204,7 @@ func file_cluster_proto_replication_proto_init() {
GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
RawDescriptor: file_cluster_proto_replication_proto_rawDesc,
NumEnums: 11,
- NumMessages: 32,
+ NumMessages: 34,
NumExtensions: 0,
NumServices: 1,
},
diff --git a/cluster/proto/replication.proto b/cluster/proto/replication.proto
index 5aec79c..268fa91 100644
--- a/cluster/proto/replication.proto
+++ b/cluster/proto/replication.proto
@@ -201,6 +201,17 @@ message SnapshotEffect {
// traverse historical adjudications via snapshots without consulting tips.
// Zero-valued if no prior snapshot exists for this key.
EffectRef prev_snapshot = 4;
+
+ // Owned proof references for the recorded bind outcomes. These are not
+ // value dependencies and do not participate in frontier or LCA selection.
+ BindEvidence evidence = 5;
+}
+
+// The evaluated binds are distinct from supporting ancestry. All references
+// own their vertices; neither kind is a value-DAG edge.
+message BindEvidence {
+ repeated EffectRef binds = 1; // includes subjects from predecessor proofs
+ repeated EffectRef deps = 2;
}
// TypedValue is a tagged literal used in Predicate comparisons and
@@ -322,6 +333,14 @@ message SerializationEffect {
bool release = 2; // false = request serialization, true = release
}
+// Inline proof payload, encoded with the normal Effect wire encoding. Packages
+// are dependency-first and deduplicated; only explicit evidence edges are
+// followed, never the payloads' ordinary value ancestry.
+message EvidenceRecord {
+ EffectRef ref = 1;
+ bytes effect_data = 2;
+}
+
// Effect is the top-level envelope stored in the log and sent over the wire.
message Effect {
// Identity & causality
@@ -333,6 +352,10 @@ message Effect {
bytes fork_choice_hash = 6; // SHA-256(node_id_LE8 || hlc_LE8), computed at emit time
bytes virtual = 7; // partition key WITHIN the key (empty = root partition); enables nested values
+ // Transport-only payloads for snapshot.evidence (including nested proofs).
+ // Receivers install these before the owner and strip this field in the pool.
+ repeated EvidenceRecord evidence_package = 8;
+
// The effect payload — exactly one of these is set.
oneof kind {
DataEffect data = 10;
diff --git a/cluster/quic_notify_transport.go b/cluster/quic_notify_transport.go
index fd17682..4ae2306 100644
--- a/cluster/quic_notify_transport.go
+++ b/cluster/quic_notify_transport.go
@@ -30,6 +30,7 @@ import (
"log/slog"
"time"
+ "github.com/puzpuzpuz/xsync/v4"
"github.com/quic-go/quic-go"
pb "github.com/swytchdb/engine/cluster/proto"
"github.com/swytchdb/engine/effects"
@@ -55,6 +56,12 @@ type QUICNotifyTransport struct {
nodeID NodeId
handler PacketHandler
+ // Active notifications, keyed by sender and request ID. Retries are shed
+ // until the original handler finishes its ACK/NACK send attempt.
+ // The sender NodeID is its startup incarnation, not its address: restarts
+ // get a new NodeID, while reconnects preserve it and the request ID.
+ activeNotifies *xsync.Map[[2]uint64, struct{}]
+
// connFunc returns the QUIC connection for a given peer.
// Set by PeerManager to look up PeerConn connections.
connFunc func(peerID NodeId) *quic.Conn
@@ -82,6 +89,7 @@ func NewQUICNotifyTransport(
return &QUICNotifyTransport{
nodeID: nodeID,
handler: handler,
+ activeNotifies: xsync.NewMap[[2]uint64, struct{}](),
connFunc: connFunc,
onPeerIdentified: onPeerIdentified,
stopCh: make(chan struct{}),
@@ -260,7 +268,25 @@ func (t *QUICNotifyTransport) dispatchPlaintext(peerID NodeId, plaintext []byte)
}
if t.handler != nil {
- go t.handler.HandleNotify(peerID, requestID, notify)
+ // One-way notifications all use zero; they are distinct messages.
+ if requestID == 0 {
+ go t.handler.HandleNotify(peerID, requestID, notify)
+
+ return
+ }
+
+ key := [2]uint64{uint64(peerID), requestID}
+ if _, loaded := t.activeNotifies.LoadOrStore(key, struct{}{}); loaded {
+ return
+ }
+
+ go func() {
+ // Keep no completed response cache: a retry after a lost ACK/NACK
+ // must be able to execute again.
+ defer t.activeNotifies.Delete(key)
+
+ t.handler.HandleNotify(peerID, requestID, notify)
+ }()
}
case PacketTypeNotifyACK:
diff --git a/cluster/quic_notify_transport_test.go b/cluster/quic_notify_transport_test.go
index 0703d9b..ca58471 100644
--- a/cluster/quic_notify_transport_test.go
+++ b/cluster/quic_notify_transport_test.go
@@ -24,11 +24,145 @@
package cluster
import (
+ "bytes"
+ "errors"
+ "sync"
+ "sync/atomic"
"testing"
+ "testing/synctest"
+ "time"
"github.com/quic-go/quic-go"
+ pb "github.com/swytchdb/engine/cluster/proto"
)
+type restartNotifyHandler struct {
+ recordingHandler
+
+ oldGate chan struct{}
+}
+
+func (h *restartNotifyHandler) HandleOffsetNotify(notify *pb.OffsetNotify) ([]*pb.NackNotify, error) {
+ h.count.Add(1)
+
+ if string(notify.GetKey()) == "old" {
+ <-h.oldGate
+ }
+
+ return nil, nil
+}
+
+func TestDispatchNotifyRequestAfterRestart(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ oldID, newID := pb.NewNodeID(), pb.NewNodeID()
+ if oldID == newID {
+ t.Fatal("restart reused a node incarnation")
+ }
+
+ handler := &restartNotifyHandler{oldGate: make(chan struct{})}
+ defer close(handler.oldGate)
+
+ receiver := NewReplicator(1, "local", NewPeerHealthTable(ClockConfig{}), nil, handler, time.Second)
+ transport := NewQUICNotifyTransport(1, &PeerManager{replicator: receiver}, nil, nil)
+ old := NewReplicator(oldID, "local", NewPeerHealthTable(ClockConfig{}), nil, nil, time.Second)
+
+ restarted := NewReplicator(newID, "local", NewPeerHealthTable(ClockConfig{}), nil, nil, time.Second)
+ defer restarted.Stop()
+
+ receiver.transport = notificationTransportFunc(func(peer NodeId, packet []byte) (int, error) {
+ from, request, status, credits, err := parseNotifyACKPacket(packet)
+ if err != nil {
+ return 0, err
+ }
+
+ if peer == newID {
+ restarted.HandleNotifyACK(from, request, status, credits)
+ } else {
+ old.HandleNotifyACK(from, request, status, credits)
+ }
+
+ return len(packet), nil
+ })
+
+ var oldPacket, newPacket []byte
+
+ old.transport = notificationTransportFunc(func(_ NodeId, packet []byte) (int, error) {
+ oldPacket = packet
+ transport.dispatchPlaintext(oldID, packet)
+
+ return len(packet), nil
+ })
+ oldDone := make(chan error, 1)
+
+ go func() {
+ _, err := old.ReplicateTo(&pb.OffsetNotify{Key: []byte("old")}, nil, 1)
+ oldDone <- err
+ }()
+
+ synctest.Wait()
+
+ // A replacement connection still delivers the same sender incarnation
+ // and request header. Its retries must not start another old handler.
+ old.transport = notificationTransportFunc(func(_ NodeId, packet []byte) (int, error) {
+ transport.dispatchPlaintext(oldID, packet)
+
+ return len(packet), nil
+ })
+ for range 4 {
+ _, _ = old.transport.Send(1, oldPacket)
+ }
+
+ synctest.Wait()
+
+ if handler.count.Load() != 1 {
+ t.Fatal("reconnect admitted duplicate active requests")
+ }
+
+ old.Stop()
+
+ if err := <-oldDone; !errors.Is(err, ErrReplicationTimeout) {
+ t.Fatalf("old sender shutdown: %v", err)
+ }
+
+ // The new sender's counter restarts at 1 while the old remote handler
+ // is still blocked. Only its NodeID distinguishes this new request.
+ restarted.transport = notificationTransportFunc(func(_ NodeId, packet []byte) (int, error) {
+ newPacket = packet
+ transport.dispatchPlaintext(newID, packet)
+
+ return len(packet), nil
+ })
+ newDone := make(chan error, 1)
+
+ go func() {
+ _, err := restarted.ReplicateTo(&pb.OffsetNotify{Key: []byte("new")}, nil, 1)
+ newDone <- err
+ }()
+
+ synctest.Wait()
+
+ select {
+ case err := <-newDone:
+ if err != nil {
+ t.Errorf("restarted sender did not receive its ACK: %v", err)
+ }
+ default:
+ t.Error("new request was shed behind the previous incarnation")
+ }
+
+ oldRequest, _, oldErr := parseNotifyPacket(oldPacket)
+
+ newRequest, _, newErr := parseNotifyPacket(newPacket)
+ if oldErr != nil || newErr != nil || oldRequest != 1 || newRequest != oldRequest {
+ t.Fatalf("test did not reuse the request ID: old=%d new=%d errors=%v,%v", oldRequest, newRequest, oldErr, newErr)
+ }
+
+ if handler.count.Load() != 2 || transport.activeNotifies.Size() != 1 {
+ t.Fatalf("handlers=%d active=%d; only the old handler should remain blocked", handler.count.Load(), transport.activeNotifies.Size())
+ }
+ })
+}
+
// TestDispatchPlaintextEmptyPayload guards the remote-input path: a zstd
// frame can decompress to zero bytes, which must not panic the dispatcher.
func TestDispatchPlaintextEmptyPayload(t *testing.T) {
@@ -37,3 +171,124 @@ func TestDispatchPlaintextEmptyPayload(t *testing.T) {
transport.dispatchPlaintext(2, nil)
transport.dispatchPlaintext(2, []byte{})
}
+
+type gatedNotifyHandler struct {
+ recordingHandler
+
+ gate chan struct{}
+ nacks []*pb.NackNotify
+}
+
+func (h *gatedNotifyHandler) HandleOffsetNotify(_ *pb.OffsetNotify) ([]*pb.NackNotify, error) {
+ h.count.Add(1)
+ <-h.gate
+
+ return h.nacks, nil
+}
+
+func TestDispatchNotifyShedsActiveRetries(t *testing.T) {
+ for _, nack := range []bool{false, true} {
+ name := "ACK"
+ if nack {
+ name = "NACK"
+ }
+
+ t.Run(name, func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ handler := &gatedNotifyHandler{gate: make(chan struct{})}
+ want := MarshalNotifyACKPacket(1, 42, 0, 0)
+
+ if nack {
+ handler.nacks = []*pb.NackNotify{{Key: []byte("key"), NotSubscribed: true}}
+ want = MarshalNotifyNACKPacket(1, 42, handler.nacks, 0)
+ }
+
+ responseGate := make(chan struct{})
+
+ var responses atomic.Int32
+
+ r := NewReplicator(1, "local", NewPeerHealthTable(ClockConfig{}), nil, handler, time.Second)
+ r.transport = notificationTransportFunc(func(peer NodeId, packet []byte) (int, error) {
+ responses.Add(1)
+
+ if peer != 2 || !bytes.Equal(packet, want) {
+ t.Error("original ACK/NACK response changed")
+ }
+
+ <-responseGate
+
+ return 0, ErrPeerUnavailable // A later retry must be admitted after a lost response.
+ })
+ transport := NewQUICNotifyTransport(1, &PeerManager{replicator: r}, nil, nil)
+
+ packet, err := MarshalNotifyPacket(42, &pb.OffsetNotify{Key: []byte("key")})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ dispatchRetries := func() {
+ var wg sync.WaitGroup
+ for range 32 {
+ wg.Go(func() { transport.dispatchPlaintext(2, packet) })
+ }
+
+ wg.Wait()
+ synctest.Wait()
+ }
+ dispatchRetries()
+
+ if got := handler.count.Load(); got != 1 {
+ t.Errorf("active handler invocations = %d, want 1", got)
+ }
+
+ if got := responses.Load(); got != 0 {
+ t.Errorf("premature responses = %d, want 0", got)
+ }
+
+ close(handler.gate)
+ synctest.Wait()
+ dispatchRetries()
+
+ if handler.count.Load() != 1 || responses.Load() != 1 {
+ t.Errorf("response in flight: handlers=%d responses=%d, want 1 each", handler.count.Load(), responses.Load())
+ }
+
+ close(responseGate)
+ synctest.Wait()
+ transport.dispatchPlaintext(2, packet)
+ synctest.Wait()
+
+ if handler.count.Load() != 2 || responses.Load() != 2 {
+ t.Errorf("retry after completion: handlers=%d responses=%d, want 2 each", handler.count.Load(), responses.Load())
+ }
+ })
+ })
+ }
+}
+
+func TestDispatchNotifyIndependentRequests(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ handler := &gatedNotifyHandler{gate: make(chan struct{})}
+ r := NewReplicator(1, "local", NewPeerHealthTable(ClockConfig{}), nil, handler, time.Second)
+ r.transport = notificationTransportFunc(func(_ NodeId, packet []byte) (int, error) { return len(packet), nil })
+ transport := NewQUICNotifyTransport(1, &PeerManager{replicator: r}, nil, nil)
+ // Requests are scoped to peers; zero identifies distinct one-way notifications.
+ requests := [][2]uint64{{2, 42}, {3, 42}, {2, 43}, {2, 0}, {2, 0}}
+ for _, request := range requests {
+ packet, err := MarshalNotifyPacket(request[1], &pb.OffsetNotify{Key: []byte("key")})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ transport.dispatchPlaintext(NodeId(request[0]), packet)
+ }
+
+ synctest.Wait()
+
+ if got := handler.count.Load(); got != int32(len(requests)) {
+ t.Errorf("independent handlers = %d, want %d", got, len(requests))
+ }
+
+ close(handler.gate)
+ })
+}
diff --git a/cluster/replication.go b/cluster/replication.go
index 7f8d03c..ca3b8cb 100644
--- a/cluster/replication.go
+++ b/cluster/replication.go
@@ -49,7 +49,8 @@ var (
// Retransmission constants.
const (
- sweepInterval = 250 * time.Millisecond // sweep tick rate — no point checking faster than rtoFloor/2
+ sweepInterval = 250 * time.Millisecond // deadline and peer-liveness checks
+ retransmitInterval = 500 * time.Millisecond
)
// Replicator manages replication with first-ACK semantics for same-region
@@ -615,9 +616,9 @@ func (r *Replicator) sweepLoop() {
// whose target has left the alive+symmetric set is given up — the peer is
// declared dead, so we stop holding the write open and let the originator
// fall back to local commit (it is now the last node standing on this key);
-// otherwise an unacked request is retransmitted. A request that ACKs is
-// removed from pending by the ACK handler, so only sends that never landed —
-// the peer's connection wasn't up at first attempt — keep reaching here.
+// otherwise an unacked request is retransmitted at retransmitInterval. Pending
+// requests may already be processing on the receiver; lack of an ACK does not
+// imply that the original notification never arrived.
func (r *Replicator) sweep() {
now := time.Now().UnixNano()
@@ -652,7 +653,7 @@ func (r *Replicator) sweep() {
return true
}
- if now-tr.lastSentAt.Load() < int64(sweepInterval) {
+ if now-tr.lastSentAt.Load() < int64(retransmitInterval) {
return true
}
diff --git a/cluster/server.go b/cluster/server.go
index 00af1b6..3226102 100644
--- a/cluster/server.go
+++ b/cluster/server.go
@@ -24,12 +24,12 @@
package cluster
import (
- "fmt"
"io"
"log/slog"
"github.com/quic-go/quic-go"
pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/effects"
)
// handleStream dispatches an incoming QUIC bidirectional stream based on its type prefix.
@@ -51,6 +51,8 @@ func handleStream(stream *quic.Stream, logReader LogReader, forwardHandler Forwa
switch typeBuf[0] {
case streamTypeFetch:
handleFetchStream(stream, logReader)
+ case streamTypeAncestry:
+ handleAncestryStream(stream, logReader)
case streamTypeForward:
handleForwardStream(stream, forwardHandler)
default:
@@ -146,4 +148,4 @@ func acceptStreams(conn *quic.Conn, logReader LogReader, forwardHandler ForwardH
}
// errFetchNotFound is returned when a fetch response contains no effect data.
-var errFetchNotFound = fmt.Errorf("effect not found on peer")
+var errFetchNotFound = effects.ErrEffectNotFound
diff --git a/effects/ancestry.go b/effects/ancestry.go
new file mode 100644
index 0000000..4e676cf
--- /dev/null
+++ b/effects/ancestry.go
@@ -0,0 +1,174 @@
+package effects
+
+import (
+ "fmt"
+ "math/rand/v2"
+ "slices"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+// AncestryFetcher streams a selected holder's available closure. have skips
+// duplicate payload installation, not traversal. receive installs bytes only;
+// isRoot records a culled history boundary. Otherwise nil bytes report a
+// missing boundary from a completed stream. Replication
+// reconciles further divergence; this is not a cluster-wide absence search.
+type AncestryFetcher interface {
+ FetchAncestry(key string, root Tip, peers []pb.NodeID, have func(Tip) bool, receive func(Tip, []byte, bool) error) error
+}
+
+// VisitAncestry visits resident dependencies, including missing boundaries.
+// The consumer bounds the transfer by returning false. It never fetches, adds
+// reverse links, publishes a frontier, or interprets snapshot state. Root
+// markers travel with the records; they do not hide resident ancestry.
+func (p *VertexPool) VisitAncestry(key string, root Tip, visit func(Tip, *pb.Effect, bool) bool) {
+ seen := make(map[Tip]struct{})
+
+ stack := []Tip{root}
+ for len(stack) > 0 {
+ tip := stack[len(stack)-1]
+ stack = stack[:len(stack)-1]
+
+ if _, ok := seen[tip]; ok {
+ continue
+ }
+
+ seen[tip] = struct{}{}
+ v := p.getVertex(tip)
+
+ var eff *pb.Effect
+
+ isRoot := false
+
+ if v != nil {
+ eff = v.eff
+ isRoot = v.root()
+ }
+
+ if !visit(tip, eff, isRoot) {
+ return
+ }
+
+ if eff != nil {
+ for ref := range keyDeps(key, eff) {
+ if dep := r(ref); dep != (Tip{}) {
+ stack = append(stack, dep)
+ }
+ }
+ }
+ }
+}
+
+// ancestryPeers uses existing subscription/filter evidence. Unknown filters
+// remain candidates. Subscription knowledge and the effect's origin override
+// negative filters; filters describe keys, not a complete history inventory.
+func (e *Engine) ancestryPeers(key string, root Tip) []pb.NodeID {
+ peers := slices.Clone(e.broadcaster.PeerIDs())
+
+ var subscribers []pb.NodeID
+
+ if leaf := e.index.LoadData(key); leaf != nil {
+ leaf.subMu.Lock()
+ subscribers = slices.Clone(leaf.subscribers)
+ leaf.subMu.Unlock()
+ }
+
+ e.keyFilterMu.RLock()
+
+ for _, pf := range e.peerKeyFilters {
+ if pf.counted {
+ peers = append(peers, pf.nodeID)
+ }
+ }
+
+ slices.Sort(peers)
+ peers = slices.Compact(peers)
+ peers = slices.DeleteFunc(peers, func(peer pb.NodeID) bool {
+ if peer == e.nodeID {
+ return true
+ }
+
+ if uint64(peer) == root[0] || slices.Contains(subscribers, peer) {
+ return false
+ }
+
+ i, found := e.peerFilterIndex(peer)
+
+ return found && e.peerKeyFilters[i].hasBulk && !e.peerKeyFilters[i].maybeContains(key)
+ })
+
+ e.keyFilterMu.RUnlock()
+ rand.Shuffle(len(peers), func(i, j int) { peers[i], peers[j] = peers[j], peers[i] })
+ peers = peers[:min(2, len(peers))]
+
+ return peers
+}
+
+func (e *Engine) fetchFrontierAncestry(key string, root Tip) ([]Tip, error) {
+ fetcher, ok := e.broadcaster.(AncestryFetcher)
+ if !ok || key == "" || e.effectCache == nil {
+ _, err := e.getEffect(key, root)
+
+ return nil, err
+ }
+ // Only active requests are retained. The vertex pool owns completed data.
+ // Different roots on this key also join the active transfer, then recheck
+ // residency: the stream may already have delivered their dependency.
+ type result struct {
+ root Tip
+ missing []Tip
+ }
+ for {
+ if e.effectCache.getVertex(root) != nil {
+ return nil, nil
+ }
+
+ value, err, _ := e.ancestryLoads.Do(key, func() (any, error) {
+ if e.effectCache.getVertex(root) != nil {
+ return result{root: root}, nil
+ }
+
+ var missing []Tip
+
+ err := fetcher.FetchAncestry(key, root, e.ancestryPeers(key, root),
+ func(tip Tip) bool { return e.effectCache.getVertex(tip) != nil }, func(tip Tip, data []byte, isRoot bool) error {
+ if isRoot {
+ if len(data) != 0 {
+ if err := e.storeWireData(tip, data); err != nil {
+ return err
+ }
+ }
+
+ if v := e.effectCache.getVertex(tip); v != nil {
+ v.markRoot()
+ }
+
+ return nil
+ }
+
+ if len(data) == 0 {
+ missing = append(missing, tip)
+
+ return nil
+ }
+
+ return e.storeWireData(tip, data)
+ })
+
+ return result{root: root, missing: missing}, err
+ })
+
+ loaded := value.(result)
+ if loaded.root == root {
+ return loaded.missing, err
+ }
+
+ if slices.Contains(loaded.missing, root) {
+ return loaded.missing, ErrEffectNotFound
+ }
+
+ if e.closed.Load() {
+ return nil, fmt.Errorf("ancestry download: engine closed")
+ }
+ }
+}
diff --git a/effects/ancestry_test.go b/effects/ancestry_test.go
new file mode 100644
index 0000000..999802a
--- /dev/null
+++ b/effects/ancestry_test.go
@@ -0,0 +1,304 @@
+package effects
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "testing/synctest"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/keytrie"
+)
+
+type ancestryBroadcaster struct {
+ mockBroadcaster
+
+ load func(string, Tip, []pb.NodeID, func(Tip) bool, func(Tip, []byte, bool) error) error
+}
+
+func (b *ancestryBroadcaster) FetchAncestry(key string, root Tip, peers []pb.NodeID, have func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ return b.load(key, root, peers, have, receive)
+}
+
+func TestAncestryConcurrentRootsJoinActiveDownload(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ root, dep := Tip{7, 2}, Tip{7, 1}
+ release := make(chan struct{})
+
+ var calls atomic.Int32
+
+ e.broadcaster = &ancestryBroadcaster{load: func(_ string, tip Tip, _ []pb.NodeID, _ func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ calls.Add(1)
+
+ if tip != root {
+ t.Errorf("duplicate download for %v", tip)
+ }
+
+ <-release
+
+ if err := receive(root, wireEffect(cutEffect(root, dep)), false); err != nil {
+ return err
+ }
+
+ return receive(dep, wireEffect(cutEffect(dep)), false)
+ }}
+
+ var wg sync.WaitGroup
+
+ fetch := func(tip Tip) {
+ wg.Go(func() {
+ if _, err := e.fetchFrontierAncestry("k", tip); err != nil {
+ t.Error(err)
+ }
+ })
+ }
+ fetch(root)
+ synctest.Wait()
+
+ for i := range 32 {
+ if i%2 == 0 {
+ fetch(root)
+ } else {
+ fetch(dep)
+ }
+ }
+
+ synctest.Wait()
+
+ if calls.Load() != 1 {
+ t.Fatalf("concurrent requests opened %d downloads", calls.Load())
+ }
+
+ close(release)
+ wg.Wait()
+
+ if calls.Load() != 1 {
+ t.Fatalf("joined dependency redownloaded: %d calls", calls.Load())
+ }
+ })
+}
+
+func TestAncestryFailedDownloadCanRetry(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ root := Tip{7, 1}
+ failure := errors.New("transport unavailable")
+ calls := 0
+
+ e.broadcaster = &ancestryBroadcaster{load: func(_ string, _ Tip, _ []pb.NodeID, _ func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ calls++
+ if calls == 1 {
+ return failure
+ }
+
+ return receive(root, wireEffect(cutEffect(root)), false)
+ }}
+ if _, err := e.fetchFrontierAncestry("k", root); !errors.Is(err, failure) {
+ t.Fatal(err)
+ }
+
+ if _, err := e.fetchFrontierAncestry("k", root); err != nil {
+ t.Fatal(err)
+ }
+
+ if calls != 2 {
+ t.Fatalf("failed transfer was cached: %d calls", calls)
+ }
+}
+
+func TestAncestryPeersUseKeyEvidenceAndMembership(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+ e.index.Insert("k", nil, keytrie.NewTipSet(Tip{1, 1}))
+
+ e.broadcaster = &mockBroadcaster{peerIDs: []pb.NodeID{10, 11, 12, 13, 15}}
+ for _, peer := range []pb.NodeID{10, 11, 13, 15} {
+ e.getOrCreatePeerFilter(peer).hasBulk = true
+ }
+
+ e.peerFilterAdd(13, "k")
+ e.addPeerSubscriber("k", 11)
+ e.addPeerSubscriber("k", 14) // removed member, stale leaf subscriber
+
+ got := e.ancestryPeers("k", Tip{15, 1})
+ if len(got) != 2 || got[0] == got[1] || !slices.Contains([]pb.NodeID{15, 11, 12, 13}, got[0]) || !slices.Contains([]pb.NodeID{15, 11, 12, 13}, got[1]) {
+ t.Fatalf("holders = %v", got)
+ }
+}
+
+func TestAncestryStreamFeedsExistingCutDiscoveryAndBoundaries(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ a, b := Tip{7, 4}, Tip{8, 1}
+ putCutEffects(t, e, map[Tip]*pb.Effect{a: cutEffect(a, Tip{7, 3}), b: cutEffect(b, Tip{7, 1})})
+
+ requests := []Tip{}
+ e.broadcaster = &ancestryBroadcaster{load: func(_ string, root Tip, _ []pb.NodeID, _ func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ requests = append(requests, root)
+ for i := root[1]; i > 0; i-- {
+ tip := Tip{7, i}
+
+ var deps []Tip
+ if i > 1 {
+ deps = []Tip{{7, i - 1}}
+ } else {
+ deps = []Tip{{9, 1}}
+ }
+
+ if err := receive(tip, wireEffect(cutEffect(tip, deps...)), false); err != nil {
+ return err
+ }
+ }
+
+ return receive(Tip{9, 1}, nil, false)
+ }}
+ got, complete, err := e.closeIngestedFrontier("k", []Tip{a, b}, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{a, b})
+
+ if err != nil || !complete {
+ t.Fatalf("closure complete=%t error=%v", complete, err)
+ }
+
+ if len(requests) != 1 || requests[0] != (Tip{7, 3}) {
+ t.Fatalf("ancestry requests = %v", requests)
+ }
+}
+
+func TestAncestryRootEvidenceSuppressesSubsequentFetches(t *testing.T) {
+ for _, alreadyResident := range []bool{false, true} {
+ t.Run(fmt.Sprintf("resident=%t", alreadyResident), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ source := newVertexPool()
+ cut, middle, tip := Tip{7, 2}, Tip{7, 3}, Tip{7, 4}
+ snapshot := cutEffect(cut, Tip{7, 1})
+
+ snapshot.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}
+ for id, eff := range map[Tip]*pb.Effect{cut: snapshot, middle: cutEffect(middle, cut)} {
+ if err := source.Put(id, eff); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ source.getVertex(cut).markRoot()
+ putCutEffects(t, e, map[Tip]*pb.Effect{tip: cutEffect(tip, middle)})
+
+ if alreadyResident {
+ putCutEffects(t, e, map[Tip]*pb.Effect{cut: snapshot})
+ }
+
+ calls, records := 0, 0
+
+ e.broadcaster = &ancestryBroadcaster{load: func(key string, root Tip, _ []pb.NodeID, have func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ calls++
+
+ var err error
+
+ source.VisitAncestry(key, root, func(id Tip, eff *pb.Effect, isRoot bool) bool {
+ records++
+
+ var data []byte
+ if eff != nil && !have(id) {
+ data = wireEffect(eff)
+ }
+
+ err = receive(id, data, isRoot)
+
+ return err == nil
+ })
+
+ return err
+ }}
+ for range 2 {
+ got, complete, err := e.closeIngestedFrontier("k", []Tip{tip}, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{tip})
+
+ if err != nil || !complete {
+ t.Fatalf("complete=%t err=%v", complete, err)
+ }
+ }
+
+ if calls != 1 || records != 3 {
+ t.Fatalf("requests=%d records=%d", calls, records)
+ }
+
+ if !e.effectCache.getVertex(cut).root() {
+ t.Fatal("remote root evidence was lost")
+ }
+
+ if e.effectCache.getVertex(tip).root() || e.effectCache.getVertex(middle).root() {
+ t.Fatal("marked history above the cut as root")
+ }
+ })
+ }
+}
+
+// A reclamation boundary suppresses fetches, not resident causal evidence.
+func TestAncestryRootStillProcessesResidentHistory(t *testing.T) {
+ for _, rootFirst := range []bool{false, true} {
+ t.Run(fmt.Sprintf("rootFirst=%t", rootFirst), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ cut, middle, old, missing := Tip{7, 4}, Tip{7, 3}, Tip{7, 2}, Tip{7, 1}
+ snapshot := cutEffect(cut, middle)
+
+ snapshot.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}
+ if rootFirst {
+ snapshot.ForkChoiceHash = []byte{0}
+ }
+
+ putCutEffects(t, e, map[Tip]*pb.Effect{
+ cut: snapshot, middle: cutEffect(middle, old), old: cutEffect(old, missing),
+ })
+ e.effectCache.getVertex(cut).markRoot()
+
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ t.Errorf("fetched below root: %v", tip)
+
+ return nil, ErrEffectNotFound
+ }}
+ for range 2 {
+ got, complete, err := e.closeIngestedFrontier("k", []Tip{cut, old}, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{cut})
+
+ if err != nil || !complete {
+ t.Fatalf("complete=%t err=%v", complete, err)
+ }
+ }
+
+ seen := make(map[Tip]bool)
+
+ e.effectCache.VisitAncestry("k", cut, func(tip Tip, eff *pb.Effect, _ bool) bool {
+ if eff != nil {
+ seen[tip] = true
+ }
+
+ return true
+ })
+
+ if !seen[cut] || !seen[middle] || !seen[old] {
+ t.Fatalf("stream omitted resident history: %v", seen)
+ }
+ })
+ }
+}
diff --git a/effects/causal_index.go b/effects/causal_index.go
index 57643a0..8f07af6 100644
--- a/effects/causal_index.go
+++ b/effects/causal_index.go
@@ -38,6 +38,7 @@ import (
type causalIndex struct {
engine *Engine
key string
+ load func(Tip) (*pb.Effect, error)
ids map[Tip]int
nodes []causalNode
heads []int
@@ -54,7 +55,18 @@ type causalNode struct {
}
func newCausalIndex(e *Engine, key string) *causalIndex {
- return &causalIndex{engine: e, key: key, ids: make(map[Tip]int)}
+ return &causalIndex{engine: e, key: key, ids: make(map[Tip]int), load: func(t Tip) (*pb.Effect, error) { return e.getEffect(key, t) }}
+}
+
+func (c *causalIndex) deps(eff *pb.Effect) []*pb.EffectRef {
+ for _, kb := range eff.GetTxnBind().GetKeys() {
+ if string(kb.GetKey()) == c.key {
+ return []*pb.EffectRef{kb.GetNewTip()}
+ }
+ }
+ // Conflict reporting can extend the index with an envelope from another
+ // key. Such an envelope has no per-key projection in this index.
+ return eff.GetDeps()
}
func (c *causalIndex) add(tips ...Tip) bool {
@@ -74,10 +86,10 @@ func (c *causalIndex) add(tips ...Tip) bool {
id := len(c.nodes)
c.ids[tip] = id
c.nodes = append(c.nodes, causalNode{})
- eff, _ := c.engine.getEffect(c.key, tip)
+ eff, _ := c.load(tip)
// As in the original walk, a missing effect is reachable but its
// unavailable parents are not. Do not infer ancestry past the gap.
- stack = append(stack, frame{id: id, deps: eff.GetDeps()})
+ stack = append(stack, frame{id: id, deps: c.deps(eff)})
}
for _, tip := range tips {
@@ -209,12 +221,12 @@ func (c *causalIndex) reachesCyclic(from, target Tip) bool {
cur := stack[len(stack)-1]
stack = stack[:len(stack)-1]
- eff, err := c.engine.getEffect(c.key, cur)
+ eff, err := c.load(cur)
if err != nil {
continue
}
- for _, dep := range eff.GetDeps() {
+ for _, dep := range c.deps(eff) {
tip := r(dep)
if _, visited := seen[tip]; !visited {
seen[tip] = struct{}{}
diff --git a/effects/cloud_closure_test.go b/effects/cloud_closure_test.go
new file mode 100644
index 0000000..67329cb
--- /dev/null
+++ b/effects/cloud_closure_test.go
@@ -0,0 +1,210 @@
+package effects
+
+import (
+ "errors"
+ "slices"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+// Synthetic stream ordering: X fails first, then Y supplies newer boundary
+// evidence for X. A usable snapshot alone must not erase availability errors.
+func TestInstallCloudTipsRetiresResolvedBoundaryErrors(t *testing.T) {
+ for _, mode := range []string{"absent", "root", "resident", "unresolved", "interrupted", "unrelated-failure"} {
+ t.Run(mode, func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ x, unrelated, y := Tip{7, 1}, Tip{8, 1}, Tip{9, 1}
+ xFailure := errors.New("X transport unavailable")
+ otherFailure := errors.New("unrelated transport unavailable")
+ tips := []Tip{x, y}
+ wantCalls := []Tip{x, y}
+
+ if mode == "unrelated-failure" {
+ tips = append(tips, unrelated)
+ wantCalls = []Tip{x, unrelated, y}
+ }
+
+ snapshot := cutEffect(y, x)
+
+ snapshot.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{
+ Scalar: scalarInsertInt(pb.MergeRule_ADDITIVE_INT, 10),
+ }}}
+ if mode == "unresolved" {
+ // Y's complete history supplies no evidence about concurrent X.
+ snapshot.Deps = nil
+ }
+
+ var calls []Tip
+
+ e.broadcaster = &ancestryBroadcaster{load: func(_ string, root Tip, _ []pb.NodeID, _ func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ calls = append(calls, root)
+ switch root {
+ case x:
+ return xFailure
+ case unrelated:
+ return otherFailure
+ case y:
+ if err := receive(y, wireEffect(snapshot), mode == "root"); err != nil {
+ return err
+ }
+ default:
+ t.Fatalf("unexpected ancestry root: %v", root)
+ }
+
+ switch mode {
+ case "absent", "unrelated-failure":
+ return receive(x, nil, false)
+ case "resident":
+ return receive(x, wireEffect(cutEffect(x)), false)
+ case "interrupted":
+ // No missing-boundary report is published by an incomplete stream.
+ return errors.New("Y stream interrupted")
+ default:
+ return nil
+ }
+ }}
+
+ err := e.InstallCloudTips("k", tips, nil)
+
+ if !slices.Equal(calls, wantCalls) {
+ t.Fatalf("fetch order = %v, want %v", calls, wantCalls)
+ }
+
+ switch mode {
+ case "unresolved", "interrupted":
+ if !errors.Is(err, xFailure) || !errors.Is(err, ErrCloudUnavailable) {
+ t.Fatalf("unresolved transport failure was lost: %v", err)
+ }
+ case "unrelated-failure":
+ if errors.Is(err, xFailure) || !errors.Is(err, otherFailure) || !errors.Is(err, ErrCloudUnavailable) {
+ t.Fatalf("only the resolved boundary error should be retired: %v", err)
+ }
+ default:
+ if err != nil {
+ t.Fatalf("resolved boundary prevented publication: %v", err)
+ }
+
+ requireFrontier(t, e, y)
+ }
+
+ if err != nil && e.index.Contains("k").Len() != 0 {
+ t.Fatal("cloud published despite unresolved availability")
+ }
+ })
+ }
+}
+
+func TestInstallCloudTipsRetainsClosureWithoutReconstruction(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+ e.broadcaster = &holedFetchBroadcaster{}
+ tip := Tip{7, 2}
+ eff := frontierEffect(2, 1)
+
+ err := e.InstallCloudTips("k", []Tip{tip}, []CloudEffect{{Tip: tip, Eff: eff}})
+ if err != nil {
+ t.Fatalf("closure should exhaust the absent dependency: %v", err)
+ }
+
+ requireFrontier(t, e, tip)
+
+ if err := newDag(e, "k", "").prepare([]Tip{tip}); !errors.Is(err, errEffectNotResident) {
+ t.Fatalf("reconstruction must detect its missing dependency: %v", err)
+ }
+}
+
+func TestInstallCloudTipsConsumesLocalAncestor(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ ancestor, tip := Tip{7, 1}, Tip{7, 2}
+ if err := e.effectCache.Put(ancestor, frontierEffect(1)); err != nil {
+ t.Fatal(err)
+ }
+
+ e.installTips("k", []Tip{ancestor})
+
+ err := e.InstallCloudTips("k", []Tip{tip}, []CloudEffect{{Tip: tip, Eff: frontierEffect(2, 1)}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ requireFrontier(t, e, tip)
+}
+
+func TestInstallCloudTipsFetchesBeyondSnapshot(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ bc := &frontierBroadcaster{
+ fetchable: map[Tip][]byte{{7, 2}: wireEffect(frontierEffect(2, 1))},
+ fetched: make(map[Tip]bool),
+ }
+ e.broadcaster = bc
+ tip := Tip{7, 3}
+ eff := frontierEffect(3, 2)
+ eff.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{
+ Scalar: scalarInsertInt(pb.MergeRule_ADDITIVE_INT, 10),
+ }}}
+
+ err := e.InstallCloudTips("k", []Tip{tip}, []CloudEffect{{Tip: tip, Eff: eff}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !bc.fetched[Tip{7, 2}] || !bc.fetched[Tip{7, 1}] {
+ t.Fatalf("closure did not exhaust snapshot ancestry: %v", bc.fetched)
+ }
+
+ requireFrontier(t, e, tip)
+}
+
+func TestInstallCloudTipsRetriesStreamingClosure(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ tip, dep := Tip{7, 3}, Tip{7, 2}
+ failure := errors.New("ancestry stream interrupted")
+ calls := 0
+ e.broadcaster = &ancestryBroadcaster{load: func(key string, root Tip, _ []pb.NodeID, _ func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ calls++
+
+ if key != "k" || root != dep {
+ t.Fatalf("unexpected ancestry request: %q %v", key, root)
+ }
+
+ if calls == 1 {
+ return failure
+ }
+
+ return receive(dep, wireEffect(frontierEffect(2, 1)), true)
+ }}
+
+ sidecar := []CloudEffect{{Tip: tip, Eff: frontierEffect(3, 2)}}
+ if err := e.InstallCloudTips("k", []Tip{tip}, sidecar); !errors.Is(err, failure) || !errors.Is(err, ErrCloudUnavailable) {
+ t.Fatalf("transport failure was lost: %v", err)
+ }
+
+ if err := e.InstallCloudTips("k", []Tip{tip}, sidecar); err != nil {
+ t.Fatal(err)
+ }
+
+ if calls != 2 || !e.effectCache.getVertex(dep).root() {
+ t.Fatalf("stream retry did not retain the remote boundary: calls=%d", calls)
+ }
+
+ requireFrontier(t, e, tip)
+}
diff --git a/effects/cloud_read_miss_test.go b/effects/cloud_read_miss_test.go
index 7314931..24e2175 100644
--- a/effects/cloud_read_miss_test.go
+++ b/effects/cloud_read_miss_test.go
@@ -281,13 +281,9 @@ func (b *partialHoleBroadcaster) FetchFromAny(ref *pb.EffectRef, _ FetchHint) ([
return nil, fmt.Errorf("cdn fetch %v: %w", r(ref), ErrCDNBlobMissing)
}
-// TestReadMissServesReadableSubsetAndMarksPending: Cloud names a frontier we
-// can only partially fetch. The readable sibling is not this engine's hole to
-// repair and must install and serve on its own; the holed sibling is handed
-// to Cloud's reconcile loop via MarkPending instead of blocking the read. A
-// repeat read must not re-consult Cloud: the leaf's cloudPending marker caps
-// the WAN round-trips for the rest of its residency.
-func TestReadMissServesReadableSubsetAndMarksPending(t *testing.T) {
+// A missing sibling remains part of the installed frontier. Reconstruction
+// fails rather than returning only the available portion of the value.
+func TestReadMissDoesNotServeReadableSubset(t *testing.T) {
const key = "evicted"
reachable, wire := makeReachableEffect(t, key, 1)
@@ -306,38 +302,29 @@ func TestReadMissServesReadableSubsetAndMarksPending(t *testing.T) {
ctx := e.NewReadOnlyContext()
- result, _, err := ctx.GetSnapshot(key)
- if err != nil {
- t.Fatalf("read of a partially holed frontier errored: %v", err)
- }
-
- if result == nil {
- t.Fatal("expected the readable sibling's state, not a miss")
+ for range 2 {
+ result, _, err := ctx.GetSnapshot(key)
+ if !errors.Is(err, errEffectNotResident) || !errors.Is(err, ErrCDNBlobMissing) || result != nil {
+ t.Fatalf("read must report required missing history, got result=%v err=%v", result, err)
+ }
}
if fake.calls != 1 {
t.Fatalf("expected one Cloud consult, got %d", fake.calls)
}
- if len(fake.pending) != 1 || fake.pending[0] != key {
- t.Fatalf("expected the key marked pending once, got %v", fake.pending)
+ if len(fake.pending) != 0 {
+ t.Fatalf("ingestion classified history for reconstruction: %v", fake.pending)
}
- if _, _, err := ctx.GetSnapshot(key); err != nil {
- t.Fatalf("second read errored: %v", err)
- }
-
- if fake.calls != 1 {
- t.Fatalf("Cloud was re-consulted on a repeat read of a pending key (%d calls)", fake.calls)
+ if !e.index.Contains(key).ContainsAll([]Tip{reachable, holed}) {
+ t.Fatalf("cloud closure lost a branch: %v", e.index.Contains(key).Tips())
}
}
-// TestReadMissAllTipsPendingFailsRead: when nothing in Cloud's named frontier
-// is walkable, the read must fail with ErrCloudUnavailable rather than
-// fabricate a miss for data Cloud provably holds — but a repeat read must not
-// re-consult Cloud: the leaf's cloudPending marker caps the WAN round-trips
-// and leaves healing to the reconcile loop's own schedule.
-func TestReadMissAllTipsPendingFailsRead(t *testing.T) {
+// Definitive absence completes closure, but cannot turn an unavailable
+// frontier into a free miss. Subsequent reads retry through reconstruction.
+func TestReadMissAbsentFrontierFailsReconstruction(t *testing.T) {
const key = "evicted"
holed := Tip{9, 4}
@@ -353,17 +340,17 @@ func TestReadMissAllTipsPendingFailsRead(t *testing.T) {
ctx := e.NewReadOnlyContext()
for i := range 2 {
- if _, _, err := ctx.GetSnapshot(key); !errors.Is(err, ErrCloudUnavailable) {
- t.Fatalf("read %d: want ErrCloudUnavailable for a wholly holed frontier, got %v", i+1, err)
+ if _, _, err := ctx.GetSnapshot(key); !errors.Is(err, errEffectNotResident) || !errors.Is(err, ErrCDNBlobMissing) {
+ t.Fatalf("read %d: want missing reconstruction dependency, got %v", i+1, err)
}
}
if fake.calls != 1 {
- t.Fatalf("expected one Cloud consult (cloudPending caps the rest), got %d", fake.calls)
+ t.Fatalf("expected one Cloud consult (frontier is already installed), got %d", fake.calls)
}
- if len(fake.pending) != 1 || fake.pending[0] != key {
- t.Fatalf("expected the key marked pending once, got %v", fake.pending)
+ if len(fake.pending) != 0 {
+ t.Fatalf("definitive absence must exhaust closure, got pending=%v", fake.pending)
}
}
diff --git a/effects/compaction_history_test.go b/effects/compaction_history_test.go
new file mode 100644
index 0000000..c1d3c9c
--- /dev/null
+++ b/effects/compaction_history_test.go
@@ -0,0 +1,545 @@
+package effects
+
+import (
+ "context"
+ "fmt"
+ "slices"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/keytrie"
+)
+
+// Compaction must preserve the history selected by an existing transaction,
+// including keys it has not accessed yet. Exercise the normal snapshot emission
+// and reclamation paths so future history releases must preserve this contract.
+func TestCompactionPreservesTransactionHistory(t *testing.T) {
+ for _, readBefore := range []bool{false, true} {
+ t.Run(fmt.Sprintf("read-before=%t", readBefore), func(t *testing.T) {
+ e := newTestEngine(nil)
+
+ t.Cleanup(func() {
+ if err := e.Close(); err != nil {
+ t.Error(err)
+ }
+ })
+ preSubscribe(e, "k")
+ historicalTestWrite(t, e, map[string]string{"k": "old"}, false)
+ oldTip := e.index.Contains("k").Tips()[0]
+
+ tx := e.NewContext()
+
+ tx.BeginTx()
+ defer tx.Abort()
+
+ if readBefore {
+ got, _, err := tx.GetSnapshot("k")
+ if err != nil || string(got.GetScalar().GetRaw()) != "old" {
+ t.Fatalf("initial transaction read: value=%v error=%v", got, err)
+ }
+ }
+
+ // Exceed the standalone compaction threshold using public writes.
+ for range 6 {
+ historicalTestWrite(t, e, map[string]string{"k": "new"}, false)
+ }
+
+ compactor := e.NewContext()
+ defer compactor.Abort()
+
+ if _, _, err := compactor.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ ck := compactor.keys["k"]
+ if ck == nil {
+ t.Fatal("read did not emit a compaction snapshot")
+ }
+
+ snapshot, ok := e.effectCache.Get(ck.lastOffset)
+ if !ok || snapshot.GetSnapshot().GetState() == nil {
+ t.Fatal("compaction did not emit a state snapshot")
+ }
+
+ if err := compactor.Flush(); err != nil {
+ t.Fatal(err)
+ }
+
+ // A second snapshot makes the first an eligible reclamation
+ // boundary. Simulate the pacing interval without sleeping.
+ leaf := e.index.LoadData("k")
+ leaf.lastCompaction.Add(-int64(compactionInterval))
+
+ for range 6 {
+ historicalTestWrite(t, e, map[string]string{"k": "new"}, false)
+ }
+
+ if _, _, err := compactor.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ if compactor.keys["k"] == nil {
+ t.Fatal("read did not emit the second compaction snapshot")
+ }
+
+ if err := compactor.Flush(); err != nil {
+ t.Fatal(err)
+ }
+
+ // Adopt the compacted DAG, dropping the previous cached DAG's pins,
+ // then drain any references compaction made reclaimable.
+ got, _, _, err := e.GetSnapshot("k")
+ if err != nil || string(got.GetScalar().GetRaw()) != "new" {
+ t.Fatalf("live read after compaction: value=%v error=%v", got, err)
+ }
+
+ e.reclaimUnreferenced()
+
+ got, _, err = tx.GetSnapshot("k")
+ if err != nil || string(got.GetScalar().GetRaw()) != "old" {
+ t.Fatalf("transaction history lost after compaction: value=%v error=%v", got, err)
+ }
+
+ tx.Abort()
+ // Ending the transaction only removes its overlay. The following
+ // write forces a new live DAG build rather than a reduced-cache hit.
+ historicalTestWrite(t, e, map[string]string{"k": "later"}, false)
+
+ if _, _, _, err := e.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ e.reclaimUnreferenced()
+
+ if _, present := e.effectCache.Get(oldTip); present {
+ t.Fatal("history remained pinned after the transaction ended and compaction ran")
+ }
+ })
+ }
+}
+
+type historyBroadcaster struct {
+ mockBroadcaster
+
+ progress []HistoryProgress
+ processed []Tip
+}
+
+func (b *historyBroadcaster) HistoryProgress() []HistoryProgress {
+ return slices.Clone(b.progress)
+}
+func (b *historyBroadcaster) RecordHistorySent(pb.NodeID, uint64) {}
+func (b *historyBroadcaster) RecordHistoryProcessed(node pb.NodeID, count uint64) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ b.processed = append(b.processed, Tip{uint64(node), count})
+}
+
+func TestCompactionRequiresSubscriberEvidence(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ members []pb.NodeID
+ progress []HistoryProgress
+ witness Tip
+ reclaimOld bool
+ }{
+ {name: "disconnected", members: []pb.NodeID{2}, witness: Tip{42, 9}},
+ {name: "unrelated-higher-completion", members: []pb.NodeID{2}, progress: []HistoryProgress{{Node: 2, Processed: 10}}, witness: Tip{42, 9}},
+ {name: "processed-witness", members: []pb.NodeID{2}, progress: []HistoryProgress{{Node: 2, Processed: 9}}, witness: Tip{42, 9}, reclaimOld: true},
+ {name: "peer-authored-witness", members: []pb.NodeID{2}, progress: []HistoryProgress{{Node: 2, Minted: 3}}, witness: Tip{2, 3}, reclaimOld: true},
+ {name: "unknown-other-subscriber", members: []pb.NodeID{2, 3}, progress: []HistoryProgress{{Node: 2, Minted: 3}}, witness: Tip{2, 3}},
+ {name: "other-subscriber-removed", members: []pb.NodeID{2}, progress: []HistoryProgress{{Node: 2, Minted: 3}}, witness: Tip{2, 3}, reclaimOld: true},
+ {name: "all-subscribers-removed", witness: Tip{2, 3}, reclaimOld: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ b := &historyBroadcaster{progress: tc.progress}
+ b.peerIDs = tc.members
+
+ e := newTestEngine(b)
+ defer func() { _ = e.Close() }()
+
+ old, cut, latest := Tip{42, 1}, Tip{42, 8}, Tip{42, 20}
+ for tip, eff := range map[Tip]*pb.Effect{
+ old: {Key: []byte("k"), Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("old"))}},
+ cut: {Key: []byte("k"), Deps: toPbRefs([]Tip{old}), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}},
+ tc.witness: {Key: []byte("k"), Deps: toPbRefs([]Tip{cut}), Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}}},
+ latest: {Key: []byte("k"), Deps: toPbRefs([]Tip{tc.witness}), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}},
+ } {
+ if err := e.effectCache.Put(tip, eff); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ e.index.Insert("k", nil, keytrie.NewTipSet(latest))
+ e.addPeerSubscriber("k", 2)
+ e.addPeerSubscriber("k", 3)
+ d := newDag(e, "k", "")
+
+ d.compact = true
+ if err := d.prepare([]Tip{latest}); err != nil {
+ t.Fatal(err)
+ }
+
+ e.reclaimUnreferenced()
+
+ if v := e.effectCache.getVertex(cut); v == nil || v.root() != tc.reclaimOld {
+ t.Fatalf("retained snapshot root marker does not match reclamation: %v", tc.reclaimOld)
+ }
+
+ if _, present := e.effectCache.Get(old); present == tc.reclaimOld {
+ t.Fatalf("old history present=%t, want reclaimed=%t", present, tc.reclaimOld)
+ }
+
+ if _, present := e.effectCache.Get(latest); !present {
+ t.Fatal("reclaimed the live LCA snapshot")
+ }
+
+ if len(tc.members) != 0 {
+ if _, present := e.effectCache.Get(cut); !present {
+ t.Fatal("reclaimed the snapshot needed beneath the peer's witness")
+ }
+ }
+ })
+ }
+}
+
+func TestCompactionRetainsTwoSnapshots(t *testing.T) {
+ e := newTestEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ old, previous, between, latest := Tip{42, 1}, Tip{42, 2}, Tip{42, 3}, Tip{42, 4}
+ for tip, eff := range map[Tip]*pb.Effect{
+ old: {Key: []byte("k"), Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("old"))}},
+ previous: {Key: []byte("k"), Deps: toPbRefs([]Tip{old}), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}},
+ between: {Key: []byte("k"), Deps: toPbRefs([]Tip{previous}), Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("new"))}},
+ latest: {Key: []byte("k"), Deps: toPbRefs([]Tip{between}), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}},
+ } {
+ if err := e.effectCache.Put(tip, eff); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ for _, current := range []Tip{previous, latest} {
+ if _, ok := e.index.Insert("k", e.index.Contains("k"), keytrie.NewTipSet(current)); !ok {
+ t.Fatal("failed to install snapshot frontier")
+ }
+
+ d := newDag(e, "k", "")
+
+ d.compact = true
+ if err := d.prepare([]Tip{current}); err != nil {
+ t.Fatal(err)
+ }
+
+ e.reclaimUnreferenced()
+
+ if _, present := e.effectCache.Get(old); present != (current == previous) {
+ t.Fatalf("frontier %v: oldest history present=%t", current, present)
+ }
+
+ for _, retained := range []Tip{previous, between, latest} {
+ if _, present := e.effectCache.Get(retained); !present {
+ t.Fatalf("frontier %v: reclaimed retained snapshot or intervening history %v", current, retained)
+ }
+ }
+ }
+}
+
+func TestHistoryOverlaysConstrainEvidenceUntilTransactionEnds(t *testing.T) {
+ b := &historyBroadcaster{progress: []HistoryProgress{{Node: 2, Minted: 22, Processed: 5, Received: 17}}}
+ b.peerIDs = []pb.NodeID{2}
+
+ e := newTestEngine(b)
+ defer func() { _ = e.Close() }()
+
+ e.nextOff.Store(7)
+ older := e.NewContext()
+
+ older.BeginTx()
+ defer older.Abort()
+
+ e.nextOff.Store(9)
+
+ b.progress[0].Received = 22
+ newer := e.NewContext()
+
+ newer.BeginTx()
+ defer newer.Abort()
+
+ check := func(wantMy, wantYour uint64) {
+ t.Helper()
+
+ my, your := e.ConstrainHistoryProgress(2, 12, 30)
+ if my != wantMy || your != wantYour {
+ t.Fatalf("effective counts=(%d,%d), want=(%d,%d)", my, your, wantMy, wantYour)
+ }
+ }
+ check(7, 17)
+ older.Abort()
+ check(9, 22)
+
+ if err := newer.Flush(); err != nil {
+ t.Fatal(err)
+ }
+
+ check(12, 30)
+
+ if e.historyOverlays != nil {
+ t.Fatal("completed transactions retained their overlays")
+ }
+}
+
+func TestHistoryProgressRetriesIncompleteIngestion(t *testing.T) {
+ b := &historyBroadcaster{}
+
+ e := newTestEngine(b)
+ defer func() { _ = e.Close() }()
+
+ snapshot, missing := Tip{7, 3}, Tip{8, 2}
+ if err := e.effectCache.Put(snapshot, &pb.Effect{Key: []byte("k"), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}}}); err != nil {
+ t.Fatal(err)
+ }
+
+ e.installIngestedTips("k", snapshot, missing)
+
+ if len(b.processed) != 0 {
+ t.Fatal("incomplete ingestion advertised processed history")
+ }
+
+ if err := e.effectCache.Put(missing, &pb.Effect{Key: []byte("k"), Deps: toPbRefs([]Tip{snapshot}), Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("new"))}}); err != nil {
+ t.Fatal(err)
+ }
+ // Retrying the same arrivals repairs ancestry and advances evidence.
+ e.installIngestedTips("k", snapshot, missing)
+
+ if !slices.Contains(b.processed, missing) {
+ t.Fatal("a repaired duplicate arrival did not advance progress")
+ }
+}
+
+func TestCompactionAdvancesPastEndedOverlayWhileNewerTransactionLives(t *testing.T) {
+ e := newTestEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+ historicalTestWrite(t, e, map[string]string{"k": "old"}, false)
+ oldTip := e.index.Contains("k").Tips()[0]
+ older := e.NewContext()
+
+ older.BeginTx()
+ defer older.Abort()
+
+ compact := func(value string) Tip {
+ t.Helper()
+
+ // Exercise successive compaction passes without waiting in the test.
+ leaf := e.index.LoadData("k")
+ if leaf != nil && leaf.lastCompaction.Load() != 0 {
+ leaf.lastCompaction.Add(-int64(compactionInterval))
+ }
+
+ for range 6 {
+ historicalTestWrite(t, e, map[string]string{"k": value}, false)
+ }
+
+ c := e.NewContext()
+ defer c.Abort()
+
+ if _, _, err := c.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ if c.keys["k"] == nil {
+ t.Fatal("compaction did not emit a snapshot")
+ }
+
+ tip := c.keys["k"].lastOffset
+ if err := c.Flush(); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, _, _, err := e.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ return tip
+ }
+ boundary := compact("old")
+ newer := e.NewContext()
+
+ newer.BeginTx()
+ defer newer.Abort()
+
+ compact("new")
+ e.reclaimUnreferenced()
+
+ if _, ok := e.effectCache.Get(oldTip); !ok {
+ t.Fatal("oldest transaction lost its history")
+ }
+
+ older.Abort()
+ historicalTestWrite(t, e, map[string]string{"k": "newer"}, false)
+
+ if _, _, _, err := e.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ e.reclaimUnreferenced()
+
+ if _, ok := e.effectCache.Get(oldTip); ok {
+ t.Fatal("newer transaction unnecessarily retained history below its snapshot")
+ }
+
+ if _, ok := e.effectCache.Get(boundary); !ok {
+ t.Fatal("reclamation removed the newer transaction's snapshot boundary")
+ }
+
+ got, _, err := newer.GetSnapshot("k")
+ if err != nil || string(got.GetScalar().GetRaw()) != "old" {
+ t.Fatalf("newer transaction read after reclamation: value=%v error=%v", got, err)
+ }
+}
+
+func TestCompactionRetainsAncestryForDelayedNotification(t *testing.T) {
+ e := newTestEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ var delayed *pb.OffsetNotify
+
+ for i := range 8 {
+ historicalTestWrite(t, e, map[string]string{"k": fmt.Sprint(i)}, false)
+
+ if i == 2 {
+ tip := e.index.Contains("k").Tips()[0]
+ eff, _ := e.effectCache.Get(tip)
+
+ data, err := MarshalEffect(eff)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ delayed = BuildOffsetNotify(e.nodeID, tip, eff, data, context.Background())
+ }
+ }
+
+ compactor := e.NewContext()
+ defer compactor.Abort()
+
+ if _, _, err := compactor.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := compactor.Flush(); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, _, _, err := e.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ e.reclaimUnreferenced()
+
+ if _, err := e.HandleRemote(delayed); err != nil {
+ t.Fatal(err)
+ }
+
+ got, _, _, err := e.GetSnapshot("k")
+ if err != nil || string(got.GetScalar().GetRaw()) != "7" {
+ t.Fatalf("delayed notification broke compacted state: value=%v error=%v", got, err)
+ }
+}
+
+func TestCompactionRetainsCommonSnapshotsDuringRejoin(t *testing.T) {
+ for _, reverse := range []bool{false, true} {
+ for _, newerWitness := range []bool{false, true} {
+ t.Run(fmt.Sprintf("reverse=%t/newerWitness=%t", reverse, newerWitness), func(t *testing.T) {
+ b := &historyBroadcaster{progress: []HistoryProgress{{Node: 2, Minted: 3}}}
+ b.peerIDs = []pb.NodeID{2}
+
+ e := newTestEngine(b)
+ defer func() { _ = e.Close() }()
+
+ old, previous, witness, common := Tip{42, 1}, Tip{42, 2}, Tip{2, 3}, Tip{42, 4}
+ left, leftSnapshot, right, rightSnapshot, latest := Tip{42, 5}, Tip{42, 6}, Tip{2, 7}, Tip{2, 8}, Tip{42, 9}
+ snapshot := func(tip Tip, value int64, deps ...Tip) *pb.Effect {
+ eff := cutEffect(tip, deps...)
+ eff.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{
+ Op: pb.EffectOp_INSERT_OP, Merge: pb.MergeRule_ADDITIVE_INT,
+ Collection: pb.CollectionKind_SCALAR, Commutative: true,
+ Scalar: scalarInsertInt(pb.MergeRule_ADDITIVE_INT, value),
+ }}}
+
+ return eff
+ }
+
+ nodes := map[Tip]*pb.Effect{
+ old: cutEffect(old), previous: snapshot(previous, 1, old), witness: cutEffect(witness, previous),
+ common: snapshot(common, 2, witness), left: cutEffect(left, common), leftSnapshot: snapshot(leftSnapshot, 3, left),
+ right: cutEffect(right, common), rightSnapshot: snapshot(rightSnapshot, 3, right),
+ latest: snapshot(latest, 4, leftSnapshot, rightSnapshot),
+ }
+ if reverse {
+ slices.Reverse(nodes[latest].GetDeps())
+ nodes[leftSnapshot].ForkChoiceHash, nodes[rightSnapshot].ForkChoiceHash = nodes[rightSnapshot].GetForkChoiceHash(), nodes[leftSnapshot].GetForkChoiceHash()
+ }
+
+ if newerWitness {
+ b.progress[0].Minted = rightSnapshot[1]
+ }
+
+ putCutEffects(t, e, nodes)
+ e.index.Insert("k", nil, keytrie.NewTipSet(latest))
+ e.addPeerSubscriber("k", 2)
+ d := newDag(e, "k", "")
+
+ d.compact = true
+ if err := d.prepare([]Tip{latest}); err != nil {
+ t.Fatal(err)
+ }
+
+ e.reclaimUnreferenced()
+
+ retained := previous
+ if newerWitness {
+ retained = common
+ }
+
+ if v := e.effectCache.getVertex(retained); v == nil || !v.root() {
+ t.Fatalf("missing retained common boundary %v", retained)
+ }
+
+ for _, tip := range []Tip{common, left, leftSnapshot, right, rightSnapshot, latest} {
+ if _, ok := e.effectCache.Get(tip); !ok {
+ t.Fatalf("reclaimed rejoining history %v", tip)
+ }
+ }
+
+ for _, tip := range []Tip{leftSnapshot, rightSnapshot} {
+ if e.effectCache.getVertex(tip).root() {
+ t.Fatalf("branch snapshot marked as root: %v", tip)
+ }
+ }
+
+ if _, ok := e.effectCache.Get(old); ok {
+ t.Fatal("eligible history below common boundary was retained")
+ }
+
+ if _, ok := e.effectCache.Get(witness); ok == newerWitness {
+ t.Fatalf("subscriber boundary ignored: witness resident=%t", ok)
+ }
+
+ // The peer writes against its branch while the merged snapshot is arriving.
+ delayed := Tip{2, 10}
+ putCutEffects(t, e, map[Tip]*pb.Effect{delayed: cutEffect(delayed, rightSnapshot)})
+
+ state, _, err := e.reconstruct("k", []Tip{latest, delayed}, "", false)
+ if err != nil || state.GetScalar().GetIntVal() != 5 {
+ t.Fatalf("rejoining reconstruction=%v error=%v, want 5", state, err)
+ }
+ })
+ }
+ }
+}
diff --git a/effects/compaction_pacing_test.go b/effects/compaction_pacing_test.go
new file mode 100644
index 0000000..6cad1b1
--- /dev/null
+++ b/effects/compaction_pacing_test.go
@@ -0,0 +1,83 @@
+package effects
+
+import (
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestCompactionPacing(t *testing.T) {
+ var leaf leafState
+
+ first := int64(time.Second)
+ if !leaf.claimCompaction(first) {
+ t.Fatal("first compaction was delayed")
+ }
+
+ if leaf.claimCompaction(first + int64(compactionInterval) - 1) {
+ t.Fatal("compaction ran before the minimum interval")
+ }
+
+ if !leaf.claimCompaction(first + int64(compactionInterval)) {
+ t.Fatal("compaction did not become eligible at the interval boundary")
+ }
+
+ if leaf.claimCompaction(first) {
+ t.Fatal("an older clock sample claimed a compaction")
+ }
+}
+
+func TestConcurrentReadersClaimOneCompaction(t *testing.T) {
+ var (
+ leaf leafState
+ winners atomic.Int32
+ wg sync.WaitGroup
+ )
+ for range 32 {
+ wg.Go(func() {
+ if leaf.claimCompaction(int64(time.Second)) {
+ winners.Add(1)
+ }
+ })
+ }
+
+ wg.Wait()
+
+ if got := winners.Load(); got != 1 {
+ t.Fatalf("concurrent readers claimed %d compactions, want 1", got)
+ }
+}
+
+func TestAutomaticCompactionPacingIsPerKey(t *testing.T) {
+ e := newTestEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "a", "b")
+
+ for _, step := range []struct {
+ key string
+ want bool
+ }{{"a", true}, {"a", false}, {"b", true}} {
+ for range 8 {
+ historicalTestWrite(t, e, map[string]string{step.key: "value"}, false)
+ }
+
+ c := e.NewContext()
+
+ got, _, err := c.GetSnapshot(step.key)
+ if err != nil || string(got.GetScalar().GetRaw()) != "value" {
+ c.Abort()
+ t.Fatalf("read while pacing: value=%v error=%v", got, err)
+ }
+
+ if emitted := c.keys[step.key] != nil; emitted != step.want {
+ c.Abort()
+ t.Fatalf("key %q: snapshot emitted=%t, want %t", step.key, emitted, step.want)
+ }
+
+ if err := c.Flush(); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
diff --git a/effects/concurrent_history_test.go b/effects/concurrent_history_test.go
index c4cdfa7..09b6dfb 100644
--- a/effects/concurrent_history_test.go
+++ b/effects/concurrent_history_test.go
@@ -248,13 +248,18 @@ func TestLosersOnKey_ConcurrentPredicatesMatchOracle(t *testing.T) {
// The fixture's barriers order all different rounds. Within a
// round, use plain integer equality as an independent predicate
- // oracle, including conservative legacy binds and LOST exclusion.
+ // oracle. Candidates are in ascending hash order within each round;
+ // only surviving predecessors can eliminate the next candidate.
for _, a := range candidates {
if invisible[a.txn] || verdicts[a.txn].verdict == pb.Verdict_LOST {
continue
}
for _, b := range candidates {
+ if _, eliminated := want[b.txn]; eliminated {
+ continue
+ }
+
if b.round != a.round || b.hash >= a.hash || invisible[b.txn] || verdicts[b.txn].verdict == pb.Verdict_LOST {
continue
}
diff --git a/effects/context.go b/effects/context.go
index 02ec940..70c7ebf 100644
--- a/effects/context.go
+++ b/effects/context.go
@@ -27,13 +27,14 @@ import (
"bytes"
"context"
"encoding/binary"
+ "errors"
"fmt"
"log/slog"
- "maps"
rand "math/rand/v2"
"slices"
"sort"
"sync"
+ "time"
pb "github.com/swytchdb/engine/cluster/proto"
"github.com/swytchdb/engine/keytrie"
@@ -64,11 +65,13 @@ type Context struct {
keys map[string]*contextKey
watchedKeys map[string]*watchedKeyState // keys registered via Watch with WATCH-time state
txSnapshot *transactionSnapshot // compact boundary and lazily captured causal history
- readErr error // failed transactional reads must not flush queued writes
+ history historyOverlay
+ readErr error // failed transactional reads must not flush queued writes
// readOnly marks a context used only for read-only, non-transactional
// commands. Such a context may answer a read-miss from the cluster key
// filters without subscribing (free misses), and skips chain compaction.
+ // Recording an observation or emitting an effect makes it writable.
readOnly bool
}
@@ -176,6 +179,7 @@ func (e *Engine) NewContext() *Context {
// NewReadOnlyContext creates a context for read-only, non-transactional
// commands. It uses the per-node cluster key filters to answer a read-miss
// without issuing a subscription — a key no peer holds returns nil for free.
+// Explicit observations or effect emission promote it to a write context.
// It must not be used inside MULTI/EXEC (SSI reads must subscribe).
func (e *Engine) NewReadOnlyContext() *Context {
return &Context{
@@ -288,6 +292,13 @@ func (c *Context) getSnapshot(key string) (*pb.ReducedEffect, []Tip, error) {
}
if !c.readOnly && result != nil && chainLen >= compactThreshold {
+ ls := c.engine.index.LoadData(key)
+
+ stamp := time.Since(compactionEpoch).Nanoseconds() + 1
+ if ls == nil || !ls.claimCompaction(stamp) {
+ return filterSnapshot(result), tips, nil
+ }
+
slog.Debug("compaction: emitting snapshot",
"key", key,
"chainLen", chainLen,
@@ -324,6 +335,7 @@ func (c *Context) getSnapshot(key string) (*pb.ReducedEffect, []Tip, error) {
Key: []byte(key),
Kind: &pb.Effect_Snapshot{Snapshot: snapEff},
}, tips); err != nil {
+ ls.lastCompaction.CompareAndSwap(stamp, 0)
slog.Error("compaction: snapshot emit failed",
"key", key,
"chainLen", chainLen,
@@ -502,18 +514,18 @@ func (c *Context) getSnapshotFromTx(key string) (*pb.ReducedEffect, []Tip, error
// between WATCH and EXEC. The NOOP offset and post-flush TipSet pointer
// are stored for comparison at BeginTx time.
//
-// The noop is tx-marked with the upcoming transaction's id (generated
-// lazily here). This is what makes write-skew detection work across
-// nodes: a competing write on a watched key becomes a structural fork
-// sibling of the watching tx's bind, surfaced by reconstruct's
-// pairwise fork-choice on the key. Without the TxnId, the noop is an
-// anonymous committed effect that other writers can chain off,
-// producing false sequential relationships (the run-25890153673 G1a).
+// The NOOP carries the upcoming transaction's id so its observation can be
+// validated with that transaction. Other observations and snapshots may consume
+// it: only observing the bind establishes that the transaction has committed.
//
// Errors fail the WATCH: a read that can't be answered (e.g. the Cloud
// consult failed for an evicted key) must not silently record hadData=false —
// the EXEC verdict would then rest on a fabricated observation.
func (c *Context) Watch(key string) error {
+ // An observation writes a NOOP into causal history. Reuse the ordinary
+ // compaction path before recording it, including for read-only callers.
+ c.readOnly = false
+
if c.watchedKeys == nil {
c.watchedKeys = make(map[string]*watchedKeyState)
}
@@ -525,7 +537,7 @@ func (c *Context) Watch(key string) error {
}
// Check if key has actual data BEFORE emitting the NOOP
- snap, _, _, err := c.engine.GetSnapshot(key)
+ snap, _, err := c.GetSnapshot(key)
if err != nil {
return err
}
@@ -590,6 +602,8 @@ func (c *Context) BeginTx() {
return // nested transaction inherits outer
}
+ c.engine.pinHistory(&c.history)
+
c.inTx = true
// A prior WATCH may have already materialized our txn id; reuse it
// so the WATCH noop and the MULTI/EXEC effects share one tx.
@@ -610,6 +624,7 @@ func (c *Context) BeginTx() {
// Returns false if any watched key was modified — the caller should abort
// the transaction and return a null array to the client.
func (c *Context) CheckWatches() bool {
+ c.engine.pinHistory(&c.history)
// BeginTx already captured the SSI snapshot at txn start. Cover the
// case where CheckWatches is called without a prior BeginTx (no
// transactional emits before EXEC).
@@ -633,11 +648,17 @@ func (c *Context) CheckWatches() bool {
if ws.flushGen != currentFlushGen && !ws.hadData && currentTips == nil {
continue // key didn't exist at WATCH time, still doesn't after flush
}
+
+ if !c.engine.hasWritesSince(key, ws.tips.Tips(), c.txnID) {
+ continue // Only observations or compaction changed the frontier.
+ }
+
// Key was modified — abort
c.watchedKeys = nil
c.inTx = false
c.txnID = ""
c.txSnapshot = nil
+ c.engine.unpinHistory(&c.history)
return false
}
@@ -656,6 +677,7 @@ func (c *Context) CheckWatches() bool {
c.inTx = false
c.txnID = ""
c.txSnapshot = nil
+ c.engine.unpinHistory(&c.history)
return false
}
@@ -692,6 +714,8 @@ func (c *Context) Abort() {
// tips the handler actually read, not whatever the index contains now.
// Pure writes (SET, LPUSH) omit snapshotTips and Emit reads the index.
func (c *Context) Emit(eff *pb.Effect, snapshotTips ...[]Tip) error {
+ c.engine.pinHistory(&c.history)
+
if c.readErr != nil {
return c.readErr
}
@@ -778,7 +802,7 @@ func (c *Context) Emit(eff *pb.Effect, snapshotTips ...[]Tip) error {
if tips != nil {
tipOffsets := tips.Tips()
- // Exclude in-progress tx tips: substitute pre-tx deps
+ // Exclude tentative values, retaining observed NOOPs as dependencies.
eff.Deps = toPbRefs(c.engine.resolveTipDeps(tipOffsets))
}
} else {
@@ -820,7 +844,7 @@ func (c *Context) Emit(eff *pb.Effect, snapshotTips ...[]Tip) error {
// A RowWriteEffect is semantic evidence of a row-level write
// on this key — flip readOnly so SSI's "any concurrent write
// aborts" doesn't fire on write-carrying txs. The refined
- // predicate check in checkCompetingBinds / evaluateBindForkChoice
+ // predicate check in checkCompetingBinds / classifyNackTip
// handles conflict detection for these txs.
if eff.GetRowWrite() != nil {
ck.readOnly = false
@@ -858,6 +882,7 @@ func (c *Context) Emit(eff *pb.Effect, snapshotTips ...[]Tip) error {
// Update contextKey
ck.lastOffset = offset
ck.notifies = append(ck.notifies, notify)
+ c.readOnly = false
// Defer peer-subscriber tracking to Flush: the subscriber set lives on the
// key's leaf, which updateIndex only creates at Flush time. Self-subscribes
@@ -963,7 +988,7 @@ func (c *Context) rawEmit(eff *pb.Effect) (Tip, *pb.OffsetNotify, error) {
return offset, nil, nil
}
- data, err := MarshalEffect(eff)
+ data, err := c.engine.effectCache.MarshalEffect(eff)
if err != nil {
return Tip{}, nil, err
}
@@ -984,6 +1009,8 @@ func (c *Context) rawEmit(eff *pb.Effect) (Tip, *pb.OffsetNotify, error) {
// Flush updates the index for all touched keys and broadcasts all
// notifications per key for durability, then resets the context for reuse.
func (c *Context) Flush() error {
+ defer c.engine.unpinHistory(&c.history)
+
if c.readErr != nil {
err := c.readErr
c.Abort()
@@ -1143,7 +1170,6 @@ func (c *Context) flushNonTx() error {
func (c *Context) flushTx() error {
slog.Debug("Flush: transactional commit", "keys", len(c.keys))
c.inTx = false
- hadSnapshot := c.txSnapshot != nil
c.txSnapshot = nil
if len(c.keys) == 0 {
@@ -1181,78 +1207,30 @@ func (c *Context) flushTx() error {
}
}
- // Protocol invariant: once this node has observed a remote bind on
- // key K, it must not emit its own bind that's DAG-concurrent with
- // the remote bind. Two ways that can happen, both caught here:
- // (a) the remote bind consumed one of our initialTips — our tip
- // is gone from current.
- // (b) the remote bind landed as a new tip after we captured
- // initialTips — current has a tip our initialTips never saw.
- // In (b), our emit won't dep-reference the remote bind so the two
- // commit DAG-concurrent. Abort here rather than let fork-choice
- // surface the inconsistency as G1a / incompatible-order later.
+ // A changed frontier can contain only observations or compaction. Validate
+ // writes, not tip identity; retain the promises made for consumed binds.
var spokenBinds []Tip
for key, ck := range c.keys {
- if ck.initialTips == nil {
- continue // first write to a brand-new key
- }
-
- initial := ck.initialTips.Tips()
- if len(initial) == 0 {
- continue
- }
-
- current := c.engine.index.Contains(key)
+ if c.engine.hasWritesSince(key, ck.initialTips.Tips(), c.txnID) {
+ slog.Debug("flushTx: write after observation, aborting", "key", key, "txn", c.txnID)
- for _, t := range initial {
- if !current.Contains(t) {
- slog.Debug("flushTx: stale consumed tip, aborting before emission",
- "key", key, "stale_tip", t, "current_tips", current)
+ for _, pending := range c.keys {
+ c.engine.pendingTxTips.Delete(pending.lastOffset)
+ }
- for _, ck3 := range c.keys {
- c.engine.pendingTxTips.Delete(ck3.lastOffset)
- }
+ c.reset()
- c.reset()
+ return ErrTxnAborted
+ }
- return ErrTxnAborted
- }
- // A retained consumed tip may be redundant ancestry. Remember the
- // consuming binds so we can check actual reachability below.
- if c.engine.spokenBinds != nil {
+ if c.engine.spokenBinds != nil {
+ for _, t := range ck.initialTips.Tips() {
if binds, spoken := c.engine.spokenBinds.Get(t, 0); spoken {
spokenBinds = append(spokenBinds, binds.snapshot()...)
}
}
}
-
- for _, t := range current.Tips() {
- if ck.initialTips.Contains(t) {
- continue
- }
- // Subscription tips are commutative metadata, not bind writes
- // — they can't be the "remote bind landed during txn window"
- // case this axis exists to catch. ensureSubscribed installs
- // the originator's own SubscriptionEffect into the index
- // during flushTx, AFTER initialTips was captured at Emit
- // time, so the originator's own subscription would otherwise
- // look like a competitor and trigger a spurious abort.
- if eff, ok := c.engine.effectCache.Get(t); ok && eff.GetSubscription() != nil {
- continue
- }
-
- slog.Debug("flushTx: concurrent remote bind landed, aborting before emission",
- "key", key, "new_tip", t, "initial_tips", initial)
-
- for _, ck3 := range c.keys {
- c.engine.pendingTxTips.Delete(ck3.lastOffset)
- }
-
- c.reset()
-
- return ErrTxnAborted
- }
}
if len(spokenBinds) > 0 {
@@ -1289,7 +1267,9 @@ func (c *Context) flushTx() error {
preTxDeps = ck.initialTips.Tips()
}
- c.engine.pendingTxTips.Store(ck.lastOffset, preTxDeps)
+ if eff, _ := c.engine.effectCache.Get(ck.lastOffset); eff.GetNoop() == nil {
+ c.engine.pendingTxTips.Store(ck.lastOffset, preTxDeps)
+ }
if c.engine.broadcaster != nil {
for _, n := range ck.notifies {
@@ -1298,111 +1278,21 @@ func (c *Context) flushTx() error {
}
}
- // Step 1.5: SSI fork validation.
- // When a snapshot is active, ALL keys use snapshot deps. Any fork means
- // a concurrent write happened since the snapshot — SSI conflict.
- // Without a snapshot, only check readOnly keys (WATCH NOOPs).
- for key, ck := range c.keys {
- if !hadSnapshot && !ck.readOnly {
- continue
- }
-
- tips := c.engine.index.Contains(key)
- if tips == nil {
- continue
- }
-
- myTip := ck.lastOffset
- for _, t := range tips.Tips() {
- if t == myTip {
- continue // our own tip
- }
-
- isOurs := false
-
- for _, ck2 := range c.keys {
- if ck2.lastOffset == t {
- isOurs = true
-
- break
- }
- }
-
- if isOurs {
- continue
- }
- // Subscription tips are commutative metadata, not data writes
- // — they cannot create a data fork. ensureSubscribed installs
- // the originator's own SubscriptionEffect into the index
- // during flushTx, after the SSI snapshot was taken by
- // BeginTx, so without this skip the originator's own
- // subscription tip would look like a concurrent external
- // write and trigger a spurious abort.
- if eff, ok := c.engine.effectCache.Get(t); ok && eff.GetSubscription() != nil {
- continue
- }
- // Concurrent effect on watched key → SSI conflict
- slog.Debug("SSI conflict: concurrent write on watched key",
- "tip", t, "our_tip", myTip)
-
- for _, ck3 := range c.keys {
- c.engine.pendingTxTips.Delete(ck3.lastOffset)
- }
-
- c.reset()
-
- return ErrTxnAborted
- }
- }
-
- // Step 2: Collect competing in-progress tx offsets for abort_deps
- var abortDeps []*pb.EffectRef
-
- for key := range c.keys {
- tips := c.engine.index.Contains(key)
- if tips == nil {
- continue
- }
-
- for _, t := range tips.Tips() {
- if _, isPending := c.engine.pendingTxTips.Load(t); isPending {
- // Check it's not our own tip
- isOurs := false
-
- for _, ck := range c.keys {
- if ck.lastOffset == t {
- isOurs = true
-
- break
- }
- }
-
- if !isOurs {
- abortDeps = append(abortDeps, toPbRef(t))
- }
- }
- }
- }
-
- // Step 3: Build TransactionalBindEffect
+ // Build the bind. Other tentative tips on the frontier are not abort
+ // evidence: competing binds are adjudicated below and by peer responses.
txnHLC := c.engine.clock.Now()
hlcTs := timestamppb.New(txnHLC)
bind := &pb.TransactionalBindEffect{
TxnHlc: timestamppb.New(txnHLC),
OriginatorNodeId: uint64(c.engine.nodeID),
- AbortDeps: abortDeps,
}
for key, ck := range c.keys {
var consumedTips []*pb.EffectRef
if ck.initialTips != nil {
- // ConsumedTips is a cluster-visible causal reference, subject to
- // the same SSI rule as eff.Deps: it must name committed state,
- // never a foreign in-flight tx tip. resolveTipDeps substitutes any
- // such tip for its pre-tx committed base — the same call that
- // produced this key's data-effect Deps at Emit time. Without it the
- // bind records an uncommitted (and possibly aborting) peer write as
- // part of its committed base.
+ // Match the data effect's value base, retaining causal observations
+ // while substituting tentative writes. A consumed NOOP does not
+ // imply that its transaction's future bind has been observed.
consumedTips = toPbRefs(c.engine.resolveTipDeps(ck.initialTips.Tips()))
}
@@ -1435,6 +1325,21 @@ func (c *Context) flushTx() error {
nodeLock := c.takeNodeLock()
defer nodeLock.release()
+ // Recheck writes at the publication boundary too.
+ for key, ck := range c.keys {
+ if c.engine.hasWritesSince(key, ck.initialTips.Tips(), c.txnID) {
+ slog.Debug("flushTx: write after observation, aborting", "key", key, "txn", c.txnID)
+
+ for _, pending := range c.keys {
+ c.engine.pendingTxTips.Delete(pending.lastOffset)
+ }
+
+ c.reset()
+
+ return ErrTxnAborted
+ }
+ }
+
// Every previously ingested structural competitor counts, including binds
// still in the horizon. We cannot publish after promising not to compete.
if conflict := c.engine.checkCompetingBinds(bind, c.txnID); conflict != "" {
@@ -1490,19 +1395,6 @@ func (c *Context) flushTx() error {
c.engine.updateIndex(key, keytrie.NewTipSet(ck.lastOffset), bindOffset)
}
- if c.engine.evaluateBindForkChoice(bind, bindOffset, bindEff.GetForkChoiceHash(), c.txnID) {
- slog.Debug("flushTx: voided by concurrent bind, aborting",
- "bind_offset", bindOffset, "txn", c.txnID)
-
- for _, ck := range c.keys {
- c.engine.pendingTxTips.Delete(ck.lastOffset)
- }
-
- c.reset()
-
- return ErrTxnAborted
- }
-
// Step 4: Create pending txn for NACK tracking
ptxn := &pendingTxn{
txnID: c.txnID,
@@ -1511,7 +1403,10 @@ func (c *Context) flushTx() error {
forkChoiceHash: bindHash,
bindOffset: bindOffset,
done: make(chan struct{}),
+ resolved: make(chan struct{}),
}
+ defer close(ptxn.resolved)
+
for key, ck := range c.keys {
var consumedTips []Tip
if ck.initialTips != nil {
@@ -1603,111 +1498,61 @@ func (c *Context) flushTx() error {
wg.Wait()
- // Competitor txnIDs we beat during the NACK loop, keyed by
- // txnID with the loser's bind anchor offset as value. The
- // offset is needed at verdict-snapshot emit time so we can
- // fetch the loser's bind and union its keyset with ours —
- // otherwise the verdict for `beaten` is only published on
- // keys WE touched, leaving any key the loser touched alone
- // without an authoritative LOST record. A subsequent reader
- // of such a key would have to rely on cross-key closure
- // expansion to discover the verdict, which is fragile under
- // tx-context reconstructs with pre-tx snapshot cutoffs.
- defeatedSet := make(map[string]Tip)
-
responseCount := 0
for _, res := range results {
if res.err != nil {
- slog.Warn("flushTx: ReplicateTo failed",
- "bind_offset", bindOffset,
- "target", res.subID,
- "error", res.err)
+ slog.Warn("flushTx: ReplicateTo failed", "bind_offset", bindOffset, "target", res.subID, "error", res.err)
continue
}
responseCount++
- for _, nack := range res.nacks {
- slog.Debug("flushTx: received NACK",
- "bind_offset", bindOffset,
- "target", res.subID,
- "nack_key", string(nack.GetKey()),
- "tip_count", len(nack.GetTipDetails()))
- // Pull every tip the peer mentions into our DAG
- // before deciding conflict — the NACK is informational
- // and ignoring it would let the cluster diverge.
- c.engine.ingestNackTips(nack)
-
- for _, detail := range nack.GetTipDetails() {
- conflict, defeated := c.engine.checkNackConflict(ptxn, string(nack.GetKey()), detail)
- if conflict {
- slog.Debug("flushTx: real conflict detected, aborting",
- "bind_offset", bindOffset,
- "nack_key", string(nack.GetKey()),
- "detail_offset", detail.GetRef(),
- "detail_is_bind", detail.GetIsBind(),
- "detail_is_data", detail.GetIsData(),
- "detail_is_tx", detail.GetIsTransactional())
- ptxn.finish(txnStateAborted)
-
- break
- }
-
- maps.Copy(defeatedSet, defeated)
- }
+ ptxn.responses = append(ptxn.responses, res.nacks...)
+ }
+ // Ingest all responses before evaluating: the origin of a better
+ // priority bind may supply evidence absent from earlier replies.
+ for _, nack := range ptxn.responses {
+ c.engine.ingestNackTips(nack)
+ }
- if ptxn.state.Load() == txnStateAborted {
- break
- }
- }
+ var (
+ defeatedSet map[string]Tip
+ proof *pb.BindEvidence
+ )
- if ptxn.state.Load() == txnStateAborted {
- break
- }
- }
+ decision := txnStateAmbiguous
- if ptxn.state.Load() != txnStateAborted {
- if responseCount < len(subscribers) {
- slog.Warn("flushTx: not all subscribers responded, outcome unknown",
- "bind_offset", bindOffset,
- "responses", responseCount,
- "subscribers", len(subscribers))
- ptxn.finish(txnStateAmbiguous)
- } else if ptxn.finish(txnStateCommitted) {
- slog.Debug("flushTx: all subscribers responded, committed",
- "bind_offset", bindOffset,
- "subscribers", len(subscribers))
+ if responseCount == len(subscribers) {
+ var err error
+
+ defeatedSet, err = c.engine.evaluateNacks(ptxn, ptxn.responses)
+ if err == nil {
+ proof, err = c.engine.bindProof(bindOffset, defeatedSet, ptxn.proofSources)
+ defer func() { c.engine.effectCache.releaseEvidence(evidenceTips(proof)) }()
}
- }
- // Drive visibility from here, AFTER the NACK loop has had a
- // chance to set txnStateAborted via isRealConflict. The bind
- // must not become visible before this decision is made.
- if c.engine.horizon != nil {
- if ptxn.state.Load() != txnStateCommitted {
- c.engine.horizon.Abort(c.txnID)
- } else {
- c.engine.horizon.MakeVisible(c.txnID)
+ switch {
+ case errors.Is(err, ErrTxnAborted):
+ decision = txnStateAborted
+ case err != nil:
+ decision = txnStateAmbiguous
+ default:
+ decision = txnStateCommitted
}
+
+ slog.Debug("flushTx: priority evaluation", "bind_offset", bindOffset, "state", decision, "error", err)
}
- if ptxn.state.Load() == txnStateCommitted {
+ if decision == txnStateCommitted {
verdicts := make(map[string]pb.Verdict, 1+len(defeatedSet))
verdicts[c.txnID] = pb.Verdict_WON
- // Emit set: every key the winner touched, plus every key
- // any beaten loser touched. The verdict for a LOST txn
- // has to be findable on each key the loser wrote — that
- // is the only way a future reconstruct on a loser-only
- // key (one the winner never touched) can prove the bind
- // is dead without re-running cross-key closure expansion
- // from scratch. Recording LOST only on the winner's keys
- // satisfies losersOnKey there but leaves loser-only keys
- // dependent on closure widening, which can be truncated
- // by a tx-context reconstruct's pre-tx cutoff and
- // surface as G1a.
+ // Cache the computed outcomes on our keys and the defeated
+ // binds' keys. Readers and deferred responses can reuse this
+ // evidence without rediscovering cross-key competitors.
+
emitKeys := make(map[string]struct{}, len(bind.GetKeys()))
for _, kb := range bind.GetKeys() {
emitKeys[string(kb.GetKey())] = struct{}{}
@@ -1743,7 +1588,9 @@ func (c *Context) flushTx() error {
}
for key := range emitKeys {
- if err := c.engine.emitSnapshot(key, verdicts); err != nil {
+ if err := c.engine.emitSnapshot(key, verdicts, proof); err != nil {
+ decision = txnStateAmbiguous
+
slog.Warn("flushTx: emitSnapshot for verdict failed",
"key", key,
"bind_offset", bindOffset,
@@ -1751,6 +1598,17 @@ func (c *Context) flushTx() error {
}
}
}
+ // Evidence ownership must be published before local completion and
+ // before deferred priority responses can use this outcome.
+ ptxn.finish(decision)
+
+ if c.engine.horizon != nil {
+ if decision == txnStateCommitted {
+ c.engine.horizon.MakeVisible(c.txnID)
+ } else {
+ c.engine.horizon.Abort(c.txnID)
+ }
+ }
}
} else {
// Standalone: commit immediately
@@ -1861,6 +1719,7 @@ func (c *Context) emitSerializationEffect(key string) {
// reset clears the context for reuse.
func (c *Context) reset() {
+ c.engine.unpinHistory(&c.history)
// Reuse small command maps, but let an unusually large transaction's map
// go: clear alone retains its buckets for the lifetime of the context.
if len(c.keys) > 16 {
@@ -1918,40 +1777,6 @@ func (e *Engine) updateIndex(key string, initialTips *keytrie.TipSet, lastOffset
}
}
-// installTips merges a whole tip frontier into the key's index in one CAS
-// transition, so a concurrent reader observes either the prior frontier or the
-// full merged one — never a partial install. Same accounting as updateIndex:
-// Insert fires the trie's refDelta hook for the diff, and NewTipSet dedupes
-// tips the index already holds.
-func (e *Engine) installTips(key string, tips []Tip) {
- for attempt := 0; ; attempt++ {
- if attempt > 0 {
- slog.Debug("installTips: CAS retry", "key", key, "attempt", attempt)
- }
-
- current := e.index.Contains(key)
- cur := current.Tips()
- offsets := make([]Tip, 0, len(cur)+len(tips))
- offsets = append(offsets, cur...)
- offsets = append(offsets, tips...)
-
- newTips := keytrie.NewTipSet(offsets...)
- if _, ok := e.index.Insert(key, current, newTips); ok {
- if current.Len() != newTips.Len() || !current.ContainsAll(newTips.Tips()) {
- e.notifyKeyChanged(key)
- }
-
- return
- }
-
- if e.index.Closed() {
- // Not a CAS race: Insert fails unconditionally on a closed trie,
- // so retrying would spin forever during shutdown.
- return
- }
- }
-}
-
// BuildOffsetNotify constructs a single-effect notification.
// EffectData is wire format: [4-byte LE keyLen][key][protoData].
// traceCtx is optional; when non-nil, OTel trace context is injected into the notification.
diff --git a/effects/cross_key_history_test.go b/effects/cross_key_history_test.go
new file mode 100644
index 0000000..e971aa9
--- /dev/null
+++ b/effects/cross_key_history_test.go
@@ -0,0 +1,468 @@
+package effects
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "google.golang.org/protobuf/proto"
+)
+
+// A synthetic version of the el-0/el-4 incident, not a replay of its logs.
+// el-4 has two state snapshots above a shared bind. el-0 still needs the
+// bind and its own data to compare it with a concurrent, losing transaction.
+// No verdict cache is supplied: reconstruction must derive the outcome.
+type crossKeyHistory struct {
+ engine *Engine
+ shared, competitor Tip
+ data0, data4 Tip
+ previous, latest Tip
+}
+
+func newCrossKeyHistory(t *testing.T, gap bool) crossKeyHistory {
+ t.Helper()
+
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "el-0", "el-4")
+ f := crossKeyHistory{engine: e}
+ f.shared = closureBind(log, "shared", 1, []string{"el-0", "el-4"}, nil)
+ bind, _ := e.effectCache.Get(f.shared)
+ f.data0 = r(bind.GetTxnBind().GetKeys()[0].GetNewTip())
+ f.data4 = r(bind.GetTxnBind().GetKeys()[1].GetNewTip())
+ otherData := log.putEffect(&pb.Effect{
+ Key: []byte("el-0"), TxnId: "competitor",
+ Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("wrong"))},
+ })
+ f.competitor = log.putEffect(&pb.Effect{
+ Key: []byte("el-0"), TxnId: "competitor", ForkChoiceHash: []byte{2},
+ Deps: toPbRefs([]Tip{otherData}),
+ Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{
+ Keys: []*pb.TransactionalBindEffect_KeyBind{{Key: []byte("el-0"), NewTip: toPbRef(otherData)}},
+ }},
+ })
+ state := func(parent Tip) Tip {
+ return log.putEffect(&pb.Effect{
+ Key: []byte("el-4"), Deps: toPbRefs([]Tip{parent}),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ State: &pb.ReducedEffect{Scalar: scalarInsertRaw([]byte("100"))},
+ }},
+ })
+ }
+
+ parent := f.shared
+ if gap {
+ parent = log.putEffect(&pb.Effect{Key: []byte("el-4"), Deps: toPbRefs([]Tip{parent}), Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}}})
+ }
+
+ f.previous = state(parent)
+ f.latest = state(f.previous)
+ e.installIngestedTips("el-0", f.shared, f.competitor)
+ e.installIngestedTips("el-4", f.latest)
+
+ return f
+}
+
+func (f crossKeyHistory) compact(t *testing.T) {
+ t.Helper()
+
+ d := newDag(f.engine, "el-4", "")
+
+ d.compact = true
+ if err := d.prepare([]Tip{f.latest}); err != nil {
+ t.Fatal(err)
+ }
+
+ if d.lcaTip != f.latest {
+ t.Fatalf("el-4 LCA = %v, want %v", d.lcaTip, f.latest)
+ }
+
+ f.engine.reclaimUnreferenced()
+}
+
+func TestCompactionPreservesOtherKeyBindOwnership(t *testing.T) {
+ f := newCrossKeyHistory(t, false)
+ p := f.engine.effectCache
+
+ shared := p.getVertex(f.shared)
+ if got := shared.refs.Load(); got != 2 {
+ t.Fatalf("shared bind has %d refs before compaction, want one per key", got)
+ }
+
+ f.compact(t)
+
+ if p.getVertex(f.shared) != shared || shared.refs.Load() != 1 {
+ t.Fatal("el-4 compaction did not preserve el-0's reference to the shared bind")
+ }
+
+ shared.creationMu.Lock()
+ owners := slices.Clone(shared.creationKeys)
+ shared.creationMu.Unlock()
+
+ if !slices.Equal(owners, []string{"el-0"}) {
+ t.Fatalf("bind owners = %v, want only el-0", owners)
+ }
+
+ if _, ok := p.Get(f.data4); ok {
+ t.Fatal("el-4 data below both snapshots was not reclaimed")
+ }
+
+ for _, tip := range []Tip{f.data0, f.competitor, f.previous, f.latest} {
+ if _, ok := p.Get(tip); !ok {
+ t.Fatalf("compaction reclaimed required vertex %v", tip)
+ }
+ }
+ // el-0's own DAG remains complete after the other key's compaction.
+ d := newDag(f.engine, "el-0", "")
+ if err := d.prepare(f.engine.index.Contains("el-0").Tips()); err != nil {
+ t.Fatal(err)
+ }
+
+ if d.nodes[f.shared] == nil || d.nodes[f.data0] == nil {
+ t.Fatal("el-0 no longer reaches its bind and data")
+ }
+
+ losers, _ := f.engine.losersOnKey("el-0", nil, nil)
+ if _, lost := losers["competitor"]; !lost || len(losers) != 1 {
+ t.Fatalf("retained el-0 evidence gives losers %v, want only competitor", losers)
+ }
+ // Releasing the final key must release the shared envelope too.
+ f.engine.releaseChainRefs("el-0", f.engine.index.Contains("el-0").Tips(), nil)
+ f.engine.reclaimUnreferenced()
+
+ if _, ok := p.Get(f.shared); ok {
+ t.Fatal("shared bind leaked after both keys released their ownership")
+ }
+}
+
+func TestBindReconstructionAfterOtherKeyCompaction(t *testing.T) {
+ for _, gap := range []bool{false, true} {
+ var baseline []*pb.ReducedEffect
+
+ for _, compact := range []bool{false, true} {
+ t.Run(fmt.Sprintf("gap=%t/compacted=%t", gap, compact), func(t *testing.T) {
+ f := newCrossKeyHistory(t, gap)
+ // The committing key retains the resolution proof before any
+ // other key can compact it. The old fixture omitted this owner.
+ proof, err := f.engine.bindProof(f.shared, map[string]Tip{"competitor": f.competitor}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if err := f.engine.emitSnapshot("el-0", nil, proof); err != nil {
+ t.Fatal(err)
+ }
+
+ f.engine.effectCache.releaseEvidence(evidenceTips(proof))
+
+ if compact {
+ f.compact(t)
+ }
+ // A strict batch exercises evidence discovery as well as value
+ // reconstruction. Returning the competitor's value is also wrong.
+ got, err := f.engine.GetSnapshots(context.Background(), []string{"el-0", "el-4"})
+ if err != nil {
+ t.Fatalf("reconstruct retained cross-key history: %v", err)
+ }
+
+ if len(got) != 2 {
+ t.Fatalf("returned %d values, want 2", len(got))
+ }
+
+ for i, value := range got {
+ if string(value.GetScalar().GetRaw()) != "100" {
+ t.Errorf("key %d reconstructed %v, want shared transaction value 100", i, value)
+ }
+
+ if compact && (len(baseline) != len(got) || !proto.Equal(value, baseline[i])) {
+ t.Errorf("key %d changed after reclamation: got %v, unreclaimed %v", i, value, baseline)
+ }
+ }
+
+ if !compact {
+ baseline = got
+ }
+ })
+ }
+ }
+}
+
+func TestCrossKeyClosureEquivalentDiscovery(t *testing.T) {
+ transport := errors.New("peer unavailable")
+ for _, tc := range []struct {
+ name string
+ subscribed bool
+ failure error
+ }{
+ {name: "available", subscribed: true},
+ {name: "unsubscribed", failure: transport},
+ {name: "peer-absent", subscribed: true, failure: ErrEffectNotFound},
+ {name: "cloud-absent", subscribed: true, failure: ErrCDNBlobMissing},
+ {name: "transport-failure", subscribed: true, failure: transport},
+ } {
+ for _, stream := range []bool{false, true} {
+ t.Run(fmt.Sprintf("%s/stream=%t", tc.name, stream), func(t *testing.T) {
+ source := newCrossKeyHistory(t, false)
+ pool := source.engine.effectCache
+ noop := Tip{9, 1}
+ putCutEffects(t, source.engine, map[Tip]*pb.Effect{
+ noop: {Key: []byte("el-0"), Deps: toPbRefs([]Tip{source.shared}), Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}}},
+ })
+
+ var direct []Tip
+
+ for _, ancestor := range []bool{false, true} {
+ e := newTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "el-0")
+
+ if tc.subscribed {
+ preSubscribe(e, "el-4")
+ }
+
+ load := func(tip Tip) ([]byte, error) {
+ if tip == source.data4 {
+ if !tc.subscribed {
+ t.Fatal("fetched the unsubscribed branch")
+ }
+
+ if tc.failure != nil {
+ return nil, tc.failure
+ }
+ }
+
+ eff, ok := pool.Get(tip)
+ if !ok {
+ t.Fatalf("unexpected fetch %v", tip)
+ }
+
+ return wireEffect(eff), nil
+ }
+
+ e.broadcaster = &cutBroadcaster{load: load}
+ if stream {
+ e.broadcaster = &ancestryBroadcaster{load: func(key string, root Tip, _ []pb.NodeID, have func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ var err error
+
+ pool.VisitAncestry(key, root, func(tip Tip, _ *pb.Effect, isRoot bool) bool {
+ if have(tip) {
+ return true
+ }
+
+ var data []byte
+
+ data, err = load(tip)
+ if err != nil && !errors.Is(err, ErrEffectNotFound) && !errors.Is(err, ErrCDNBlobMissing) {
+ return false
+ }
+
+ err = receive(tip, data, isRoot)
+
+ return err == nil
+ })
+
+ return err
+ }}
+ }
+
+ incoming := source.shared
+ if ancestor {
+ incoming = noop
+ }
+
+ eff, _ := pool.Get(incoming)
+ if err := e.putIngested(incoming, eff, proto.Size(eff)); err != nil {
+ t.Fatal(err)
+ }
+
+ _, complete, err := e.closeIngestedFrontier("el-0", []Tip{incoming}, make(map[Tip]error))
+
+ wantFailure := tc.subscribed && tc.failure == transport
+ if complete == wantFailure || errors.Is(err, transport) != wantFailure || (!wantFailure && err != nil) {
+ t.Fatalf("ancestor=%t: complete=%t error=%v, want transport failure=%t", ancestor, complete, err, wantFailure)
+ }
+
+ var available []Tip
+
+ for _, tip := range []Tip{source.shared, source.data0, source.data4} {
+ _, resident := e.effectCache.Get(tip)
+
+ want := tip != source.data4 || tc.subscribed && tc.failure == nil
+ if resident != want {
+ t.Errorf("ancestor=%t: vertex %v resident=%t, want %t", ancestor, tip, resident, want)
+ }
+
+ if resident {
+ available = append(available, tip)
+ }
+ }
+
+ if ancestor && !slices.Equal(available, direct) {
+ t.Errorf("ancestry discovery closure %v differs from direct receipt %v", available, direct)
+ }
+
+ direct = available
+ }
+ })
+ }
+ }
+}
+
+// Folding a bind into another key's state does not prove that it won. The
+// shared bind beats competitor on el-0, but loses to winner on el-4; therefore
+// competitor must survive on el-0. No cached verdicts supply this evidence.
+func TestHistoricalBindRetainsOtherKeyLoss(t *testing.T) {
+ f := newCrossKeyHistory(t, false)
+ data, bind, snapshot, latest := Tip{2, 1}, Tip{2, 2}, Tip{2, 3}, Tip{2, 4}
+ state := &pb.ReducedEffect{Scalar: scalarInsertRaw([]byte("other"))}
+ putCutEffects(t, f.engine, map[Tip]*pb.Effect{
+ data: {Key: []byte("el-4"), TxnId: "winner", Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("other"))}},
+ bind: {
+ Key: []byte("el-4"), TxnId: "winner", ForkChoiceHash: []byte{0}, Deps: toPbRefs([]Tip{data}),
+ Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{
+ Keys: []*pb.TransactionalBindEffect_KeyBind{{Key: []byte("el-4"), NewTip: toPbRef(data)}},
+ }},
+ },
+ snapshot: {Key: []byte("el-4"), Deps: toPbRefs([]Tip{f.latest, bind}), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: state}}},
+ latest: {Key: []byte("el-4"), Deps: toPbRefs([]Tip{snapshot}), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: state}}},
+ })
+ f.engine.installIngestedTips("el-4", latest)
+
+ got, err := f.engine.GetSnapshots(context.Background(), []string{"el-0", "el-4"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for i, want := range []string{"wrong", "other"} {
+ if value := string(got[i].GetScalar().GetRaw()); value != want {
+ t.Errorf("key %d reconstructed %q, want %q", i, value, want)
+ }
+ }
+}
+
+// Ingestion must close each subscribed key's branch of a shared bind without
+// merging their value DAGs. Exercise both a directly received bind and a bind
+// discovered in ancestry, with individual fetches and streamed replication.
+func TestBindIngestionClosesAllSubscribedKeys(t *testing.T) {
+ for _, ancestor := range []bool{false, true} {
+ for _, stream := range []bool{false, true} {
+ for _, remote := range []bool{false, true} {
+ t.Run(fmt.Sprintf("ancestor=%t/stream=%t/remote=%t", ancestor, stream, remote), func(t *testing.T) {
+ log := newSnapshotLog()
+ base0 := log.putEffect(&pb.Effect{Key: []byte("el-0"), Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("old0"))}})
+ base4 := log.putEffect(&pb.Effect{Key: []byte("el-4"), Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("old4"))}})
+
+ keys := make([]*pb.TransactionalBindEffect_KeyBind, 0, 2)
+ deps := make([]Tip, 0, 2)
+
+ for i, key := range []string{"el-0", "el-4"} {
+ consumed := toPbRefs([]Tip{[]Tip{base0, base4}[i]})
+ data := log.putEffect(&pb.Effect{
+ Key: []byte(key), TxnId: "shared", Deps: consumed,
+ Kind: &pb.Effect_Data{Data: scalarInsertRaw([]byte("100"))},
+ })
+ deps = append(deps, data)
+ keys = append(keys, &pb.TransactionalBindEffect_KeyBind{
+ Key: []byte(key), NewTip: toPbRef(data), ConsumedTips: consumed,
+ })
+ }
+
+ bindTip := log.putEffect(&pb.Effect{
+ Key: []byte("el-0"), TxnId: "shared", Deps: toPbRefs(deps),
+ Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{Keys: keys}},
+ })
+ bind, _ := log.effectCache.Get(bindTip)
+
+ incomingTip, incoming := bindTip, bind
+ if ancestor {
+ incomingTip = log.putEffect(&pb.Effect{
+ Key: []byte("el-0"), Deps: toPbRefs([]Tip{bindTip}),
+ Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}},
+ })
+ incoming, _ = log.effectCache.Get(incomingTip)
+ }
+
+ loads := make(map[Tip]int)
+
+ var b Broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ loads[tip]++
+
+ eff, ok := log.effectCache.Get(tip)
+ if !ok {
+ t.Fatalf("unexpected dependency fetch %v", tip)
+ }
+
+ return wireEffect(eff), nil
+ }}
+ if stream {
+ b = &ancestryBroadcaster{load: func(key string, root Tip, _ []pb.NodeID, have func(Tip) bool, receive func(Tip, []byte, bool) error) error {
+ var err error
+
+ log.effectCache.VisitAncestry(key, root, func(tip Tip, eff *pb.Effect, isRoot bool) bool {
+ if have(tip) {
+ return true
+ }
+
+ if eff == nil {
+ t.Fatalf("unexpected missing source vertex %v", tip)
+ }
+
+ loads[tip]++
+ err = receive(tip, wireEffect(eff), isRoot)
+
+ return err == nil
+ })
+
+ return err
+ }}
+ }
+
+ e := newTestEngine(b)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "el-0", "el-4")
+
+ notify := &pb.OffsetNotify{Origin: toPbRef(incomingTip), EffectData: wireEffect(incoming)}
+ if remote {
+ if _, err := e.HandleRemote(notify); err != nil {
+ t.Fatal(err)
+ }
+ } else if err := e.handleBackfill(notify); err != nil {
+ t.Fatal(err)
+ }
+
+ for i, key := range []string{"el-0", "el-4"} {
+ if !ancestor || i == 0 {
+ requireCutTips(t, e.index.Contains(key).Tips(), []Tip{incomingTip})
+ }
+
+ data := r(bind.GetTxnBind().GetKeys()[i].GetNewTip())
+
+ base := []Tip{base0, base4}[i]
+ for _, tip := range []Tip{data, base} {
+ if _, ok := e.effectCache.Get(tip); !ok || loads[tip] != 1 {
+ t.Errorf("%s dependency %v resident=%t fetches=%d, want true/1", key, tip, ok, loads[tip])
+ }
+ }
+
+ d := newDag(e, key, "")
+ if err := d.prepare([]Tip{bindTip}); err != nil {
+ t.Fatal(err)
+ }
+
+ other := r(bind.GetTxnBind().GetKeys()[1-i].GetNewTip())
+ if d.nodes[other] != nil {
+ t.Errorf("%s DAG includes the other key's data %v", key, other)
+ }
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/effects/dag.go b/effects/dag.go
index 84304e5..78d083f 100644
--- a/effects/dag.go
+++ b/effects/dag.go
@@ -100,7 +100,7 @@ func normalizeTipCandidates(key string, candidates, roots []Tip, load func(Tip)
supersededSet := make(map[Tip]struct{}, len(unique))
- nodes, _, err := walkKeyHistory(key, slices.Concat(unique, roots), load)
+ nodes, err := walkKeyHistory(key, slices.Concat(unique, roots), load)
for cur, eff := range nodes {
for ref := range keyDeps(key, eff) {
dep := r(ref)
@@ -128,14 +128,17 @@ type dag struct {
nodes map[Tip]*pb.Effect
lcaTip Tip
// topoOrder is the post-order DFS sequence of every tip reached
- // during phase 2, including tx-data effects that iterate filters
+ // during preparation, including tx-data effects that iterate filters
// out of the processor stream. Callers that need to propagate
// per-tip state along dependency edges (e.g. HorizonSet
// invisibility propagation in reconstruct) read this between
// prepare and iterate.
topoOrder []Tip
+ compact bool // only a live frontier may release folded history
}
+var errEffectNotResident = errors.New("required effect is not resident")
+
// dagTip carries the effect while sorting, avoiding map lookups per comparison.
type dagTip struct {
tip Tip
@@ -150,39 +153,15 @@ func newDag(engine *Engine, key string, activeTxnID string) *dag {
return &dag{engine: engine, key: key, activeTxnID: activeTxnID}
}
-// walk performs a two-phase traversal of the effect graph:
-//
-// 1. BFS from tips following deps, tracking path convergence. Stops at the
-// LCA snapshot (first snapshot where all paths have converged). Collects
-// all nodes between tips and the LCA.
-// 2. Topological replay (DFS post-order) from tips within the collected set,
-// ordered by fork choice hash at each fork. Calls processor in causal
-// order (LCA snapshot → tips).
-//
-// Transactional effects (non-empty TxnId, not a bind) are skipped from the
-// processor output but their deps are still followed. Bind effects appear
-// in the sequence as envelopes.
-//
-// walk is a convenience wrapper over prepare + iterate; callers that need
-// to inspect d.topoOrder between BFS and the processor loop should call
-// prepare and iterate explicitly.
-func (d *dag) walk(tips []Tip, processor func(*pb.Effect) error) error {
- if err := d.prepare(tips); err != nil {
- return err
- }
-
- return d.iterate(func(_ Tip, eff *pb.Effect) error {
- return processor(eff)
- })
-}
-
-// prepare runs BFS + LCA trim and builds d.topoOrder. Safe to read
-// d.nodes and d.topoOrder after this returns successfully.
+// prepare builds the causal order directly from resident vertices. Safe to read
+// d.nodes and d.topoOrder after this returns successfully. Ingestion owns fetches;
+// iterate applies transaction and snapshot filtering to the resulting order.
func (d *dag) prepare(tips []Tip) error {
d.lcaTip = Tip{}
d.topoOrder = nil
+ d.nodes = make(map[Tip]*pb.Effect)
- if err := d.bfs(tips, &d.lcaTip); err != nil {
+ if _, err := d.topoCollect(tips, true); err != nil {
return err
}
@@ -190,37 +169,68 @@ func (d *dag) prepare(tips []Tip) error {
return nil
}
- // Phase 1.5: trim nodes already folded into the LCA snapshot. BFS may
- // have walked into the snapshot's dep chain (because the snapshot was
- // dequeued with a non-empty queue, so it wasn't terminal at that point).
- // Those nodes are already represented in the snapshot's reduced state;
- // leaving them in d.nodes makes ReduceChain process the same data
- // effects twice — duplicates in list reads, double-counts in scalars.
- d.trimAncestorsOfLCA()
+ // Snapshot relationships were established from resident edges. Remove the
+ // folded ancestors from the output, retaining compaction's ownership checks.
+ if err := d.trimAncestorsOfLCA(tips); err != nil {
+ return err
+ }
+
+ order := d.topoOrder[:0]
+ for _, tip := range d.topoOrder {
+ eff, selected := d.nodes[tip]
+ if !selected {
+ continue
+ }
+
+ if eff == nil {
+ return fmt.Errorf("reconstruct %q: effect %v: %w", d.key, tip, errEffectNotResident)
+ }
+
+ order = append(order, tip)
+ }
+
+ d.topoOrder = order
if slog.Default().Enabled(context.Background(), slog.LevelDebug) {
slog.Debug("dag.walk", "key", d.key, "nodes", len(d.nodes),
"lca", d.lcaTip, "encoded", d.encode(tips))
}
- // Phase 2: topo-order via DFS post-order within collected nodes.
- d.topoOrder = make([]Tip, 0, len(d.nodes))
- topoVisited := make(map[Tip]bool, len(d.nodes))
+ d.recordHistoryProcessed()
- sortedTips := make([]dagTip, 0, len(tips))
- for _, tip := range tips {
- if eff, ok := d.nodes[tip]; ok {
- sortedTips = append(sortedTips, dagTip{tip, eff})
- }
+ return nil
+}
+
+// ingest is used by recovery entry points, not by the resident graph traversal.
+// Preserve the fetch outcome when a definitively absent dependency is still
+// required for reconstruction; absence below the selected LCA is harmless.
+func (d *dag) ingest(tips []Tip) error {
+ attempted := make(map[Tip]error)
+ if _, _, err := d.engine.closeIngestedFrontier(d.key, tips, attempted); err != nil {
+ return err
}
- slices.SortFunc(sortedTips, compareDagTips)
+ err := d.prepare(tips)
+ if !errors.Is(err, errEffectNotResident) {
+ return err
+ }
- for _, entry := range sortedTips {
- d.topoCollect(entry.tip, topoVisited)
+ var missing []error
+
+ for tip, eff := range d.nodes {
+ if eff != nil {
+ continue
+ }
+
+ failure := attempted[tip]
+ if !errors.Is(failure, ErrCDNBlobMissing) {
+ return errors.Join(err, failure)
+ }
+
+ missing = append(missing, failure)
}
- return nil
+ return errors.Join(append(missing, err)...)
}
// iterate calls processor for each emitted tip in d.topoOrder, filtering
@@ -254,26 +264,6 @@ func (d *dag) shouldEmit(t Tip, eff *pb.Effect) bool {
return true
}
-// bfs explores the graph from tips following deps. The dependency frontier
-// is separate from the queue: visiting a root finishes work, but does not
-// make that path converge with another path. A snapshot can terminate the
-// walk only when it is the sole remaining frontier entry.
-//
-// Only state-carrying snapshots terminate the walk. Verdict-only snapshots
-// (State == nil) are opaque markers for fork-choice adjudication; their
-// chain content is reachable only via their Deps, so we follow them like
-// any other effect. Treating them as LCAs would leave reconstruct with a
-// single dead-end tip and no way to recover the underlying chain.
-func (d *dag) bfs(tips []Tip, lcaTip *Tip) error {
- nodes, lca, err := walkKeyHistory(d.key, tips, func(tip Tip) (*pb.Effect, error) {
- return d.engine.getEffect(d.key, tip)
- })
- d.nodes = nodes
- *lcaTip = lca
-
- return err
-}
-
// An empty subscription records no observed data. Referencing it records an
// observation of metadata, not another data branch. Subscription membership is
// maintained separately; keep the effect for causal bookkeeping only.
@@ -324,18 +314,17 @@ func visitDataDeps(key string, eff *pb.Effect, visit func(Tip) *pb.Effect) bool
return hasDataDeps
}
-// walkKeyHistory is the shared dependency walk for replay and tip normalization.
+// walkKeyHistory loads history for cloud tip normalization.
// Missing nodes remain on the dependency frontier, so an exhausted queue cannot
// disguise an unresolved branch. Non-missing load errors take precedence over
// confirmed missing blobs: joining both would let errors.Is classify a transport
// failure as a recoverable hole. State snapshots stop traversal only at
// convergence; their folded history need not remain available. Callers decide
// how to use partial nodes when errors occur.
-func walkKeyHistory(key string, tips []Tip, load func(Tip) (*pb.Effect, error)) (map[Tip]*pb.Effect, Tip, error) {
+func walkKeyHistory(key string, tips []Tip, load func(Tip) (*pb.Effect, error)) (map[Tip]*pb.Effect, error) {
nodes := make(map[Tip]*pb.Effect, len(tips)*8)
var (
- lcaTip Tip
loadErrors []error
missingErrors []error
)
@@ -425,9 +414,7 @@ func walkKeyHistory(key string, tips []Tip, load func(Tip) (*pb.Effect, error))
eff := nodes[cur]
if snap := eff.GetSnapshot(); snap != nil && snap.GetState() != nil && len(frontier) == 1 {
- lcaTip = cur
-
- return nodes, lcaTip, walkError()
+ return nodes, walkError()
}
// A data effect with only subscription observations is a data root,
@@ -438,114 +425,255 @@ func walkKeyHistory(key string, tips []Tip, load func(Tip) (*pb.Effect, error))
}
}
- // A root snapshot may have been processed before a longer path reached
- // it. Its frontier entry survives even though its queue entry is gone.
- if len(frontier) == 1 {
- for t := range frontier {
- if snap := nodes[t].GetSnapshot(); snap != nil && snap.GetState() != nil {
- lcaTip = t
- }
- }
+ return nodes, walkError()
+}
+
+// trimAncestorsOfLCA removes folded history from reconstruction. Its existing
+// resident history walk also selects reclamation's boundary: keep the live LCA
+// and the preceding COMMON snapshot, moving further back for subscriber
+// witnesses or transaction overlays. A snapshot on one branch is not a cut.
+func (d *dag) trimAncestorsOfLCA(tips []Tip) error {
+ if d.lcaTip == (Tip{}) || d.nodes[d.lcaTip].GetSnapshot().GetState() == nil {
+ return nil
}
- return nodes, lcaTip, walkError()
-}
+ var boundary *historyBoundary
+
+ if d.compact && !isSystemKey([]byte(d.key)) {
+ d.engine.historyMu.Lock()
+ defer d.engine.historyMu.Unlock()
-// trimAncestorsOfLCA removes from d.nodes every node in the transitive
-// dep-ancestry of the LCA snapshot. Those nodes' state is already folded
-// into the snapshot's reduced effect; including them in ReduceChain would
-// double-count their data effects (the symptom is duplicate elements in
-// a list-append read).
-//
-// Only nodes already in d.nodes are removed — we don't fetch additional
-// effects from the engine. The bfs that populated d.nodes is the only
-// source of folded-ancestor entries we need to clear.
-func (d *dag) trimAncestorsOfLCA() {
- var zero Tip
- if d.lcaTip == zero {
- return
+ if tipsEqual(d.engine.index.Contains(d.key).Tips(), tips) {
+ boundary = d.engine.historyBoundary(d.key)
+ }
}
- lcaEff, ok := d.nodes[d.lcaTip]
- if !ok || lcaEff.GetSnapshot() == nil {
- return
+ // Use the same snapshot-chain intersection as reconstruction. Without a
+ // reclamation boundary, only trim the nodes reconstruction already visited.
+ history := &dag{key: d.key, nodes: d.nodes}
+ if boundary != nil {
+ history.engine = d.engine
}
- stack := make([]Tip, 0, len(lcaEff.GetDeps()))
- for _, dep := range lcaEff.GetDeps() {
- stack = append(stack, r(dep))
+ paths, err := history.topoCollect([]Tip{d.lcaTip}, false)
+ if err != nil {
+ return err
+ }
+
+ var cut Tip
+ if boundary != nil {
+ cut = paths[d.lcaTip].parent
+
+ known := make([]Tip, len(boundary.peers))
+ for _, tip := range history.topoOrder {
+ for i, peer := range boundary.peers {
+ if (tip[0] == peer.Node && tip[1] <= peer.Minted) ||
+ (tip[0] == uint64(d.engine.nodeID) && tip[1] == peer.Processed) {
+ // A witness proves only its own ancestry. Intersect it with the
+ // common snapshot chain before it can authorize reclamation.
+ safe := paths.intersect(cut, paths[tip].root)
+ if safe != (Tip{}) && (known[i] == (Tip{}) || paths[safe].order > paths[known[i]].order) {
+ known[i] = safe
+ }
+ }
+ }
+ }
+
+ for _, witness := range known {
+ cut = paths.intersect(cut, witness)
+ }
+
+ for cut != (Tip{}) && !boundary.beforeOverlays(cut) {
+ cut = paths[cut].parent
+ }
}
- for len(stack) > 0 {
- cur := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
+ release := false
- eff, ok := d.nodes[cur]
- if !ok {
- continue
+ if cut != (Tip{}) {
+ if v := d.engine.effectCache.getVertex(cut); v != nil {
+ v.markRoot()
+
+ release = true
+ }
+ }
+ // A common snapshot dominates every data path. Its data ancestors are
+ // exactly the post-order prefix before it; unrelated metadata observations
+ // are not reclamation evidence and remain owned by subscription machinery.
+ for _, tip := range history.topoOrder {
+ if tip == cut {
+ release = false
}
- // Removing a node also marks it visited, so shared ancestors need no
- // separate set and are expanded only once.
- delete(d.nodes, cur)
+ if release && d.nodes[tip].GetSubscription() == nil {
+ d.engine.effectCache.decrefChainNode(d.key, tip)
+ }
- for dep := range keyDeps(d.key, eff) {
- stack = append(stack, r(dep))
+ if tip != d.lcaTip {
+ delete(d.nodes, tip)
}
}
+
+ return nil
+}
+
+// dagPaths is temporary traversal state shared by reconstruction and
+// reclamation. parent links state snapshots on the common-ancestor chain.
+type dagPath struct {
+ root, parent Tip
+ order int
+ data bool
}
-// topoCollect does a post-order DFS within the collected nodes, recording
-// every visited tip into d.topoOrder. At forks, deps are visited in fork
-// choice hash order. Filtering (tx-data effects, non-LCA snapshots) is
-// applied by iterate, not here — d.topoOrder is the complete causal order
-// of visited tips so callers can propagate per-tip state along dep edges.
-func (d *dag) topoCollect(t Tip, visited map[Tip]bool) {
- if visited[t] {
- return
+type dagPaths map[Tip]dagPath
+
+func (paths dagPaths) intersect(a, b Tip) Tip {
+ for a != b && a != (Tip{}) && b != (Tip{}) {
+ if paths[a].order > paths[b].order {
+ a = paths[a].parent
+ } else {
+ b = paths[b].parent
+ }
}
- visited[t] = true
+ if a == b {
+ return a
+ }
- eff, ok := d.nodes[t]
- if !ok {
- return
+ return Tip{}
+}
+
+// topoCollect visits resident vertices once in fork-choice DFS post-order.
+// For each vertex, root is the nearest state snapshot crossed by EVERY data
+// path from it. A snapshot's parent is the next such snapshot in its ancestry.
+// Intersecting those chains at forks establishes the LCA without depending on
+// discovery order. The order numbers are local DFS positions, not clocks.
+func (d *dag) topoCollect(tips []Tip, stopAtSnapshot bool) (dagPaths, error) {
+ paths := make(dagPaths)
+ resident := func(tip Tip) *pb.Effect {
+ if eff, seen := d.nodes[tip]; seen {
+ return eff
+ }
+
+ var eff *pb.Effect
+ if d.engine != nil && d.engine.effectCache != nil {
+ eff, _ = d.engine.effectCache.Get(tip)
+ }
+
+ d.nodes[tip] = eff
+
+ return eff
}
- // LCA snapshot: record and don't descend past it
- if eff.GetSnapshot() != nil && t == d.lcaTip {
- d.topoOrder = append(d.topoOrder, t)
+ var visit func(Tip, bool) (dagPath, error)
+
+ visit = func(tip Tip, dominates bool) (dagPath, error) {
+ if p, seen := paths[tip]; seen {
+ if p.order < 0 {
+ return p, fmt.Errorf("reconstruct %q: dependency cycle at %v", d.key, tip)
+ }
+
+ return p, nil
+ }
+
+ paths[tip] = dagPath{order: -1}
+ eff := resident(tip)
+ // Along a sole data path a state snapshot already dominates the
+ // requested frontier. There are no sibling paths to reconcile.
+ if stopAtSnapshot && dominates && eff.GetSnapshot().GetState() != nil {
+ p := dagPath{root: tip, order: len(d.topoOrder), data: true}
+ paths[tip] = p
+ d.topoOrder = append(d.topoOrder, tip)
+
+ return p, nil
+ }
+
+ var local [8]dagTip
- return
+ deps := local[:0]
+ dataDeps := 0
+
+ for ref := range keyDeps(d.key, eff) {
+ dep := r(ref)
+ value := resident(dep)
+
+ deps = append(deps, dagTip{dep, value})
+ if !emptySubscription(value) {
+ dataDeps++
+ }
+ }
+
+ slices.SortFunc(deps, compareDagTips)
+
+ p := dagPath{}
+
+ for _, dep := range deps {
+ ancestor, err := visit(dep.tip, dominates && dataDeps == 1 && !emptySubscription(dep.eff))
+ if err != nil {
+ return p, err
+ }
+
+ if !ancestor.data {
+ continue
+ }
+
+ if p.data {
+ p.root = paths.intersect(p.root, ancestor.root)
+ } else {
+ p.root = ancestor.root
+ }
+
+ p.data = true
+ }
+ // Metadata-only observations do not introduce a data root. Missing
+ // payloads do: they cannot be assumed to contain only metadata.
+ p.data = p.data || eff.GetSubscription() == nil
+ if eff.GetSnapshot().GetState() != nil {
+ p.parent, p.root = p.root, tip
+ }
+
+ p.order = len(d.topoOrder)
+ paths[tip] = p
+ d.topoOrder = append(d.topoOrder, tip)
+
+ return p, nil
}
+ sorted := make([]dagTip, 0, len(tips))
+ dataTips := 0
- // Ordinary chains need neither an iterator nor a sorting buffer.
- if eff.GetTxnBind() == nil && len(eff.GetDeps()) == 1 {
- d.topoCollect(r(eff.GetDeps()[0]), visited)
- d.topoOrder = append(d.topoOrder, t)
+ for _, tip := range tips {
+ eff := resident(tip)
- return
+ sorted = append(sorted, dagTip{tip, eff})
+ if !emptySubscription(eff) {
+ dataTips++
+ }
}
- // Typical forks fit on the stack; unusually wide forks can grow the slice.
- var local [8]dagTip
+ slices.SortFunc(sorted, compareDagTips)
- deps := local[:0]
+ hasData := false
- for ref := range keyDeps(d.key, eff) {
- tip := r(ref)
- if dep, ok := d.nodes[tip]; ok {
- deps = append(deps, dagTip{tip, dep})
+ for _, entry := range sorted {
+ p, err := visit(entry.tip, dataTips == 1 && !emptySubscription(entry.eff))
+ if err != nil {
+ return nil, err
}
- }
- slices.SortFunc(deps, compareDagTips)
+ if !p.data {
+ continue
+ }
+
+ if hasData {
+ d.lcaTip = paths.intersect(d.lcaTip, p.root)
+ } else {
+ d.lcaTip = p.root
+ }
- for _, dep := range deps {
- d.topoCollect(dep.tip, visited)
+ hasData = true
}
- d.topoOrder = append(d.topoOrder, t)
+ return paths, nil
}
// encode produces a compact string encoding of the collected dag nodes.
diff --git a/effects/dag_load_errors_test.go b/effects/dag_load_errors_test.go
new file mode 100644
index 0000000..429d555
--- /dev/null
+++ b/effects/dag_load_errors_test.go
@@ -0,0 +1,93 @@
+package effects
+
+import (
+ "errors"
+ "fmt"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+func TestWalkKeyHistoryTransportErrorPrecedesMissing(t *testing.T) {
+ transportErr := errors.New("dependency fetch timed out")
+
+ for _, dependencies := range []bool{false, true} {
+ for _, missingFirst := range []bool{false, true} {
+ t.Run(fmt.Sprintf("dependencies=%t/missing-first=%t", dependencies, missingFirst), func(t *testing.T) {
+ tips := []Tip{{7, 1}, {7, 2}}
+ if !missingFirst {
+ tips[0], tips[1] = tips[1], tips[0]
+ }
+
+ root := &pb.Effect{Deps: toPbRefs(tips)}
+ if dependencies {
+ tips = []Tip{{7, 3}}
+ }
+
+ _, err := walkKeyHistory("k", tips, func(tip Tip) (*pb.Effect, error) {
+ switch tip {
+ case (Tip{7, 1}):
+ return nil, fmt.Errorf("fetch %v: %w", tip, ErrCDNBlobMissing)
+ case (Tip{7, 2}):
+ return nil, fmt.Errorf("fetch %v: %w", tip, transportErr)
+ default:
+ return root, nil
+ }
+ })
+ if !errors.Is(err, transportErr) || errors.Is(err, ErrCDNBlobMissing) {
+ t.Fatalf("mixed load failures must classify as transport failure: %v", err)
+ }
+ })
+ }
+ }
+}
+
+type mixedFailureBroadcaster struct {
+ mockBroadcaster
+
+ transportErr error
+}
+
+func (b *mixedFailureBroadcaster) FetchFromAny(ref *pb.EffectRef, _ FetchHint) ([]byte, error) {
+ switch r(ref) {
+ case (Tip{7, 4}):
+ return wireEffect(frontierEffect(4, 2, 3)), nil
+ case (Tip{7, 2}):
+ return nil, ErrCDNBlobMissing
+ default:
+ return nil, b.transportErr
+ }
+}
+
+func TestCloudReadMixedWalkFailuresDoesNotServeSubset(t *testing.T) {
+ transportErr := errors.New("dependency fetch connection reset")
+ bc := &mixedFailureBroadcaster{transportErr: transportErr}
+ bc.allRegionPeersReachable = true
+ bc.peerIDs = []pb.NodeID{7}
+
+ e := newTestEngine(bc)
+ defer func() { _ = e.Close() }()
+
+ readable, mixed := Tip{7, 1}, Tip{7, 5}
+ fake := &fakeCloudReader{
+ tips: map[string][]Tip{"k": {readable, mixed}},
+ sidecar: map[string][]CloudEffect{"k": {
+ cloudEffect(t, readable, wireEffect(frontierEffect(1))),
+ cloudEffect(t, mixed, wireEffect(frontierEffect(5, 4))),
+ }},
+ // Cloud lacks this intermediate effect, but a peer still has it.
+ // Resolving it during DAG validation exposes both a hole and an
+ // unavailable dependency beneath the same frontier tip.
+ missing: map[string][]Tip{"k": {{7, 4}}},
+ }
+ e.SetCloudReader(fake)
+
+ result, _, err := e.NewReadOnlyContext().GetSnapshot("k")
+ if !errors.Is(err, ErrCloudUnavailable) || !errors.Is(err, transportErr) {
+ t.Fatalf("read must preserve transport failure, got result=%v err=%v", result, err)
+ }
+
+ if len(fake.pending) != 0 {
+ t.Fatalf("transport failure was classified as a pending hole: %v", fake.pending)
+ }
+}
diff --git a/effects/dag_resident_test.go b/effects/dag_resident_test.go
new file mode 100644
index 0000000..0169280
--- /dev/null
+++ b/effects/dag_resident_test.go
@@ -0,0 +1,85 @@
+package effects
+
+import (
+ "errors"
+ "fmt"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+// A short branch reaches S before a longer branch's nested snapshots:
+// left -> S; right -> S2 -> B -> S1 -> A -> S. The entire data frontier
+// crosses S, irrespective of tip order or fork choice. Its ancestry may be gone.
+func TestResidentReconstructionNestedSnapshots(t *testing.T) {
+ for _, reverse := range []bool{false, true} {
+ for _, absent := range []string{"none", "folded", "required"} {
+ t.Run(fmt.Sprintf("reverse=%t/absent=%s", reverse, absent), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ old, s, a, s1, b, s2, left, right := Tip{7, 1}, Tip{7, 2}, Tip{7, 3}, Tip{7, 4}, Tip{7, 5}, Tip{7, 6}, Tip{7, 7}, Tip{7, 8}
+ snapshot := func(tip, dep Tip, value int64) *pb.Effect {
+ eff := cutEffect(tip, dep)
+ eff.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{
+ Op: pb.EffectOp_INSERT_OP, Merge: pb.MergeRule_ADDITIVE_INT,
+ Collection: pb.CollectionKind_SCALAR, Commutative: true,
+ Scalar: scalarInsertInt(pb.MergeRule_ADDITIVE_INT, value),
+ }}}
+
+ return eff
+ }
+ nodes := map[Tip]*pb.Effect{
+ old: cutEffect(old), s: snapshot(s, old, 10),
+ a: cutEffect(a, s), s1: snapshot(s1, a, 11),
+ b: cutEffect(b, s1), s2: snapshot(s2, b, 12),
+ left: cutEffect(left, s), right: cutEffect(right, s2),
+ }
+ nodes[old].GetData().Value = &pb.DataEffect_IntVal{IntVal: 10}
+
+ tips := []Tip{left, right}
+ if reverse {
+ tips[0], tips[1] = tips[1], tips[0]
+ nodes[left].ForkChoiceHash, nodes[right].ForkChoiceHash = nodes[right].GetForkChoiceHash(), nodes[left].GetForkChoiceHash()
+ }
+
+ switch absent {
+ case "folded":
+ delete(nodes, old)
+ case "required":
+ delete(nodes, b)
+ }
+
+ putCutEffects(t, e, nodes)
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ t.Fatalf("resident preparation fetched %v", tip)
+
+ return nil, ErrEffectNotFound
+ }}
+ d := newDag(e, "k", "")
+
+ err := d.prepare(tips)
+ if absent == "required" {
+ if !errors.Is(err, errEffectNotResident) {
+ t.Fatalf("missing required vertex accepted: %v", err)
+ }
+
+ return
+ }
+
+ if err != nil || d.lcaTip != s {
+ t.Fatalf("LCA=%v error=%v, want %v", d.lcaTip, err, s)
+ }
+
+ if _, retained := d.nodes[old]; retained {
+ t.Fatal("reconstruction retained folded ancestry")
+ }
+
+ state, _, err := e.reconstruct("k", tips, "", false)
+ if err != nil || state.GetScalar().GetIntVal() != 14 {
+ t.Fatalf("reconstruction=%v error=%v, want 14", state, err)
+ }
+ })
+ }
+ }
+}
diff --git a/effects/dag_test.go b/effects/dag_test.go
index 7c08000..8fc64b4 100644
--- a/effects/dag_test.go
+++ b/effects/dag_test.go
@@ -970,8 +970,7 @@ func TestCompaction_SnapshotAppearsInDAG(t *testing.T) {
d := newDag(e, "k", "")
- d.nodes = make(map[Tip]*pb.Effect, 64)
- if err := d.bfs(tips.Tips(), &d.lcaTip); err != nil {
+ if err := d.prepare(tips.Tips()); err != nil {
t.Fatal(err)
}
@@ -1108,8 +1107,7 @@ func TestCompaction_BeaconPattern(t *testing.T) {
d := newDag(e, key, "")
- d.nodes = make(map[Tip]*pb.Effect, 256)
- if err := d.bfs(tips.Tips(), &d.lcaTip); err != nil {
+ if err := d.prepare(tips.Tips()); err != nil {
t.Fatal(err)
}
diff --git a/effects/effect.go b/effects/effect.go
index c694e92..c594500 100644
--- a/effects/effect.go
+++ b/effects/effect.go
@@ -43,6 +43,7 @@ import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"golang.org/x/sync/semaphore"
+ "golang.org/x/sync/singleflight"
"google.golang.org/protobuf/types/known/timestamppb"
)
@@ -107,9 +108,10 @@ type Engine struct {
// which carries the cached subdag (and, later, eviction metadata) for
// each key — one structure, the trie, owns both the tips and the
// derived read cache.
- index *keytrie.Critbit[leafState]
- broadcaster Broadcaster // nil for standalone
- cloudReader CloudReader // nil when no cloud is configured (standalone/dev)
+ index *keytrie.Critbit[leafState]
+ broadcaster Broadcaster // nil for standalone
+ cloudReader CloudReader // nil when no cloud is configured (standalone/dev)
+ ancestryLoads singleflight.Group // active per-key downloads; no completed-result cache
nodeID pb.NodeID
clock *crdt.HLC
@@ -118,6 +120,9 @@ type Engine struct {
// In-memory offset generator (no durable log)
nextOff atomic.Uint64
+ historyMu sync.Mutex
+ historyOverlays *historyOverlay
+
subscriptions *xsync.Map[string, *subscriptionState]
safety atomic.Pointer[safetyMap]
@@ -209,6 +214,10 @@ type Engine struct {
// Striped locks for per-key atomic operations
locks [4096]sync.Mutex
+ // Closure admission is separate from command locks, which callers may
+ // already hold. One active walk per shard bounds traversal allocations.
+ closureLocks [64]sync.Mutex
+
// FlushGeneration is incremented on each FlushIndex call.
// Used by WATCH to detect FLUSHALL/FLUSHDB between WATCH and EXEC.
flushGeneration atomic.Uint64
@@ -264,10 +273,12 @@ type Engine struct {
// subscriptionState tracks a subscription's readiness.
// The ready channel is closed once bootstrapping completes.
type subscriptionState struct {
- ownedKey bool
- ready chan struct{}
- incomplete atomic.Bool // true if bootstrap saw unreachable effects
- closeOnce sync.Once // guards close(ready)
+ bootstrapMu sync.Mutex // serializes setup and upgrade from an async announcement
+ announcement *pb.OffsetNotify // non-nil until a blocking caller starts discovery
+ ownedKey bool
+ ready chan struct{}
+ incomplete atomic.Bool // true if bootstrap saw unreachable effects
+ closeOnce sync.Once // guards close(ready)
}
// markReady unblocks any waiters on this subscription state. Safe to
@@ -575,15 +586,15 @@ const cloudTipsBackstop = 60 * time.Second
// indistinguishable from data loss.
var ErrCloudUnavailable = errors.New("cloud unavailable")
-// ErrCDNBlobMissing marks a CDN fetch that answered 404: the blob is provably
-// absent from cloud storage, as opposed to the cloud being unreachable. Cloud
-// tip markers are candidates, never an authoritative frontier — a node builds
-// its own truth by walking the DAG it can actually read — so a hole under
-// this sentinel is not this engine's to repair. classifyCloudTips reports the
-// holed tip as pending; the cluster layer's reconcile loop retries it, and
-// Cloud's own billing sweep is what eventually heals or prunes the marker.
+// ErrCDNBlobMissing marks a definitive CDN 404, not a transport failure.
+// Closure exhausts that boundary. Reconstruction separately determines whether
+// the absent history is needed to produce a value.
var ErrCDNBlobMissing = errors.New("cdn blob missing")
+// ErrEffectNotFound is a peer's definitive empty fetch response. An unreachable
+// peer must return a transport error instead; closure cannot exhaust that path.
+var ErrEffectNotFound = errors.New("effect not found on peer")
+
// NormalizeCloudTipCandidates reduces Cloud's marker candidates to the
// maximal tips of the effect DAG. Cloud's tips directory is an index: stale
// markers can remain after a newer effect has dep-referenced them, so callers
@@ -610,214 +621,60 @@ func NormalizeCloudTipCandidates(key string, candidates, localRoots []Tip, load
})
}
-// hydrateFromCloud is the tiered-storage rehydrate on a read-miss: it asks
-// Cloud for key's candidate tip frontier and installs whatever portion of it
-// is locally walkable (pulling blobs via FetchFromAny → CDN), so a subsequent
-// reconstruct finds that portion locally. installed reports whether the index
-// now holds anything for key (already-present tips count, so the caller
-// always re-tries reconstructLocal after a successful consult); consulted
-// reports whether Cloud's answer was fully resolved — no pending holes — so
-// GetSnapshot can mark the leaf authoritative and stop re-consulting for a key
-// whose state reduces to nil. A partially or wholly holed frontier is not an
-// error: the holed tips are reported to Cloud's reconcile loop via
-// MarkPending and retried there, on Cloud's own schedule, without blocking
-// this read.
-//
-// A transport error or timeout returns ErrCloudUnavailable — the read fails
-// rather than fabricating a miss for data Cloud may hold.
-func (e *Engine) hydrateFromCloud(key string) (installed, consulted bool, err error) {
+// hydrateFromCloud obtains Cloud's candidate tips and ingests their available
+// closure. Reconstruction remains responsible for deciding whether that graph
+// can produce a value. Transport failures surface as ErrCloudUnavailable.
+func (e *Engine) hydrateFromCloud(key string) (installed bool, err error) {
if e.cloudReader == nil {
- return false, false, nil
+ return false, nil
}
ctx, cancel := context.WithTimeout(context.Background(), cloudTipsBackstop)
defer cancel()
- tips, sidecar, missing, err := e.cloudReader.CloudTips(ctx, key)
+ tips, sidecar, _, err := e.cloudReader.CloudTips(ctx, key)
if err != nil {
slog.Warn("cloud read-miss backstop: get tips failed; failing read", "key", key, "error", err)
- return false, false, fmt.Errorf("%w: get tips for %q: %w", ErrCloudUnavailable, key, err)
+ return false, fmt.Errorf("%w: get tips for %q: %w", ErrCloudUnavailable, key, err)
}
if len(tips) == 0 {
- return false, true, nil // Cloud holds nothing either: an authoritative miss.
- }
-
- pending, err := e.InstallCloudTips(key, tips, sidecar, missing)
- if err != nil {
- return false, false, err
- }
-
- if len(pending) > 0 {
- e.cloudReader.MarkPending(key)
- slog.Warn("cloud read-miss backstop: frontier partially holed; serving readable subset, leaving the rest pending for reconcile",
- "key", key, "pending", pending)
- }
-
- return e.index.Contains(key) != nil, len(pending) == 0, nil
-}
-
-// InstallCloudTips classifies Cloud-provided marker candidates into the
-// subset that is locally walkable (install) and the subset still holed
-// (pending), merges the walkable subset into the index alongside whatever is
-// already there, and returns the holed subset for the caller to track. Besides
-// the read-miss rehydrate it serves the cloud-sync reconcile path: a read
-// served from the outbox during a Cloud outage re-installs the missed Cloud
-// frontier here once Cloud answers again.
-//
-// It never mints and never consumes local tips: Cloud's tips directory is a
-// candidate index, not authoritative state, so nothing here may supersede
-// local ancestry. A hole is not this engine's to repair — see ErrCDNBlobMissing.
-func (e *Engine) InstallCloudTips(key string, tips []Tip, sidecar []CloudEffect, missing []Tip) (pending []Tip, err error) {
- classification, err := e.classifyCloudTips(key, tips, sidecar, missing)
- if err != nil {
- return nil, err
+ return false, nil // Cloud holds nothing either: an authoritative miss.
}
- if len(classification.install) > 0 {
- slog.Debug("cloud tips: installing readable frontier", "key", key, "tips", classification.install)
- e.installTips(key, classification.install)
+ if err := e.InstallCloudTips(key, tips, sidecar); err != nil {
+ return false, err
}
- if len(classification.dropped) > 0 {
- slog.Debug("cloud tips: dropped candidate markers superseded by ancestry", "key", key, "tips", classification.dropped)
- }
-
- if len(classification.pending) > 0 {
- slog.Debug("cloud tips: tips left pending on a hole", "key", key, "tips", classification.pending)
- }
-
- return classification.pending, nil
-}
-
-type cloudTipClassification struct {
- install []Tip
- pending []Tip
- dropped []Tip
+ return e.index.Contains(key) != nil, nil
}
-// classifyCloudTips splits Cloud's candidate tips for key into three sets:
-// install (readable state tips plus dependency-less subscription metadata),
-// pending (tips whose ancestry hits ErrCDNBlobMissing somewhere in the walk —
-// a durable hole, not a transport failure), and dropped (candidates
-// NormalizeCloudTipCandidates proved non-maximal). A transport error or
-// timeout anywhere in the walk returns ErrCloudUnavailable instead of
-// classifying anything, since that failure mode is retryable and must not be
-// confused with a durable hole.
-//
-// missing is GetTips' own report of refs its closure walk found reachable but
-// whose blobs it does not hold — stated as fact by the storage side. Tips
-// named there classify pending without paying a redundant CDN round trip to
-// re-learn the same 404; anything the walk discovers missing on its own still
-// gets the network round trip, since Cloud's declared list is a hint, not a
-// complete inventory of every hole beneath the frontier.
-func (e *Engine) classifyCloudTips(key string, tips []Tip, sidecar []CloudEffect, missing []Tip) (cloudTipClassification, error) {
- // Warm the cache with the closure effects GetTips delivered inline, so the
- // walk below runs locally instead of one WAN fetch per dep (n+1). Same
- // owned-vs-cache routing as every other ingest path; stragglers the
- // sidecar didn't carry are pulled by the walk itself.
- if e.effectCache != nil {
- for _, ce := range sidecar {
- if _, ok := e.effectCache.Get(ce.Tip); ok {
- continue
- }
-
- if err := e.putIngested(ce.Tip, ce.Eff, ce.ProtoLen); err != nil {
- return cloudTipClassification{}, err
- }
- }
- }
-
- missingSet := make(map[Tip]struct{}, len(missing))
- for _, t := range missing {
- missingSet[t] = struct{}{}
- }
-
- load := func(tip Tip) (*pb.Effect, error) {
- if _, known := missingSet[tip]; known {
- return nil, ErrCDNBlobMissing
- }
-
- return e.getEffect(key, tip)
- }
-
- var localRoots []Tip
- if currentSet := e.index.Contains(key); currentSet != nil {
- localRoots = currentSet.Tips()
- }
-
- var classification cloudTipClassification
-
- realTips, superseded, err := NormalizeCloudTipCandidates(key, tips, localRoots, load)
- if err != nil {
- return classification, fmt.Errorf("normalize cloud candidates for %q: %w", key, err)
- }
-
- classification.dropped = superseded
-
- // Dependency-less subscriptions are frontier metadata, not state
- // branches: they install unconditionally, since they carry no ancestry
- // for a hole to hide in. State tips still need walk validation below.
- stateTips := make([]Tip, 0, len(realTips))
- for _, tip := range realTips {
- eff, getErr := load(tip)
- switch {
- case getErr == nil:
- if eff.GetSubscription() != nil && len(eff.GetDeps()) == 0 {
- classification.install = append(classification.install, tip)
+// InstallCloudTips supplies Cloud's candidates and inline effects to the same
+// closure walk as peer ingestion. It publishes graph relationships without
+// selecting snapshots, reconstructing values, or filtering unreadable branches.
+// Missing reports from GetTips describe only Cloud's storage; closure may still
+// obtain that history from peers through the existing fetch path.
+func (e *Engine) InstallCloudTips(key string, tips []Tip, sidecar []CloudEffect) error {
+ // Closure may discover a bind even when the supplied tip is ordinary data.
+ // Preserve its publication ordering with local commits, before the key lock.
+ nodeLock := e.takeNodeLock()
+ defer nodeLock.release()
- continue
- }
-
- stateTips = append(stateTips, tip)
- case errors.Is(getErr, ErrCDNBlobMissing):
- classification.pending = append(classification.pending, tip)
- default:
- return cloudTipClassification{}, fmt.Errorf("%w: cloud tip %v unreachable for %q: %w",
- ErrCloudUnavailable, tip, key, getErr)
+ for _, ce := range sidecar {
+ if err := e.putIngested(ce.Tip, ce.Eff, ce.ProtoLen); err != nil {
+ return err
}
}
- if len(stateTips) == 0 {
- return classification, nil
- }
-
- // Validate the joint frontier first. Each sibling branch can be
- // independently readable while their union descends beneath both into a
- // shared hole below their LCA, so per-tip validation alone is
- // insufficient to prove the whole frontier installable together.
- rd := newDag(e, key, "")
-
- walkErr := rd.walk(stateTips, func(*pb.Effect) error { return nil })
- switch {
- case walkErr == nil:
- classification.install = append(classification.install, stateTips...)
-
- return classification, nil
- case errors.Is(walkErr, ErrCDNBlobMissing):
- // Fall through: split per-tip below.
- default:
- return cloudTipClassification{}, fmt.Errorf("%w: real frontier for %q unreachable: %w",
- ErrCloudUnavailable, key, walkErr)
- }
-
- for _, tip := range stateTips {
- rd := newDag(e, key, "")
+ op := newClosureOperation(e, make(map[Tip]error))
+ defer op.release()
- walkErr := rd.walk([]Tip{tip}, func(*pb.Effect) error { return nil })
- switch {
- case walkErr == nil:
- classification.install = append(classification.install, tip)
- case errors.Is(walkErr, ErrCDNBlobMissing):
- classification.pending = append(classification.pending, tip)
- default:
- return cloudTipClassification{}, fmt.Errorf("%w: tip %v unreachable for %q: %w",
- ErrCloudUnavailable, tip, key, walkErr)
- }
+ if _, _, err := op.publish(key, tips, true); err != nil {
+ return fmt.Errorf("%w: close cloud frontier for %q: %w", ErrCloudUnavailable, key, err)
}
- return classification, nil
+ return nil
}
// mintLocalEffect persists a fully-built effect as a local mint: marshal,
@@ -825,7 +682,7 @@ func (e *Engine) classifyCloudTips(key string, tips []Tip, sidecar []CloudEffect
// durability hook, broadcast. The effect must already carry its key, HLC,
// fork-choice hash, and deps.
func (e *Engine) mintLocalEffect(key string, currentSet *keytrie.TipSet, eff *pb.Effect) (Tip, error) {
- data, err := MarshalEffect(eff)
+ data, err := e.effectCache.MarshalEffect(eff)
if err != nil {
return Tip{}, err
}
@@ -851,6 +708,7 @@ func (e *Engine) mintLocalEffect(key string, currentSet *keytrie.TipSet, eff *pb
// fireLocalEffect invokes OnLocalEffect for a freshly-minted persisted effect.
func (e *Engine) fireLocalEffect(offset Tip, eff *pb.Effect) {
e.observeTemporal(offset, eff, false)
+ e.recordHistorySent(offset, eff)
e.localEffectMu.RLock()
defer e.localEffectMu.RUnlock()
@@ -923,13 +781,9 @@ func fromPbRefs(refs []*pb.EffectRef) []Tip {
return tips
}
-// resolveTipDeps walks each tip back through pendingTxTips until it
-// lands on a committed ancestor (a bind or a non-tx effect). SSI
-// requires that every dep we hand a new effect references state that
-// has already been committed somewhere in the cluster — never a tip
-// belonging to a foreign in-flight transaction. Single-level
-// substitution would leave us pointing at another tx-tipped offset of
-// the same in-flight txn, which is what we're trying to avoid.
+// resolveTipDeps excludes tentative writes from the value base. Observations
+// remain causal dependencies even when their transaction has not bound: seeing
+// a NOOP does not make that transaction committed or its future bind an ancestor.
func (e *Engine) resolveTipDeps(tips []Tip) []Tip {
// Nearly every read is already at a committed frontier. Avoid allocating
// the recursive walk's closure, seen set, and output slice until at least
@@ -1000,7 +854,7 @@ func (e *Engine) resolveTipDeps(tips []Tip) []Tip {
// via prev_snapshot to the latest existing snapshot tip. Deps are the
// current tips with pending-tx tips substituted; the index is updated
// and the effect is broadcast.
-func (e *Engine) emitSnapshot(key string, verdicts map[string]pb.Verdict) error {
+func (e *Engine) emitSnapshot(key string, verdicts map[string]pb.Verdict, evidence ...*pb.BindEvidence) error {
hlc := timestamppb.New(e.clock.Now())
eff := &pb.Effect{
Key: []byte(key),
@@ -1012,6 +866,10 @@ func (e *Engine) emitSnapshot(key string, verdicts map[string]pb.Verdict) error
}},
}
+ if len(evidence) != 0 {
+ eff.GetSnapshot().Evidence = evidence[0]
+ }
+
currentSet := e.index.Contains(key)
if currentSet != nil {
tips := currentSet.Tips()
@@ -1067,57 +925,6 @@ func (e *Engine) applySnapshotVerdicts(eff *pb.Effect) {
}
}
-// lookupSnapshotVerdict walks the snapshot chain on `key` for `txnID`'s
-// adjudication outcome. Used by arrival-time fork-choice sites to skip a
-// competing bind a prior snapshot already locked in as LOST.
-func (e *Engine) lookupSnapshotVerdict(key, txnID string) (pb.Verdict, bool) {
- tipSet := e.index.Contains(key)
- if tipSet == nil {
- return pb.Verdict_VERDICT_UNSPECIFIED, false
- }
-
- visited := make(map[Tip]bool)
-
- stack := append([]Tip(nil), tipSet.Tips()...)
- for len(stack) > 0 {
- t := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
-
- if visited[t] {
- continue
- }
-
- visited[t] = true
-
- eff, err := e.getEffect(key, t)
- if err != nil {
- continue
- }
-
- if snap := eff.GetSnapshot(); snap != nil {
- if v, ok := snap.GetTxnVerdicts()[txnID]; ok && v != pb.Verdict_VERDICT_UNSPECIFIED {
- return v, true
- }
-
- if snap.GetPrevSnapshot() != nil {
- stack = append(stack, r(snap.GetPrevSnapshot()))
- }
-
- for _, dep := range eff.GetDeps() {
- stack = append(stack, r(dep))
- }
-
- continue
- }
-
- for dep := range keyDeps(key, eff) {
- stack = append(stack, r(dep))
- }
- }
-
- return pb.Verdict_VERDICT_UNSPECIFIED, false
-}
-
// HandleRemote processes a remote effect notification: stores the effect
// in the log, updates the index, and returns NACKs if deps don't match tips.
//
@@ -1345,8 +1152,9 @@ func (e *Engine) HandleRemote(notify *pb.OffsetNotify) ([]*pb.NackNotify, error)
}
}
- // Track in-progress transactional tips
- if eff.GetTxnId() != "" && eff.GetTxnBind() == nil {
+ // Track tentative transaction effects. NOOPs carry observations, not
+ // tentative values, and remain consumable before their transaction binds.
+ if eff.GetTxnId() != "" && eff.GetTxnBind() == nil && eff.GetNoop() == nil {
slog.Debug("HandleRemote: tracking pending tx tip", "offset", notify.GetOrigin(), "deps", eff.GetDeps())
e.pendingTxTips.Store(r(notify.GetOrigin()), fromPbRefs(eff.GetDeps()))
}
@@ -1499,10 +1307,16 @@ func (e *Engine) HandleRemote(notify *pb.OffsetNotify) ([]*pb.NackNotify, error)
nacks = append(nacks, e.buildEnrichedNack(key, notify.GetOrigin(), tipOffsets))
}
- // The bind and its promise are installed, and the response is fixed.
- // Wake application waiters and apply deferred memory pacing outside the lock.
+ var prior []*pendingTxn
+ if eff.GetTxnBind() != nil {
+ prior = e.priorityPredecessors(r(notify.GetOrigin()), eff)
+ }
+ // The bind and its promise are installed. Priority waiting must release
+ // nodeMu so higher-priority binds and their evidence can arrive.
nodeLock.release()
+ nacks = e.finishPriorityResponse(notify.GetOrigin(), nacks, prior)
+
// Fire notification callbacks for remote effects
if eff.GetTxnId() != "" && eff.GetTxnBind() == nil {
// In-progress transactional effect — skip notification until bind
@@ -1654,7 +1468,7 @@ func (e *Engine) ingestBackfill(notify *pb.OffsetNotify, historyKey *string) err
// Verdict-snapshot arrival ends horizon wait for every txn it adjudicates.
e.applySnapshotVerdicts(eff)
- if eff.GetTxnId() != "" && eff.GetTxnBind() == nil {
+ if eff.GetTxnId() != "" && eff.GetTxnBind() == nil && eff.GetNoop() == nil {
e.pendingTxTips.Store(r(notify.GetOrigin()), fromPbRefs(eff.GetDeps()))
}
@@ -1707,24 +1521,10 @@ func (e *Engine) handleRemoteBind(bind *pb.TransactionalBindEffect, bindOffset T
}
}
- // Process abort_deps: remove those offsets and abort any local pending txn
- for _, abortedOffset := range bind.GetAbortDeps() {
- aborted := r(abortedOffset)
- e.pendingTxTips.Delete(aborted)
-
- // Check if any local pending txn has this offset → abort it
- e.pendingTxns.Range(func(_ Tip, ptxn *pendingTxn) bool {
- for _, pk := range ptxn.keys {
- if pk.newTip == aborted {
- ptxn.finish(txnStateAborted)
-
- return false // stop ranging
- }
- }
-
- return true
- })
- }
+ // Ignore legacy abort_deps. A tentative tip does not establish a losing
+ // bind; aborting by offset here can contradict fork choice and expose an
+ // aborted transaction during reconstruction. Peer conflict responses decide
+ // published binds, while the node-locked preflight protects unpublished ones.
// Retain each consuming bind, not just a boolean for its shared base.
// flushTx must prove those binds are ancestors before reusing that base.
@@ -1749,7 +1549,7 @@ func (e *Engine) handleRemoteBind(bind *pb.TransactionalBindEffect, bindOffset T
}
// checkCompetingBinds walks the DAG fragment for each key the bind touches
-// and returns the txnID of any bind whose NewTip on that key is not a
+// and returns the txnID of any bind whose envelope is not a
// structural ancestor of our ConsumedTips — i.e., a bind we know about
// that's space-like concurrent with our forthcoming bind. Returns "" if
// no such bind exists.
@@ -1788,7 +1588,8 @@ func (e *Engine) checkCompetingBinds(bind *pb.TransactionalBindEffect, txnID str
}
// Ancestor closure of our consumed tips on this key. A bind whose
- // NewTip on `k` lands in this set is structurally in our past.
+ // envelope lands in this set is structurally in our past. Observing its
+ // NewTip alone does not establish that it has bound.
ourAncestors := make(map[Tip]struct{}, len(consumedTips)*8)
ancestorStack := make([]Tip, 0, len(consumedTips))
@@ -1810,7 +1611,7 @@ func (e *Engine) checkCompetingBinds(bind *pb.TransactionalBindEffect, txnID str
continue
}
- for _, dep := range eff.GetDeps() {
+ for dep := range keyDeps(k, eff) {
dt := r(dep)
if _, dup := ourAncestors[dt]; dup {
continue
@@ -1840,13 +1641,21 @@ func (e *Engine) checkCompetingBinds(bind *pb.TransactionalBindEffect, txnID str
}
if otherBind := eff.GetTxnBind(); otherBind != nil {
+ if eff.GetTxnId() == txnID {
+ for dep := range keyDeps(k, eff) {
+ walkStack = append(walkStack, r(dep))
+ }
+
+ continue
+ }
+
for _, okb := range otherBind.GetKeys() {
if string(okb.GetKey()) != k {
continue
}
theirNewTip := r(okb.GetNewTip())
- if _, inAncestors := ourAncestors[theirNewTip]; inAncestors {
+ if _, inAncestors := ourAncestors[t]; inAncestors {
walkStack = append(walkStack, theirNewTip)
break
@@ -1874,7 +1683,7 @@ func (e *Engine) checkCompetingBinds(bind *pb.TransactionalBindEffect, txnID str
walkStack = append(walkStack, r(snap.GetPrevSnapshot()))
}
- for _, dep := range eff.GetDeps() {
+ for dep := range keyDeps(k, eff) {
walkStack = append(walkStack, r(dep))
}
}
@@ -1883,117 +1692,6 @@ func (e *Engine) checkCompetingBinds(bind *pb.TransactionalBindEffect, txnID str
return ""
}
-// evaluateBindForkChoice returns true if the bind would lose at the
-// originator's commit point against any existing competitor on its keys
-// (hash-based fork choice with predicate-refinement and shared-base
-// gating) or against a concurrent non-tx data effect (SSI). flushTx
-// uses this to abort before emitting.
-func (e *Engine) evaluateBindForkChoice(bind *pb.TransactionalBindEffect, bindOffset Tip, bindHash []byte, txnID string) bool {
- for _, kb := range bind.GetKeys() {
- k := string(kb.GetKey())
-
- tips := e.index.Contains(k)
- if tips == nil {
- continue
- }
-
- for _, tipOff := range tips.Tips() {
- if tipOff == bindOffset {
- continue
- }
-
- eff, err := e.getEffect(k, tipOff)
- if err != nil {
- continue
- }
-
- // Competing bind: fork-choice determines winner
- if otherBind := eff.GetTxnBind(); otherBind != nil {
- if eff.GetTxnId() == txnID {
- continue // same transaction
- }
- // Already adjudicated as LOST by a snapshot? skip
- if v, ok := e.lookupSnapshotVerdict(k, eff.GetTxnId()); ok && v == pb.Verdict_LOST {
- continue
- }
-
- if bindsShareBase(bind, otherBind) {
- var theirTip Tip
-
- for _, otherKey := range otherBind.GetKeys() {
- if bytes.Equal(kb.GetKey(), otherKey.GetKey()) {
- theirTip = r(otherKey.GetNewTip())
- }
- }
- // Predicate refinement: suppress the fork-choice
- // when both txs carry obs/row-write evidence on
- // this key and their predicates don't intersect
- // the other's writes. Fallback to shared-base
- // conflict when either side lacks evidence
- // (schema-only mutations, non-SQL binds, etc.).
- conflict, bothHadEvidence := e.hasPredicateConflict(
- txnID, eff.GetTxnId(), k,
- []Tip{r(kb.GetNewTip()), bindOffset},
- []Tip{theirTip, tipOff})
- if bothHadEvidence && !conflict {
- continue
- }
-
- if !ForkChoiceLess(bindHash, eff.GetForkChoiceHash()) {
- return true
- }
- }
- }
-
- // Non-transactional data effect: SSI invalidation
- if eff.GetTxnId() == "" && eff.GetData() != nil {
- // Check if this effect is concurrent with our bind on this key:
- // not in the ancestor set of our consumed tips for this key.
- if !refsContain(kb.GetConsumedTips(), tipOff) {
- // Not a consumed tip — could be concurrent.
- // Check if it's a descendant of our bind (depends on us).
- // If it depends on our NewTip or bindOffset, it's sequential.
- isDescendant := false
- // Walk backward from the non-tx effect to see if it reaches our bind
- visited := make(map[Tip]bool)
- stack := []Tip{tipOff}
- zero := Tip{0, 0}
-
- for len(stack) > 0 {
- off := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
-
- if off == bindOffset || off == r(kb.GetNewTip()) {
- isDescendant = true
-
- break
- }
-
- if visited[off] || off == zero {
- continue
- }
-
- visited[off] = true
-
- depEff, err := e.getEffect(k, off)
- if err != nil {
- continue
- }
-
- stack = appendRefs(stack, depEff.GetDeps())
- }
-
- if !isDescendant {
- return true
- }
- }
- }
- }
- }
-
- return false
-}
-
// HandleNack processes a NACK from a remote peer.
func (e *Engine) HandleNack(nack *pb.NackNotify) error {
if nack == nil {
@@ -2016,36 +1714,9 @@ func (e *Engine) HandleNack(nack *pb.NackNotify) error {
// anything else. The NACK is informational; we must consume it.
e.ingestNackTips(nack)
- // Check if this NACK is for a transactional key
- var matchedTxn *pendingTxn
-
- e.pendingTxns.Range(func(_ Tip, ptxn *pendingTxn) bool {
- for _, pk := range ptxn.keys {
- if pk.key == string(nack.GetKey()) {
- matchedTxn = ptxn
-
- return false
- }
- }
-
- return true
- })
-
- if matchedTxn == nil {
- slog.Debug("HandleNack: non-transactional, ignoring", "key", nack.GetKey())
-
- return nil
- }
-
- slog.Debug("HandleNack: transactional conflict check", "key", nack.GetKey(), "bind_offset", matchedTxn.bindOffset)
- // Transactional NACK: smart conflict detection across all keys in the tx
- for _, detail := range nack.GetTipDetails() {
- if e.isRealConflict(matchedTxn, string(nack.GetKey()), detail) {
- matchedTxn.finish(txnStateAborted)
-
- return nil
- }
- }
+ // Asynchronous NACKs carry history, not permission to evaluate a bind.
+ // flushTx collects the tracked responses from every subscriber before
+ // evaluating in priority order. An early NACK cannot abort it here.
return nil
}
@@ -2332,10 +2003,8 @@ func (e *Engine) ingestCausalChain(nack *pb.NackNotify) {
}
// prefetchEffects fetches every listed ref not already resident in parallel
-// and caches the deserialized effects, so a subsequent dag.bfs finds
-// everything locally instead of doing sequential FetchFromAny calls. Purely a
-// cache warmer: it installs no tips, and a failed fetch is skipped — the walk
-// that follows fetches stragglers itself.
+// and caches the deserialized effects for synchronous ingestion. It installs
+// no tips; ingestion resolves dependencies omitted by a failed prefetch.
func (e *Engine) prefetchEffects(refs []Tip) {
if e.effectCache == nil || len(refs) == 0 {
return
@@ -2726,7 +2395,7 @@ func (e *Engine) probeAndFetchKey(key string) {
}},
}
- data, err := MarshalEffect(eff)
+ data, err := e.effectCache.MarshalEffect(eff)
if err != nil {
return
}
diff --git a/effects/emit_test.go b/effects/emit_test.go
index d677382..bf25655 100644
--- a/effects/emit_test.go
+++ b/effects/emit_test.go
@@ -891,7 +891,7 @@ func TestFlushTx_NoNACKs_Commits(t *testing.T) {
}
}
-func TestFlushTx_RealConflict_Aborts(t *testing.T) {
+func TestFlushTx_NonTransactionalNack_Commits(t *testing.T) {
bc := &txnMockBroadcaster{
replicateToResults: map[pb.NodeID]*pb.NackNotify{
99: {
@@ -934,9 +934,8 @@ func TestFlushTx_RealConflict_Aborts(t *testing.T) {
t.Fatal(err)
}
- err := ctx.Flush()
- if !errors.Is(err, ErrTxnAborted) {
- t.Fatalf("expected ErrTxnAborted, got %v", err)
+ if err := ctx.Flush(); err != nil {
+ t.Fatalf("non-transactional NACK must not abort a published bind: %v", err)
}
}
@@ -1069,6 +1068,17 @@ func TestFlushTx_VerdictSnapshotEmittedOnLoserOnlyKeys(t *testing.T) {
t.Fatalf("Put: %v", err)
}
+ // A recomputable verdict owns the losing transaction's payloads, not
+ // just its envelope. Supply the effects referenced by this fixture.
+ for tip, key := range map[Tip]string{loserDataShared: sharedKey, loserDataLoserOnly: loserOnlyKey} {
+ eff := dataEffect(key)
+
+ eff.TxnId = loserTxnID
+ if err := e.effectCache.Put(tip, eff); err != nil {
+ t.Fatal(err)
+ }
+ }
+
// Subscribe peer 99 to the shared key so flushTx replicates our
// bind to peer 99 and processes its NACK response.
subCtx := e.NewContext()
@@ -1135,6 +1145,11 @@ func TestFlushTx_VerdictSnapshotEmittedOnLoserOnlyKeys(t *testing.T) {
loserOnlyKey, loserTxnID, v)
}
+ derived, err := e.proofVerdicts(tip, eff)
+ if err != nil || derived[loserTxnID].verdict != pb.Verdict_LOST {
+ t.Fatalf("loser-only proof: verdicts=%v error=%v", derived, err)
+ }
+
foundLoserOnlyVerdict = true
}
@@ -1372,49 +1387,109 @@ func TestFlushTx_SerializationEscalation_AfterNAborts(t *testing.T) {
}
}
-func TestHandleRemoteBind_AbortsLocalPendingTx(t *testing.T) {
- e := newTxnTestEngine(nil)
+func TestHandleRemoteBind_AbortDepsCannotOverrideForkChoice(t *testing.T) {
+ for _, localWins := range []bool{true, false} {
+ t.Run(fmt.Sprintf("localWins=%t", localWins), func(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ base := log.putEffect(scalarSet("k", "base"))
+ makeBind := func(txn string, node pb.NodeID, stamp int64) (*pb.Effect, Tip, Tip) {
+ log.nodeID = uint64(node)
+ write := scalarSet("k", txn)
+ write.TxnId, write.Deps = txn, toPbRefs([]Tip{base})
+ write.NodeId = uint64(node)
+ tip := log.putEffect(write)
+ bind := &pb.Effect{
+ Key: []byte("k"), TxnId: txn, NodeId: uint64(node), Hlc: sTs(stamp),
+ ForkChoiceHash: ComputeForkChoiceHash(node, sTs(stamp)), Deps: toPbRefs([]Tip{tip}),
+ Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{
+ OriginatorNodeId: uint64(node), TxnHlc: sTs(stamp),
+ Keys: []*pb.TransactionalBindEffect_KeyBind{
+ {Key: []byte("k"), NewTip: toPbRef(tip), ConsumedTips: toPbRefs([]Tip{base})},
+ },
+ }},
+ }
- // Set up a local pending txn
- ptxn := &pendingTxn{
- txnHLC: time.Unix(0, 100),
- originNode: 42,
- bindOffset: Tip{42, 8888},
- keys: []pendingTxnKey{
- {key: "k", newTip: Tip{42, 500}},
- },
- done: make(chan struct{}),
- }
- e.pendingTxns.Store(Tip{42, 8888}, ptxn)
- e.pendingTxTips.Store(Tip{42, 500}, []Tip{{42, 200}})
+ return bind, log.putEffect(bind), tip
+ }
- // Remote bind arrives that aborts our offset 500
- bind := &pb.TransactionalBindEffect{
- TxnHlc: timestamppb.New(time.Unix(0, 50)),
- OriginatorNodeId: 99,
- AbortDeps: []*pb.EffectRef{{NodeId: 42, Offset: 500}},
- Keys: []*pb.TransactionalBindEffect_KeyBind{
- {Key: []byte("k"), NewTip: &pb.EffectRef{NodeId: 99, Offset: 600}},
- },
- }
+ localStamp, remoteStamp := int64(1), int64(2)
+ for ForkChoiceLess(ComputeForkChoiceHash(1, sTs(localStamp)), ComputeForkChoiceHash(2, sTs(remoteStamp))) != localWins {
+ remoteStamp++
+ }
- e.handleRemoteBind(bind, Tip{99, 7777}, "99:50:1")
+ local, localBind, localTip := makeBind("local", 1, localStamp)
+ remote, remoteBind, _ := makeBind("remote", 2, remoteStamp)
+ ptxn := &pendingTxn{
+ txnID: "local", bindOffset: localBind, forkChoiceHash: local.GetForkChoiceHash(),
+ keys: []pendingTxnKey{{key: "k", newTip: localTip, consumedTips: []Tip{base}}},
+ done: make(chan struct{}),
+ }
+ e.pendingTxns.Store(localBind, ptxn)
+ e.pendingTxTips.Store(localTip, []Tip{base})
+ e.updateIndex("k", nil, localBind)
+ e.updateIndex("k", nil, remoteBind)
+
+ // Legacy senders may list our tentative tip even when our already
+ // published bind wins. Arrival alone cannot decide either outcome.
+ remote.GetTxnBind().AbortDeps = toPbRefs([]Tip{localTip})
+ e.handleRemoteBind(remote.GetTxnBind(), remoteBind, "remote")
+
+ if ptxn.state.Load() != txnStatePending {
+ t.Fatal("abort_deps decided the transaction before bind conflict classification")
+ }
- // Our pending txn should be aborted
- if ptxn.state.Load() != txnStateAborted {
- t.Fatal("expected pending txn to be aborted")
- }
+ if _, ok := e.pendingTxTips.Load(localTip); !ok {
+ t.Fatal("abort_deps released another transaction's tentative tip")
+ }
- // done channel should be closed
- select {
- case <-ptxn.done:
- default:
- t.Fatal("done channel should be closed")
- }
+ if err := e.HandleNack(&pb.NackNotify{
+ Key: []byte("k"), Tips: toPbRefs([]Tip{remoteBind}),
+ TipDetails: []*pb.NackTipDetail{nackTipDetail(remoteBind, remote)},
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ if ptxn.state.Load() != txnStatePending {
+ t.Fatal("asynchronous NACK evaluated before priority was established")
+ }
+ // Supply the winner's evidence before evaluating the tracked
+ // response. A bare competing hash is only a priority relation.
+ proof := log.putEffect(&pb.Effect{Key: []byte("k"), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"remote": pb.Verdict_WON},
+ }}})
+
+ nacks := []*pb.NackNotify{e.buildEnrichedNack("k", toPbRef(localBind), []Tip{remoteBind})}
+ if !localWins {
+ nacks = append(nacks, e.buildEnrichedNack("k", toPbRef(localBind), []Tip{proof}))
+ }
- // pendingTxTips should be cleaned up
- if _, ok := e.pendingTxTips.Load(Tip{42, 500}); ok {
- t.Fatal("pendingTxTips should be cleaned for aborted offset")
+ _, err := e.evaluateNacks(ptxn, nacks)
+ if aborted := errors.Is(err, ErrTxnAborted); aborted == localWins {
+ t.Fatalf("abort=%t, expected %t (error %v)", aborted, !localWins, err)
+ }
+
+ if err != nil && !errors.Is(err, ErrTxnAborted) {
+ t.Fatal(err)
+ }
+
+ e.pendingTxns.Delete(localBind)
+ e.pendingTxTips.Delete(localTip)
+
+ value, _, _, err := e.GetSnapshot("k")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ want := "remote"
+ if localWins {
+ want = "local"
+ }
+
+ if got := string(value.GetScalar().GetRaw()); got != want {
+ t.Fatalf("reconstruction returned %q, want %q", got, want)
+ }
+ })
}
}
@@ -1494,7 +1569,7 @@ func TestHandleNack_NonTransactional_NoOp(t *testing.T) {
}
}
-func TestHandleNack_TransactionalKey_AbortsOnConflict(t *testing.T) {
+func TestHandleNack_TransactionalKey_DefersEvaluation(t *testing.T) {
e := newTxnTestEngine(nil)
ptxn := &pendingTxn{
@@ -1524,8 +1599,12 @@ func TestHandleNack_TransactionalKey_AbortsOnConflict(t *testing.T) {
t.Fatal(err)
}
- if ptxn.state.Load() != txnStateAborted {
- t.Fatal("expected pending txn to be aborted by HandleNack")
+ if ptxn.state.Load() != txnStatePending {
+ t.Fatal("asynchronous NACK must not evaluate before all subscriber responses")
+ }
+
+ if _, err := e.evaluateNacks(ptxn, []*pb.NackNotify{nack}); err != nil {
+ t.Fatalf("completed responses must not treat non-transactional data as a defeated bind: %v", err)
}
}
@@ -1710,7 +1789,7 @@ func TestSSI_ReadWriteConflict_Aborts(t *testing.T) {
}
}
-func TestSSI_SubscriberNACK_AbortsLoser(t *testing.T) {
+func TestSSI_SubscriberNACK_UnresolvedPriorityIsAmbiguous(t *testing.T) {
// Use the smallest possible hash (all zeros) to guarantee FWW victory
// against any real SHA-256 hash the engine will compute.
theirNode := pb.NodeID(0)
@@ -1786,10 +1865,10 @@ func TestSSI_SubscriberNACK_AbortsLoser(t *testing.T) {
t.Fatal(err)
}
- // Flush — subscriber NACKs with a competing bind that wins FWW
+ // A bare competing hash cannot establish whether that bind survived.
err = ctx.Flush()
- if !errors.Is(err, ErrTxnAborted) {
- t.Fatalf("expected ErrTxnAborted from subscriber NACK, got %v", err)
+ if !errors.Is(err, ErrTxnAmbiguous) {
+ t.Fatalf("expected ErrTxnAmbiguous from unresolved subscriber NACK, got %v", err)
}
}
diff --git a/effects/evidence.go b/effects/evidence.go
new file mode 100644
index 0000000..5f141e6
--- /dev/null
+++ b/effects/evidence.go
@@ -0,0 +1,451 @@
+/*
+ * Copyright 2026 Swytch Labs BV
+ *
+ * This file is part of Swytch.
+ *
+ * Swytch is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Swytch is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Swytch. If not, see .
+ *
+ * Additional permissions under GNU AGPL version 3 section 7 apply to
+ * this file. See the NOTICE.md file distributed with this source code
+ * for the current set of additional permissions.
+ */
+
+package effects
+
+import (
+ "fmt"
+ "slices"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "google.golang.org/protobuf/proto"
+)
+
+// proofRecords reads explicit ownership edges only. Value ancestry is neither
+// fetched nor interpreted here; the snapshot already owns these vertices.
+func (p *VertexPool) proofRecords(eff *pb.Effect) (map[Tip]*pb.Effect, error) {
+ records := make(map[Tip]*pb.Effect)
+
+ queue := uniqueEvidence(eff)
+ for len(queue) != 0 {
+ tip := queue[len(queue)-1]
+ queue = queue[:len(queue)-1]
+
+ if records[tip] != nil {
+ continue
+ }
+
+ proof, ok := p.Get(tip)
+ if !ok {
+ return nil, fmt.Errorf("snapshot evidence %v: %w", tip, errEffectNotResident)
+ }
+
+ records[tip] = proof
+ queue = append(queue, uniqueEvidence(proof)...)
+ }
+
+ return records, nil
+}
+
+// proofVerdicts uses the ordinary bind evaluator against the packaged graph.
+// Cached verdict labels are deliberately excluded from this computation.
+func (e *Engine) proofVerdicts(owner Tip, eff *pb.Effect) (verdictMap, error) {
+ if !e.effectCache.Incref(owner) {
+ return nil, fmt.Errorf("proof owner %v: %w", owner, errEffectNotResident)
+ }
+ defer e.effectCache.Decref(owner)
+
+ records, err := e.effectCache.proofRecords(eff)
+ if err != nil {
+ return nil, err
+ }
+
+ binds := make(map[Tip]*pb.Effect)
+
+ for _, ref := range eff.GetSnapshot().GetEvidence().GetBinds() {
+ tip := r(ref)
+
+ bind := records[tip]
+ if bind.GetTxnBind() == nil || bind.GetTxnId() == "" {
+ return nil, fmt.Errorf("invalid proof bind %v", tip)
+ }
+
+ binds[tip] = bind
+ }
+
+ keys := make(map[string][]Tip)
+
+ for tip, record := range binds {
+ for _, kb := range record.GetTxnBind().GetKeys() {
+ key := string(kb.GetKey())
+ keys[key] = append(keys[key], tip)
+ }
+ }
+
+ lost, _ := e.losersOnKeysAt(keys, func(string) []Tip { return nil }, nil, nil, false, records)
+ verdicts := make(verdictMap)
+
+ for _, record := range binds {
+ verdict := pb.Verdict_WON
+ if _, loses := lost[record.GetTxnId()]; loses {
+ verdict = pb.Verdict_LOST
+ }
+
+ verdicts[record.GetTxnId()] = verdictEntry{verdict: verdict, snapshotTip: owner, proven: true}
+ }
+
+ return verdicts, nil
+}
+
+func (e *Engine) harvestProofVerdicts(tip Tip, eff *pb.Effect, verdicts verdictMap) error {
+ if eff.GetSnapshot().GetEvidence() == nil {
+ verdicts.harvest(tip, eff.GetSnapshot())
+
+ return nil
+ }
+
+ derived, err := e.proofVerdicts(tip, eff)
+ if err != nil {
+ return err
+ }
+
+ for txn, entry := range derived {
+ verdicts.retain(txn, entry)
+ }
+
+ return nil
+}
+
+// bindProof retains the transaction records used for predicate evaluation and
+// predecessor proofs that allowed this bind to proceed. It does not retain a
+// key's reconstructed value or unconnected history. The returned temporary pins
+// last until the emitted snapshots acquire their own references.
+func (e *Engine) bindProof(bind Tip, defeated map[string]Tip, sources []Tip) (proof *pb.BindEvidence, err error) {
+ var pins []Tip
+
+ seen := make(map[Tip]bool)
+
+ defer func() {
+ if err != nil {
+ e.effectCache.releaseEvidence(pins)
+ pins = nil
+ }
+ }()
+
+ retain := func(tip Tip) (*pb.Effect, error) {
+ eff, loadErr := e.getEffect("", tip)
+ if loadErr != nil {
+ return nil, loadErr
+ }
+
+ if !seen[tip] {
+ if !e.effectCache.Incref(tip) {
+ return nil, fmt.Errorf("proof member %v: %w", tip, errEffectNotResident)
+ }
+
+ seen[tip] = true
+ pins = append(pins, tip)
+ }
+
+ return eff, nil
+ }
+ binds := []Tip{bind}
+
+ for _, source := range sources {
+ eff, loadErr := retain(source)
+ if loadErr != nil {
+ return nil, loadErr
+ }
+
+ if eff.GetSnapshot().GetEvidence() == nil {
+ // Old peers supply cached labels without a recomputable proof.
+ // Preserve legacy resolution, but do not claim it is a new proof.
+ e.effectCache.releaseEvidence(pins)
+
+ return nil, nil
+ }
+
+ binds = append(binds, fromPbRefs(eff.GetSnapshot().GetEvidence().GetBinds())...)
+ }
+
+ for _, tip := range defeated {
+ binds = append(binds, tip)
+ }
+
+ for _, tip := range binds {
+ eff, loadErr := retain(tip)
+ if loadErr != nil {
+ return nil, loadErr
+ }
+
+ for _, kb := range eff.GetTxnBind().GetKeys() {
+ boundary := make(map[Tip]bool)
+ for _, ref := range kb.GetConsumedTips() {
+ boundary[r(ref)] = true
+ }
+
+ visited := make(map[Tip]bool)
+
+ queue := []Tip{r(kb.GetNewTip())}
+ for len(queue) != 0 {
+ next := queue[len(queue)-1]
+ queue = queue[:len(queue)-1]
+
+ if next == (Tip{}) || boundary[next] || visited[next] {
+ continue
+ }
+
+ visited[next] = true
+
+ data, loadErr := e.getEffect(string(kb.GetKey()), next)
+ if loadErr != nil {
+ return nil, loadErr
+ }
+
+ if data.GetTxnId() != eff.GetTxnId() {
+ continue
+ }
+
+ if _, loadErr := retain(next); loadErr != nil {
+ return nil, loadErr
+ }
+
+ for ref := range keyDeps(string(kb.GetKey()), data) {
+ queue = append(queue, r(ref))
+ }
+ }
+ }
+ }
+ // Retain a connecting path when two proof binds are ordered. Keeping only
+ // their envelopes would otherwise turn missing connectors into competition.
+ all := make(map[Tip]*pb.Effect, len(binds))
+ for _, tip := range binds {
+ all[tip], _ = e.effectCache.Get(tip)
+ }
+
+ for from, first := range all {
+ for to, second := range all {
+ if from == to {
+ continue
+ }
+
+ for _, kb := range first.GetTxnBind().GetKeys() {
+ key := string(kb.GetKey())
+
+ if !slices.ContainsFunc(second.GetTxnBind().GetKeys(), func(k *pb.TransactionalBindEffect_KeyBind) bool { return string(k.GetKey()) == key }) {
+ continue
+ }
+
+ for _, t := range e.bindProofPath(key, from, to) {
+ if _, loadErr := retain(t); loadErr != nil {
+ return nil, loadErr
+ }
+ }
+ }
+ }
+ }
+
+ slices.SortFunc(pins, func(a, b Tip) int { return slices.Compare(a[:], b[:]) })
+
+ proof = &pb.BindEvidence{}
+
+ for _, tip := range pins {
+ if slices.Contains(binds, tip) {
+ proof.Binds = append(proof.Binds, toPbRef(tip))
+ } else {
+ proof.Deps = append(proof.Deps, toPbRef(tip))
+ }
+ }
+
+ return proof, nil
+}
+
+func (e *Engine) bindProofPath(key string, from, to Tip) []Tip {
+ parents := map[Tip]Tip{from: {}}
+
+ queue := []Tip{from}
+ for i := 0; i < len(queue); i++ {
+ tip := queue[i]
+ if tip == to {
+ var path []Tip
+ for tip != from {
+ path = append(path, tip)
+ tip = parents[tip]
+ }
+
+ return path
+ }
+
+ eff, _ := e.effectCache.Get(tip)
+ for ref := range keyDeps(key, eff) {
+ dep := r(ref)
+ if _, visited := parents[dep]; !visited && dep != (Tip{}) {
+ parents[dep] = tip
+ queue = append(queue, dep)
+ }
+ }
+ }
+
+ return nil
+}
+
+func uniqueEvidence(eff *pb.Effect) []Tip {
+ return evidenceTips(eff.GetSnapshot().GetEvidence())
+}
+
+func evidenceTips(proof *pb.BindEvidence) []Tip {
+ refs := slices.Concat(proof.GetBinds(), proof.GetDeps())
+ if len(refs) == 0 {
+ return nil
+ }
+
+ seen := make(map[Tip]bool, len(refs))
+
+ tips := make([]Tip, 0, len(refs))
+ for _, ref := range refs {
+ tip := r(ref)
+ if !seen[tip] {
+ seen[tip] = true
+ tips = append(tips, tip)
+ }
+ }
+
+ return tips
+}
+
+func (p *VertexPool) releaseEvidence(tips []Tip) {
+ for _, tip := range tips {
+ p.Decref(tip)
+ }
+}
+
+func (p *VertexPool) pinEvidence(owner Tip, eff *pb.Effect) ([]Tip, error) {
+ tips := uniqueEvidence(eff)
+ for i, tip := range tips {
+ if tip == (Tip{}) || tip == owner || !p.Incref(tip) {
+ p.releaseEvidence(tips[:i])
+
+ return nil, fmt.Errorf("snapshot %v: evidence %v is not resident or is self-referential", owner, tip)
+ }
+ }
+
+ return tips, nil
+}
+
+// MarshalEffect packages only explicit proof references. Ordinary dependencies
+// retain their existing wire semantics. Records precede their proof owners and
+// shared nested proofs appear once, avoiding repeated serialization of history.
+func (p *VertexPool) MarshalEffect(eff *pb.Effect) ([]byte, error) {
+ if eff.GetSnapshot().GetEvidence() == nil || len(eff.GetEvidencePackage()) != 0 {
+ return MarshalEffect(eff)
+ }
+
+ if p == nil {
+ return nil, fmt.Errorf("cannot package snapshot evidence without a vertex pool")
+ }
+
+ seen := make(map[Tip]uint8)
+
+ var pins []Tip
+ defer func() { p.releaseEvidence(pins) }()
+
+ var (
+ records []*pb.EvidenceRecord
+ visit func(Tip) error
+ )
+
+ visit = func(tip Tip) error {
+ if seen[tip] == 2 {
+ return nil
+ }
+
+ if seen[tip] == 1 {
+ return fmt.Errorf("cyclic snapshot evidence at %v", tip)
+ }
+
+ if !p.Incref(tip) {
+ return fmt.Errorf("snapshot evidence %v is not resident", tip)
+ }
+
+ pins = append(pins, tip)
+ seen[tip] = 1
+
+ proof, _ := p.Get(tip)
+ for _, dep := range uniqueEvidence(proof) {
+ if err := visit(dep); err != nil {
+ return err
+ }
+ }
+
+ data, err := MarshalEffect(proof)
+ if err != nil {
+ return err
+ }
+
+ records = append(records, &pb.EvidenceRecord{Ref: toPbRef(tip), EffectData: data})
+ seen[tip] = 2
+
+ return nil
+ }
+ for _, tip := range uniqueEvidence(eff) {
+ if err := visit(tip); err != nil {
+ return nil, err
+ }
+ }
+
+ wire := proto.Clone(eff).(*pb.Effect)
+ wire.EvidencePackage = records
+
+ return MarshalEffect(wire)
+}
+
+func (p *VertexPool) installEvidencePackage(owner Tip, eff *pb.Effect) (*pb.Effect, func(), error) {
+ var pins []Tip
+
+ release := func() { p.releaseEvidence(pins) }
+
+ for _, record := range eff.GetEvidencePackage() {
+ tip := r(record.GetRef())
+ if tip == (Tip{}) || tip == owner {
+ release()
+
+ return nil, nil, fmt.Errorf("invalid evidence identity %v for %v", tip, owner)
+ }
+
+ proof := &pb.Effect{}
+ if err := UnmarshalEffect(record.GetEffectData(), proof); err != nil {
+ release()
+
+ return nil, nil, err
+ }
+
+ if len(proof.GetEvidencePackage()) != 0 {
+ release()
+
+ return nil, nil, fmt.Errorf("nested evidence packages must be flattened")
+ }
+
+ if err := p.putWithPin(tip, proof, false, nil, true); err != nil {
+ release()
+
+ return nil, nil, err
+ }
+
+ pins = append(pins, tip)
+ }
+
+ clean := proto.Clone(eff).(*pb.Effect)
+ clean.EvidencePackage = nil
+
+ return clean, release, nil
+}
diff --git a/effects/evidence_test.go b/effects/evidence_test.go
new file mode 100644
index 0000000..feed587
--- /dev/null
+++ b/effects/evidence_test.go
@@ -0,0 +1,384 @@
+package effects
+
+import (
+ "context"
+ "sync"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+func evidenceSnapshot(key string, refs ...Tip) *pb.Effect {
+ return &pb.Effect{Key: []byte(key), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{Evidence: &pb.BindEvidence{Deps: toPbRefs(refs)}}}}
+}
+
+func bindEvidenceSnapshot(key string, proof *pb.BindEvidence) *pb.Effect {
+ return &pb.Effect{Key: []byte(key), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{Evidence: proof}}}
+}
+
+func TestSnapshotEvidenceReconstructsPriorityChainWithoutLabels(t *testing.T) {
+ log := newSnapshotLog()
+ source := newSnapshotEngine(log)
+
+ t.Cleanup(func() { _ = source.Close() })
+
+ c := closureBind(log, "C", 0, []string{"bc"}, nil)
+ b := closureBind(log, "B", 1, []string{"winner", "bc"}, nil)
+ a := closureBind(log, "A", 2, []string{"winner"}, nil)
+ // Unrelated history is available at the sender but is not proof.
+ unrelated := log.putEffect(scalarSet("bc", "not part of the proof"))
+
+ cProof, err := source.bindProof(c, map[string]Tip{"B": b}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cSnapshot := log.putEffect(bindEvidenceSnapshot("bc", cProof))
+ source.effectCache.releaseEvidence(evidenceTips(cProof))
+
+ aProof, err := source.bindProof(a, nil, []Tip{cSnapshot})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ snapshot := bindEvidenceSnapshot("winner", aProof)
+ snapshot.Deps = toPbRefs([]Tip{a})
+ owner := log.putEffect(snapshot)
+
+ source.effectCache.releaseEvidence(evidenceTips(aProof))
+ snapshot, _ = source.effectCache.Get(owner)
+
+ data, err := source.effectCache.MarshalEffect(snapshot)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fresh := newTestEngine(nil)
+
+ t.Cleanup(func() { _ = fresh.Close() })
+ preSubscribe(fresh, "winner")
+
+ if _, err := fresh.HandleRemote(&pb.OffsetNotify{Origin: toPbRef(owner), Key: []byte("winner"), EffectData: data}); err != nil {
+ t.Fatal(err)
+ }
+
+ got, err := fresh.GetSnapshots(context.Background(), []string{"winner"})
+ if err != nil || len(got) != 1 || string(got[0].GetScalar().GetRaw()) != "A" {
+ t.Fatalf("fresh-node reconstruction=%v err=%v, want A", got, err)
+ }
+
+ if _, ok := fresh.subscriptions.Load("bc"); ok {
+ t.Fatal("proof evaluation subscribed to another key")
+ }
+
+ if _, ok := fresh.effectCache.Get(unrelated); ok {
+ t.Fatal("proof package included unrelated key history")
+ }
+
+ stored, _ := fresh.effectCache.Get(owner)
+
+ verdicts, err := fresh.proofVerdicts(owner, stored)
+ if err != nil || verdicts["A"].verdict != pb.Verdict_WON || verdicts["B"].verdict != pb.Verdict_LOST || verdicts["C"].verdict != pb.Verdict_WON {
+ t.Fatalf("derived verdicts=%v err=%v", verdicts, err)
+ }
+
+ fresh.releaseChainRefs("winner", fresh.index.Contains("winner").Tips(), fresh.index.LoadData("winner"))
+ drainEvidence(fresh.effectCache)
+
+ if fresh.effectCache.EntryCount() != 0 {
+ t.Fatal("reclaiming the final proof owner leaked evidence")
+ }
+}
+
+func drainEvidence(p *VertexPool) {
+ for p.reclaimPending.Load() != 0 {
+ p.drainReclaim()
+ }
+}
+
+func TestSnapshotEvidenceOwnership(t *testing.T) {
+ p := newVertexPool()
+
+ proof, first, second := Tip{1, 1}, Tip{1, 2}, Tip{1, 3}
+ if err := p.PutSizedCache(proof, scalarSet("foreign", "proof"), 0); err != nil {
+ t.Fatal(err)
+ }
+
+ for _, tip := range []Tip{first, second, first} {
+ if err := p.Put(tip, evidenceSnapshot("k", proof, proof)); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ drainEvidence(p)
+
+ if got := p.getVertex(proof).refs.Load(); got != 2 {
+ t.Fatalf("evidence refs=%d, want one per snapshot vertex", got)
+ }
+
+ if !p.Incref(first) {
+ t.Fatal("pin first snapshot")
+ }
+
+ p.decrefChainNode("k", first)
+ p.decrefChainNode("k", second)
+ drainEvidence(p)
+
+ if got := p.getVertex(proof).refs.Load(); got != 1 {
+ t.Fatalf("evidence refs after key release=%d, want independent snapshot pin to keep one", got)
+ }
+
+ p.Decref(first)
+ drainEvidence(p)
+
+ if p.EntryCount() != 0 || p.vertexBytes.Load() != 0 || p.owned.residentBytes() != 0 {
+ t.Fatal("snapshot reclamation leaked its proof or accounting")
+ }
+}
+
+func TestSnapshotEvidenceMissingMemberRollsBack(t *testing.T) {
+ p := newVertexPool()
+
+ proof, missing, owner := Tip{1, 1}, Tip{1, 2}, Tip{1, 3}
+ if err := p.PutSizedCache(proof, scalarSet("foreign", "proof"), 0); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := p.Put(owner, evidenceSnapshot("k", proof, missing)); err == nil {
+ t.Fatal("published snapshot with missing evidence")
+ }
+
+ drainEvidence(p)
+
+ if p.EntryCount() != 0 {
+ t.Fatal("failed evidence installation retained partial ownership")
+ }
+}
+
+func TestSnapshotEvidencePackage(t *testing.T) {
+ source := newVertexPool()
+ proof, previous, owner := Tip{1, 1}, Tip{1, 2}, Tip{1, 3}
+ // This unavailable ordinary dependency must never be fetched merely
+ // because the proof package includes this vertex.
+ payload := scalarSet("foreign", "proof")
+
+ payload.Deps = toPbRefs([]Tip{{99, 99}})
+ if err := source.Put(proof, payload); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := source.Put(previous, evidenceSnapshot("foreign", proof)); err != nil {
+ t.Fatal(err)
+ }
+
+ snapshot := evidenceSnapshot("k", previous, proof)
+
+ data, err := source.MarshalEffect(snapshot)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ decoded := &pb.Effect{}
+ if err := UnmarshalEffect(data, decoded); err != nil {
+ t.Fatal(err)
+ }
+
+ if len(decoded.GetEvidencePackage()) != 2 {
+ t.Fatalf("package has %d records, want shared proof once", len(decoded.GetEvidencePackage()))
+ }
+
+ e := newTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "k")
+
+ var wg sync.WaitGroup
+ for range 16 {
+ wg.Go(func() {
+ if err := e.putIngested(owner, decoded, len(data)); err != nil {
+ t.Error(err)
+ }
+ })
+ }
+
+ done := make(chan struct{})
+
+ go func() { wg.Wait(); close(done) }()
+
+reclaim:
+ for {
+ select {
+ case <-done:
+ break reclaim
+ default:
+ e.effectCache.drainReclaim()
+ }
+ }
+
+ stored, _ := e.effectCache.Get(owner)
+ if stored == nil || len(stored.GetEvidencePackage()) != 0 {
+ t.Fatal("pool retained transport package or failed to install owner")
+ }
+
+ if _, complete, err := e.closeIngestedFrontier("k", []Tip{owner}, make(map[Tip]error)); err != nil || !complete {
+ t.Fatalf("closure tried to reconstruct the foreign proof: complete=%t err=%v", complete, err)
+ }
+
+ if _, subscribed := e.subscriptions.Load("foreign"); subscribed {
+ t.Fatal("receiving proof subscribed to the foreign key")
+ }
+
+ if got := e.effectCache.getVertex(proof).refs.Load(); got != 2 {
+ t.Fatalf("duplicate package installation added evidence ownership: %d", got)
+ }
+
+ e.effectCache.decrefChainNode("k", owner)
+ drainEvidence(e.effectCache)
+
+ if e.effectCache.EntryCount() != 0 {
+ t.Fatal("nested proof leaked after owner reclamation")
+ }
+}
+
+func TestSnapshotEvidenceSupportingBindIsNotCandidate(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+
+ t.Cleanup(func() { _ = e.Close() })
+
+ old := closureBind(log, "old", 1, []string{"k"}, nil)
+
+ oldProof, err := e.bindProof(old, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ prior := log.putEffect(bindEvidenceSnapshot("k", oldProof))
+ e.effectCache.releaseEvidence(evidenceTips(oldProof))
+
+ connector := closureBind(log, "connector", 0, []string{"k"}, map[string]Tip{"k": old})
+ current := closureBind(log, "current", 2, []string{"k"}, map[string]Tip{"k": connector})
+
+ proof, err := e.bindProof(current, nil, []Tip{prior})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ owner := log.putEffect(bindEvidenceSnapshot("k", proof))
+ e.effectCache.releaseEvidence(evidenceTips(proof))
+ eff, _ := e.effectCache.Get(owner)
+
+ data, err := e.effectCache.MarshalEffect(eff)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ fresh := newSnapshotEngine(nil)
+
+ t.Cleanup(func() { _ = fresh.Close() })
+
+ wire := &pb.Effect{}
+ if err := UnmarshalEffect(data, wire); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := fresh.effectCache.Put(owner, wire); err != nil {
+ t.Fatal(err)
+ }
+
+ stored, _ := fresh.effectCache.Get(owner)
+
+ verdicts, err := fresh.proofVerdicts(owner, stored)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(verdicts) != 2 || verdicts["old"].verdict != pb.Verdict_WON || verdicts["current"].verdict != pb.Verdict_WON {
+ t.Fatalf("ordered proof binds must both win, connector must not be evaluated: %v", verdicts)
+ }
+
+ if _, ok := fresh.effectCache.Get(connector); !ok {
+ t.Fatal("proof omitted connecting ancestry")
+ }
+}
+
+func TestSnapshotEvidenceMalformedPackageRollsBack(t *testing.T) {
+ p := newVertexPool()
+ member, nested, owner := Tip{1, 1}, Tip{1, 2}, Tip{1, 3}
+
+ data, err := MarshalEffect(scalarSet("foreign", "proof"))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ broken, err := MarshalEffect(evidenceSnapshot("foreign", member, Tip{99, 99}))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ eff := evidenceSnapshot("k", nested)
+
+ eff.EvidencePackage = []*pb.EvidenceRecord{
+ {Ref: toPbRef(member), EffectData: data},
+ {Ref: toPbRef(nested), EffectData: broken},
+ }
+ if err := p.Put(owner, eff); err == nil {
+ t.Fatal("installed incomplete nested evidence")
+ }
+
+ drainEvidence(p)
+
+ if p.EntryCount() != 0 || p.owned.residentBytes() != 0 {
+ t.Fatal("failed package retained partial ownership")
+ }
+}
+
+func TestSnapshotEvidenceDurabilityKeepsProofOutOfFrontiers(t *testing.T) {
+ dir := t.TempDir()
+ source := newVertexPool()
+
+ member, owner := Tip{1, 1}, Tip{1, 2}
+ if err := source.Put(member, scalarSet("foreign", "proof")); err != nil {
+ t.Fatal(err)
+ }
+
+ data, err := source.MarshalEffect(evidenceSnapshot("k", member))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ wire := &pb.Effect{}
+ if err := UnmarshalEffect(data, wire); err != nil {
+ t.Fatal(err)
+ }
+
+ e := durableTestEngine(t, dir)
+ if err := e.effectCache.Put(owner, wire); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := e.Close(); err != nil {
+ t.Fatal(err)
+ }
+
+ e = durableTestEngine(t, dir)
+ if !e.index.Contains("k").Contains(owner) {
+ t.Fatal("proof owner not restored")
+ }
+
+ if e.index.Contains("foreign").Len() != 0 {
+ t.Fatal("proof became foreign value frontier after restart")
+ }
+
+ if v := e.effectCache.getVertex(member); v == nil || v.refs.Load() != 1 {
+ t.Fatal("proof ownership not restored")
+ }
+
+ e.releaseChainRefs("k", e.index.Contains("k").Tips(), nil)
+ e.index.Delete("k", e.index.Contains("k"))
+ drainEvidence(e.effectCache)
+
+ if _, ok := e.effectCache.Get(member); ok {
+ t.Fatal("restored proof leaked after owner reclaim")
+ }
+}
diff --git a/effects/frontier.go b/effects/frontier.go
index 72501b5..77013d8 100644
--- a/effects/frontier.go
+++ b/effects/frontier.go
@@ -1,57 +1,710 @@
package effects
import (
+ "bytes"
+ "cmp"
+ "errors"
+ "fmt"
+ "hash/maphash"
"log/slog"
"slices"
+ "sync"
+ "github.com/cespare/xxhash/v2"
pb "github.com/swytchdb/engine/cluster/proto"
"github.com/swytchdb/engine/keytrie"
)
-// installIngestedTips merges arrivals with the live frontier using the shared
-// causal normalization. An arrival may be an ancestor of an existing tip, including
-// when it fills a previously missing link. Consuming only its immediate deps
-// would resurrect that ancestor on out-of-order delivery or replay.
-//
-// Missing history is not evidence of dominance: keep unresolved candidates and
-// report the error without rejecting the received effect. getEffect uses the
-// same cache-first dependency resolution as reads and cloud normalization.
+// closureLock must be acquired before allocating traversal state. Keep at
+// most one shard held, including for multi-key binds. When nodeMu is needed,
+// it is acquired first; closure must never acquire it or reenter ingestion.
+func (e *Engine) closureLock(key string) *sync.Mutex {
+ return &e.closureLocks[xxhash.Sum64String(key)%uint64(len(e.closureLocks))]
+}
+
+// installIngestedTips reconciles the incoming and existing causal regions.
+// Closure discovers graph relationships, not values: snapshots have the same
+// dependency semantics as any other vertex here.
func (e *Engine) installIngestedTips(key string, arrivals ...Tip) {
+ op := newClosureOperation(e, make(map[Tip]error))
+ defer op.release()
+
+ _, _, err := op.publish(key, arrivals, false)
+ if err != nil {
+ slog.Warn("frontier closure incomplete; retaining unresolved tips", "key", key, "error", err)
+ }
+}
+
+// closureOperation owns discovery through publication. Cloud and notification
+// ingestion share the publisher; recovery uses the same discovery without
+// publishing. Optional observation and an explicit CAS let regressions count
+// actual graph work and force contention without timing-dependent goroutines.
+type closureOperation struct {
+ engine *Engine
+ attempted map[Tip]error
+ work map[string]*closureWork
+ observe func(Tip, *pb.EffectRef) // nil ref: vertex expansion; otherwise: dependency edge
+ cas func(string, *keytrie.TipSet, *keytrie.TipSet) (*keytrie.TipSet, bool)
+}
+
+func newClosureOperation(e *Engine, attempted map[Tip]error) *closureOperation {
+ return &closureOperation{
+ engine: e, attempted: attempted, work: make(map[string]*closureWork),
+ cas: e.index.Insert,
+ }
+}
+
+func (op *closureOperation) release() {
+ for _, w := range op.work {
+ w.release()
+ }
+
+ clear(op.work)
+}
+
+func (op *closureOperation) publish(key string, arrivals []Tip, failOnError bool) ([]Tip, bool, error) {
+ return op.run(key, arrivals, true, failOnError)
+}
+
+func (e *Engine) closeIngestedFrontier(key string, supplied []Tip, attempted map[Tip]error) ([]Tip, bool, error) {
+ op := newClosureOperation(e, attempted)
+ defer op.release()
+
+ return op.run(key, supplied, false, false)
+}
+
+// run keeps discovery alive through the CAS. Same-key work and publication use
+// one shard hold; only associated-key discovery releases it, before taking any
+// other shard. Each key keeps its own cuts and consumption relationships.
+func (op *closureOperation) run(key string, supplied []Tip, publish, failOnError bool) ([]Tip, bool, error) {
+ e := op.engine
+
+ type branch struct {
+ key string
+ tip Tip
+ }
+
+ seen := make(map[branch]bool)
+ pending := make(map[string][]Tip)
+
+ var queue []string
+
+ walk := func(key string, tips []Tip) []Tip {
+ return op.walk(key, tips, func(tip Tip, bind *pb.TransactionalBindEffect) {
+ for _, kb := range bind.GetKeys() {
+ other := string(kb.GetKey())
+
+ b := branch{other, tip}
+ if seen[b] || !e.isServed(kb.GetKey()) {
+ continue
+ }
+
+ seen[b] = true
+
+ if other != key {
+ if _, queued := pending[other]; !queued {
+ queue = append(queue, other)
+ }
+
+ pending[other] = append(pending[other], tip)
+ }
+ }
+ })
+ }
+
+ mu := e.closureLock(key)
+ mu.Lock()
+
+ locked := true
+ defer func() {
+ if locked {
+ mu.Unlock()
+ }
+ }()
+
+ retry := false
+
for {
current := e.index.Contains(key)
- candidates := keytrie.NewTipSet(slices.Concat(current.Tips(), arrivals)...)
- tips := candidates.Tips()
- if len(tips) > 1 {
- live, _, err := normalizeTipCandidates(key, tips, nil, func(tip Tip) (*pb.Effect, error) {
- return e.getEffect(key, tip)
- })
- if err != nil {
- slog.Warn("frontier normalization incomplete; retaining unresolved tips", "key", key, "error", err)
+ tips := supplied
+ if publish {
+ tips = slices.Concat(current.Tips(), supplied)
+ }
+
+ if retry {
+ // Associated frontiers and unresolved boundaries may also have
+ // advanced. Extend their existing cuts without repeating history.
+ for other := range op.work {
+ if other != key {
+ pending[other] = nil
+ queue = append(queue, other)
+ }
}
- candidates = keytrie.NewTipSet(live...)
+ slices.Sort(queue)
+
+ retry = false
}
- // Preserve index identity for an unchanged frontier. Readers use that
- // identity to detect concurrent writes, so duplicate recovery must not
- // invalidate their snapshots.
- if current.Len() == candidates.Len() && current.ContainsAll(candidates.Tips()) {
- if e.index.Contains(key) == current {
- return
+ live := walk(key, tips)
+
+ if len(queue) != 0 {
+ locked = false
+ mu.Unlock()
+
+ for next := 0; next < len(queue); next++ {
+ other := queue[next]
+
+ func() {
+ otherMu := e.closureLock(other)
+ otherMu.Lock()
+ defer otherMu.Unlock()
+ // Associated branches supply closure, not an index update on
+ // another key. Ingestion still owns bind publication ordering.
+ tips := slices.Concat(e.index.Contains(other).Tips(), pending[other])
+ delete(pending, other)
+ walk(other, tips)
+ }()
}
+ queue = queue[:0]
+
+ mu.Lock()
+ locked = true
+ // Incorporate local publications made while another key was
+ // being explored before trying to publish this key's frontier.
continue
}
- if _, ok := e.index.Insert(key, current, candidates); ok {
+ var err error
+
+ for _, w := range op.work {
+ for _, failure := range w.failures {
+ err = errors.Join(err, failure)
+ }
+ }
+
+ complete := err == nil
+ if !publish || failOnError && err != nil {
+ return live, complete, err
+ }
+
+ next := keytrie.NewTipSet(live...)
+
+ if e.index.Contains(key) != current {
+ retry = true
+
+ continue
+ }
+
+ if current.Len() == next.Len() && current.ContainsAll(live) {
+ } else if _, ok := op.cas(key, current, next); !ok {
+ if e.index.Closed() {
+ return nil, false, errors.Join(err, errors.New("engine closed"))
+ }
+
+ retry = true
+
+ continue
+ } else {
e.notifyKeyChanged(key)
+ }
+ // Notification ingestion records processing only after publication,
+ // under the same shard as before. Cloud retains its existing policy.
+ if !failOnError && complete && len(live) == 1 && e.effectCache != nil {
+ tip := live[0]
+ eff, _ := e.effectCache.Get(tip)
+ e.recordHistoryProcessed(tip, eff)
+ }
- return
+ return live, complete, err
+ }
+}
+
+// closureVisit keeps the per-invocation state of an identity in one place.
+// Indices, rather than pointers into visits, survive table and slice growth.
+// Cached vertices are only navigation hints: every reuse checks reclamation.
+type closureVisit struct {
+ tip Tip
+ vertex *vertex
+ owner *closureCut
+ flags uint8
+}
+
+const (
+ closureScheduled uint8 = 1 << iota
+ closureExpanded
+ closureNoFetch
+ closureLive
+)
+
+// Cuts express connected membership, never dominance or downward expansion.
+type closureCut struct {
+ parent *closureCut
+ first Tip
+ ranges map[uint64][2]uint64
+ size int
+}
+
+// The table stores index+1, with zero denoting an empty slot. Keeping identity
+// lookup separate from the dense state array makes queue processing hash-free.
+// A randomized hash avoids depending on origin/offset distributions.
+type closureWork struct {
+ seed maphash.Seed
+ slots []int
+ visits []closureVisit
+ queue []int
+ pending map[int][]Tip
+ failures map[int]error
+}
+
+var closureWorkPool = sync.Pool{New: func() any {
+ return &closureWork{seed: maphash.MakeSeed(), slots: make([]int, 128)}
+}}
+
+func (w *closureWork) grow() {
+ w.slots = make([]int, len(w.slots)*2)
+
+ mask := uint64(len(w.slots) - 1)
+ for i := range w.visits {
+ slot := maphash.Comparable(w.seed, w.visits[i].tip) & mask
+ for w.slots[slot] != 0 {
+ slot = (slot + 1) & mask
+ }
+
+ w.slots[slot] = i + 1
+ }
+}
+
+func (w *closureWork) intern(tip Tip) int {
+ if (len(w.visits)+1)*4 >= len(w.slots)*3 {
+ w.grow()
+ }
+
+ mask := uint64(len(w.slots) - 1)
+
+ slot := maphash.Comparable(w.seed, tip) & mask
+ for {
+ entry := w.slots[slot]
+ if entry == 0 {
+ i := len(w.visits)
+ w.visits = append(w.visits, closureVisit{tip: tip})
+ w.slots[slot] = i + 1
+
+ return i
+ }
+
+ if w.visits[entry-1].tip == tip {
+ return entry - 1
+ }
+
+ slot = (slot + 1) & mask
+ }
+}
+
+func (w *closureWork) release() {
+ // No graph, key, cut, or error references survive the invocation. Large
+ // exceptional walks are discarded rather than retaining their peak size.
+ if cap(w.visits) > 32768 || cap(w.queue) > 65536 {
+ return
+ }
+
+ clear(w.visits)
+ clear(w.pending)
+ clear(w.failures)
+ clear(w.slots)
+ w.visits = w.visits[:0]
+ w.queue = w.queue[:0]
+ closureWorkPool.Put(w)
+}
+
+// walk extends this operation's per-key cuts under the key's closure shard.
+// New seeds add work; established consumption and expanded ancestry survive CAS
+// retries. Cross-key branches are queued rather than taking a second shard.
+func (op *closureOperation) walk(key string, supplied []Tip, visitBind func(Tip, *pb.TransactionalBindEffect)) []Tip {
+ e, attempted := op.engine, op.attempted
+
+ w := op.work[key]
+ if w == nil {
+ w = closureWorkPool.Get().(*closureWork)
+ if w.pending == nil {
+ w.pending = make(map[int][]Tip)
+ }
+
+ op.work[key] = w
+ }
+
+ pending := w.pending
+ resident := func(i int) *vertex {
+ v := w.visits[i].vertex
+ if v != nil && v.refs.Load() >= 0 {
+ return v
+ }
+
+ if e.effectCache == nil {
+ return nil
+ }
+
+ v = e.effectCache.getVertex(w.visits[i].tip)
+ w.visits[i].vertex = v
+
+ return v
+ }
+ // Comparison is rare (seeds, cut joins, and fetch scheduling), and uses
+ // current pool residency just as publication-order selection always has.
+ compareVertex := func(tip Tip) *vertex {
+ if e.effectCache == nil {
+ return nil
+ }
+
+ return e.effectCache.getVertex(tip)
+ }
+ compare := func(a, b Tip) int {
+ var ah, bh []byte
+ if v := compareVertex(a); v != nil {
+ ah = v.eff.GetForkChoiceHash()
+ }
+
+ if v := compareVertex(b); v != nil {
+ bh = v.eff.GetForkChoiceHash()
+ }
+
+ if order := bytes.Compare(ah, bh); order != 0 {
+ return order
+ }
+
+ if order := cmp.Compare(a[0], b[0]); order != 0 {
+ return order
}
- if e.index.Closed() {
+ return cmp.Compare(a[1], b[1])
+ }
+ result := func() []Tip {
+ var tips []Tip
+
+ for i := range w.visits {
+ if w.visits[i].flags&closureLive != 0 {
+ tips = append(tips, w.visits[i].tip)
+ }
+ }
+
+ slices.SortFunc(tips, compare)
+
+ return tips
+ }
+ root := func(c *closureCut) *closureCut {
+ for c.parent != nil {
+ if c.parent.parent != nil {
+ c.parent = c.parent.parent
+ }
+
+ c = c.parent
+ }
+
+ return c
+ }
+ observe := func(c *closureCut, tip Tip) {
+ r, ok := c.ranges[tip[0]]
+ if !ok {
+ r = [2]uint64{tip[1], tip[1]}
+ } else {
+ r[0], r[1] = min(r[0], tip[1]), max(r[1], tip[1])
+ }
+
+ c.ranges[tip[0]] = r
+ }
+ join := func(a, b *closureCut) {
+ a, b = root(a), root(b)
+ if a == b {
return
}
+
+ if a.size < b.size {
+ a, b = b, a
+ }
+
+ b.parent = a
+
+ a.size += b.size
+ if compare(b.first, a.first) < 0 {
+ a.first = b.first
+ }
+
+ for node, r := range b.ranges {
+ observe(a, Tip{node, r[0]})
+ observe(a, Tip{node, r[1]})
+ }
+ }
+ claim := func(i int, c *closureCut) {
+ c = root(c)
+ w.visits[i].owner = c
+ c.size++
+ observe(c, w.visits[i].tip)
+ }
+ enqueue := func(i int) {
+ if w.visits[i].flags&closureScheduled == 0 {
+ w.visits[i].flags |= closureScheduled
+ w.queue = append(w.queue, i)
+ }
+ }
+
+ // 1. Supplied tips seed downward discovery in fork-choice order. They do
+ // not initiate upward searches, including when an input is already local.
+ seeds := keytrie.NewTipSet(supplied...).Tips()
+ slices.SortFunc(seeds, compare)
+
+ for _, tip := range seeds {
+ i := w.intern(tip)
+ if w.visits[i].owner != nil {
+ // In particular, do not make a previously consumed seed live.
+ // An unresolved identity may have become resident in the meantime.
+ if w.visits[i].flags&closureExpanded == 0 {
+ w.queue = append(w.queue, i)
+ }
+
+ continue
+ }
+
+ c := &closureCut{first: tip, ranges: make(map[uint64][2]uint64)}
+ claim(i, c)
+ w.visits[i].flags |= closureLive
+ enqueue(i)
+ }
+
+ // 2. A preexisting reverse link identifies established history. Follow
+ // one chain to its cut BEFORE installing the newly discovered incoming
+ // edge. Record membership along the chain so another join stops there.
+ // This never consumes tips and never marks a vertex expanded downward.
+ ascend := func(v *vertex, c *closureCut) {
+ for {
+ parent, ok := v.firstParent(key)
+ if !ok {
+ return // the downward edge is already consuming this dependency
+ }
+
+ i := w.intern(parent)
+ if known := w.visits[i].owner; known != nil {
+ join(c, known)
+
+ return
+ }
+
+ claim(i, c)
+
+ v = resident(i)
+ if v == nil {
+ // The retained link proves a join to a reclaimed cut. The
+ // missing parent is neither a new tip nor a fetch obligation.
+ return
+ }
+
+ if _, hasParent := v.firstParent(key); !hasParent {
+ w.visits[i].flags |= closureLive
+
+ c = root(c)
+ if compare(parent, c.first) < 0 {
+ c.first = parent
+ }
+
+ enqueue(i)
+
+ return
+ }
+ }
+ }
+
+ for {
+ // 3. Drain resident downward work before requesting any boundary.
+ for next := 0; next < len(w.queue); next++ {
+ i := w.queue[next]
+
+ tip := w.visits[i].tip
+ if w.visits[i].flags&closureExpanded != 0 {
+ continue
+ }
+
+ v := resident(i)
+ if v == nil {
+ if w.visits[i].flags&closureNoFetch != 0 {
+ delete(pending, i)
+ delete(w.failures, i)
+
+ continue
+ }
+
+ if _, exists := pending[i]; !exists {
+ pending[i] = nil
+ }
+
+ continue
+ }
+
+ if parents := pending[i]; len(parents) != 0 {
+ // A fetched dependency may now connect to established local
+ // history. Its incoming edges must not mask those older links.
+ ascend(v, w.visits[i].owner)
+
+ for _, parent := range parents {
+ e.effectCache.addParent(v, parent, key)
+ }
+ }
+
+ delete(pending, i)
+ delete(w.failures, i)
+
+ w.visits[i].flags |= closureExpanded
+
+ if bind := v.eff.GetTxnBind(); bind != nil {
+ visitBind(tip, bind)
+ }
+
+ if v.root() {
+ w.visits[i].flags |= closureNoFetch
+ }
+
+ if op.observe != nil {
+ op.observe(tip, nil)
+ }
+
+ for ref := range keyDeps(key, v.eff) {
+ dep := r(ref)
+ if dep == (Tip{}) || dep == tip {
+ continue
+ }
+
+ if op.observe != nil {
+ op.observe(tip, ref)
+ }
+
+ j := w.intern(dep)
+ // Only this genuine downward edge establishes consumption.
+ w.visits[j].flags &^= closureLive
+
+ if w.visits[i].flags&closureNoFetch != 0 && w.visits[j].flags&closureNoFetch == 0 {
+ w.visits[j].flags |= closureNoFetch
+ // Newly discovered boundary evidence still propagates
+ // through ancestry that another path expanded earlier.
+ if w.visits[j].flags&closureExpanded != 0 {
+ w.visits[j].flags &^= closureExpanded
+ w.queue = append(w.queue, j)
+ }
+ }
+
+ dv := resident(j)
+ if known := w.visits[j].owner; known != nil {
+ join(w.visits[i].owner, known)
+ } else {
+ claim(j, w.visits[i].owner)
+
+ if dv != nil {
+ ascend(dv, w.visits[j].owner)
+ }
+ }
+
+ if dv != nil {
+ e.effectCache.addParent(dv, tip, key)
+ } else if w.visits[j].flags&closureExpanded == 0 {
+ pending[j] = append(pending[j], tip)
+ }
+
+ enqueue(j)
+ }
+ }
+
+ w.queue = w.queue[:0]
+
+ if len(pending) == 0 {
+ return result()
+ }
+
+ // 4. Resolve each boundary once. Exhaustion and failed availability
+ // are different outcomes even though neither produces runnable work.
+ var roots []int
+
+ for i := range pending {
+ tip := w.visits[i].tip
+ if resident(i) != nil {
+ w.queue = append(w.queue, i)
+
+ continue
+ }
+
+ if w.visits[i].flags&closureNoFetch != 0 {
+ delete(pending, i)
+ delete(w.failures, i)
+
+ continue
+ }
+
+ if err, tried := attempted[tip]; tried {
+ if err == nil {
+ // A successful load lost during a publication retry is
+ // not a definitive response from the history holders.
+ err = fmt.Errorf("effect %v reclaimed during frontier closure", tip)
+ }
+
+ if !errors.Is(err, ErrEffectNotFound) && !errors.Is(err, ErrCDNBlobMissing) {
+ if w.failures == nil {
+ w.failures = make(map[int]error)
+ }
+
+ w.failures[i] = err
+ } else {
+ delete(pending, i)
+ delete(w.failures, i)
+ }
+ } else {
+ roots = append(roots, i)
+ }
+ }
+
+ if len(w.queue) != 0 {
+ continue
+ }
+
+ if len(roots) == 0 {
+ return result()
+ }
+
+ // 5. Counter ranges prioritize cuts; they never establish dominance.
+ // Pairwise range preferences need not be transitive, so use a stable
+ // scan rather than passing them to a sorting comparator.
+ slices.SortFunc(roots, func(a, b int) int { return compare(w.visits[a].tip, w.visits[b].tip) })
+
+ best := roots[0]
+ for _, candidate := range roots[1:] {
+ a, b := root(w.visits[candidate].owner), root(w.visits[best].owner)
+ newer, older := false, false
+
+ for node, ar := range a.ranges {
+ if br, ok := b.ranges[node]; ok {
+ newer = newer || ar[0] > br[1]
+ older = older || ar[1] < br[0]
+ }
+ }
+
+ if newer && !older || newer == older && compare(a.first, b.first) < 0 {
+ best = candidate
+ }
+ }
+
+ missing, err := e.fetchFrontierAncestry(key, w.visits[best].tip)
+ for _, tip := range missing {
+ attempted[tip] = ErrEffectNotFound
+ }
+
+ if err == nil && resident(best) == nil {
+ if absent := attempted[w.visits[best].tip]; errors.Is(absent, ErrEffectNotFound) {
+ err = absent
+ } else {
+ err = fmt.Errorf("effect %v reclaimed during frontier closure", w.visits[best].tip)
+ }
+ }
+
+ attempted[w.visits[best].tip] = err
+ if err == nil {
+ w.queue = append(w.queue, best)
+ }
}
}
diff --git a/effects/frontier_admission_test.go b/effects/frontier_admission_test.go
new file mode 100644
index 0000000..d1346ca
--- /dev/null
+++ b/effects/frontier_admission_test.go
@@ -0,0 +1,94 @@
+package effects
+
+import (
+ "fmt"
+ "testing"
+ "time"
+)
+
+func TestClosureAdmissionSerializesShardAcrossEntryPoints(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ key, collision, independent := "k", "", ""
+ for i := 0; collision == "" || independent == ""; i++ {
+ candidate := fmt.Sprintf("key-%d", i)
+ if e.closureLock(candidate) == e.closureLock(key) {
+ collision = candidate
+ } else {
+ independent = candidate
+ }
+ }
+
+ entered := make(chan Tip, 4)
+
+ release := make(chan struct{})
+ defer close(release)
+
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ entered <- tip
+
+ if tip == (Tip{7, 1}) {
+ <-release
+ }
+
+ return nil, ErrEffectNotFound
+ }}
+
+ done := make(chan struct{}, 4)
+
+ go func() {
+ e.installIngestedTips(key, Tip{7, 1})
+
+ done <- struct{}{}
+ }()
+
+ select {
+ case <-entered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("first closure did not start")
+ }
+
+ // Recovery of the same key and ingestion of a colliding key both wait.
+ go func() {
+ _, _, _ = e.closeIngestedFrontier(key, []Tip{{8, 1}}, make(map[Tip]error))
+
+ done <- struct{}{}
+ }()
+ go func() {
+ e.installIngestedTips(collision, Tip{9, 1})
+
+ done <- struct{}{}
+ }()
+ go func() {
+ e.installIngestedTips(independent, Tip{10, 1})
+
+ done <- struct{}{}
+ }()
+
+ select {
+ case tip := <-entered:
+ if tip != (Tip{10, 1}) {
+ t.Fatalf("same-shard walk entered concurrently: %v", tip)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("independent shard could not progress")
+ }
+
+ select {
+ case tip := <-entered:
+ t.Fatalf("same-shard walk entered concurrently: %v", tip)
+ case <-time.After(20 * time.Millisecond):
+ }
+
+ // Release by sending so the deferred close also unblocks on failure.
+ release <- struct{}{}
+
+ for range 4 {
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("closure did not finish after admission")
+ }
+ }
+}
diff --git a/effects/frontier_cut_test.go b/effects/frontier_cut_test.go
new file mode 100644
index 0000000..f60c0d8
--- /dev/null
+++ b/effects/frontier_cut_test.go
@@ -0,0 +1,583 @@
+package effects
+
+import (
+ "errors"
+ "fmt"
+ "slices"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/keytrie"
+)
+
+type cutBroadcaster struct {
+ mockBroadcaster
+
+ load func(Tip) ([]byte, error)
+}
+
+func (b *cutBroadcaster) FetchFromAny(ref *pb.EffectRef, _ FetchHint) ([]byte, error) {
+ return b.load(r(ref))
+}
+
+func cutEffect(tip Tip, deps ...Tip) *pb.Effect {
+ return &pb.Effect{
+ Key: []byte("k"), NodeId: tip[0], Deps: toPbRefs(deps),
+ ForkChoiceHash: []byte{byte(tip[0]), byte(tip[1])},
+ Kind: &pb.Effect_Data{Data: scalarInsertInt(pb.MergeRule_ADDITIVE_INT, 1)},
+ }
+}
+
+func putCutEffects(t *testing.T, e *Engine, effects map[Tip]*pb.Effect) {
+ t.Helper()
+
+ for tip, eff := range effects {
+ if err := e.effectCache.Put(tip, eff); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+func requireCutTips(t *testing.T, got, want []Tip) {
+ t.Helper()
+
+ set := keytrie.NewTipSet(got...)
+ if len(got) != len(want) || !set.ContainsAll(want) {
+ t.Fatalf("frontier = %v, want %v", got, want)
+ }
+}
+
+func TestFrontierCutDominanceStillResolvesBoundary(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ old, middle, newest, gone := Tip{7, 1}, Tip{8, 1}, Tip{9, 1}, Tip{6, 1}
+ putCutEffects(t, e, map[Tip]*pb.Effect{
+ old: cutEffect(old, gone), middle: cutEffect(middle, old), newest: cutEffect(newest, middle),
+ })
+
+ transport := errors.New("peer unavailable")
+ for _, failure := range []error{ErrEffectNotFound, ErrCDNBlobMissing, transport} {
+ loads := 0
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ loads++
+
+ if tip != gone {
+ t.Fatalf("unexpected boundary %v", tip)
+ }
+
+ return nil, failure
+ }}
+
+ for _, tips := range [][]Tip{{old, newest, middle}, {newest, middle, old}} {
+ got, complete, err := e.closeIngestedFrontier("k", tips, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{newest})
+
+ if complete != (failure != transport) || errors.Is(err, transport) != (failure == transport) {
+ t.Fatalf("boundary %v: complete=%t error=%v", failure, complete, err)
+ }
+ }
+
+ if loads != 2 {
+ t.Fatalf("boundary %v fetched %d times, want once per closure", failure, loads)
+ }
+ }
+}
+
+func TestFrontierCutSingleTipLoadsRequiredHistory(t *testing.T) {
+ for _, remote := range []bool{false, true} {
+ t.Run(fmt.Sprintf("remote=%t", remote), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ preSubscribe(e, "k")
+
+ loads := make(map[Tip]int)
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ loads[tip]++
+ switch tip {
+ case (Tip{7, 2}):
+ return wireEffect(frontierEffect(2, 1)), nil
+ case (Tip{7, 1}):
+ return wireEffect(frontierEffect(1)), nil
+ default:
+ t.Fatalf("unexpected fetch %v", tip)
+
+ return nil, ErrEffectNotFound
+ }
+ }}
+ notify := &pb.OffsetNotify{Origin: toPbRef(Tip{7, 3}), EffectData: wireEffect(frontierEffect(3, 2))}
+
+ var err error
+ if remote {
+ _, err = e.HandleRemote(notify)
+ } else {
+ err = e.handleBackfill(notify)
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ requireFrontier(t, e, Tip{7, 3})
+
+ for _, tip := range []Tip{{7, 1}, {7, 2}} {
+ if _, resident := e.effectCache.Get(tip); !resident || loads[tip] != 1 {
+ t.Fatalf("after ingestion: dependency %v resident=%t loads=%d", tip, resident, loads[tip])
+ }
+ }
+
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ t.Fatalf("reconstruction fetched %v after ingestion", tip)
+
+ return nil, ErrEffectNotFound
+ }}
+
+ state, _, err := e.reconstruct("k", []Tip{{7, 3}}, "", false)
+ if err != nil || state.GetScalar().GetIntVal() != 3 {
+ t.Fatalf("reconstruction: state=%v error=%v", state, err)
+ }
+ })
+ }
+}
+
+func TestFrontierCutDiscoversUnrelatedRemoteHistories(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ a, b := Tip{7, 3}, Tip{8, 3}
+ putCutEffects(t, e, map[Tip]*pb.Effect{
+ a: cutEffect(a, Tip{7, 2}), b: cutEffect(b, Tip{8, 2}),
+ })
+
+ fetched := make(map[Tip]int)
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ fetched[tip]++
+ if tip[1] == 0 {
+ return nil, ErrEffectNotFound
+ }
+
+ return wireEffect(cutEffect(tip, Tip{tip[0], tip[1] - 1})), nil
+ }}
+
+ got, complete, err := e.closeIngestedFrontier("k", []Tip{b, a}, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{a, b})
+
+ if !complete || err != nil {
+ t.Fatalf("exhausted separate regions: complete=%t error=%v", complete, err)
+ }
+
+ for _, node := range []uint64{7, 8} {
+ for id := range uint64(3) {
+ if got := fetched[Tip{node, id}]; got != 1 {
+ t.Fatalf("boundary %v fetched %d times, want once", Tip{node, id}, got)
+ }
+ }
+ }
+}
+
+func TestFrontierCutResidentPhasePrecedesRemoteLoads(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ a, b, x, y := Tip{7, 3}, Tip{8, 3}, Tip{7, 2}, Tip{8, 2}
+ putCutEffects(t, e, map[Tip]*pb.Effect{
+ a: cutEffect(a, x), x: cutEffect(x, Tip{7, 1}),
+ b: cutEffect(b, y), y: cutEffect(y, Tip{8, 1}),
+ })
+ e.broadcaster = &cutBroadcaster{load: func(Tip) ([]byte, error) {
+ for dep, parent := range map[Tip]Tip{x: a, y: b} {
+ v := e.effectCache.getVertex(dep)
+ if got, ok := v.firstParent("k"); !ok || got != parent {
+ t.Fatalf("remote load began before resident edge %v -> %v was visited", parent, dep)
+ }
+ }
+
+ return nil, ErrEffectNotFound
+ }}
+ e.installIngestedTips("k", a, b)
+ requireFrontier(t, e, a, b)
+}
+
+func TestFrontierCutDownwardEncounterDiscoversEstablishedCut(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ x, middle, established, separate := Tip{7, 1}, Tip{7, 2}, Tip{7, 3}, Tip{9, 1}
+ putCutEffects(t, e, map[Tip]*pb.Effect{
+ x: cutEffect(x), middle: cutEffect(middle, x),
+ established: cutEffect(established, middle), separate: cutEffect(separate),
+ })
+ // Reverse links arise by establishing this region, not by fabricating
+ // arbitrary upward edges in the fixture.
+ e.installIngestedTips("k", established, separate)
+ requireFrontier(t, e, established, separate)
+
+ incoming, gap := Tip{8, 2}, Tip{8, 1}
+ putCutEffects(t, e, map[Tip]*pb.Effect{incoming: cutEffect(incoming, gap)})
+
+ loads := 0
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ loads++
+
+ if tip != gap {
+ t.Fatalf("unexpected fetch %v", tip)
+ }
+
+ return wireEffect(cutEffect(gap, x)), nil
+ }}
+
+ // Reconcile the supplied unknown region. Reaching X downward discovers
+ // the previously established tip through its resident reverse chain.
+ got, complete, err := e.closeIngestedFrontier("k", []Tip{incoming, separate}, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{incoming, established, separate})
+
+ if !complete || err != nil || loads != 1 {
+ t.Fatalf("joined history: complete=%t error=%v loads=%d", complete, err, loads)
+ }
+}
+
+func TestFrontierCutReclaimedParentIsAnEstablishedJoin(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ x, middle, old, current, separate := Tip{7, 1}, Tip{7, 2}, Tip{7, 3}, Tip{7, 4}, Tip{9, 1}
+ putCutEffects(t, e, map[Tip]*pb.Effect{
+ x: cutEffect(x), middle: cutEffect(middle, x), old: cutEffect(old, middle),
+ separate: cutEffect(separate),
+ })
+ e.installIngestedTips("k", old, separate)
+ putCutEffects(t, e, map[Tip]*pb.Effect{current: cutEffect(current, old)})
+ e.installIngestedTips("k", current)
+ requireFrontier(t, e, current, separate)
+
+ // Retire payloads from the established region while X survives. Its
+ // reverse link remains evidence; it does not own the retired parent.
+ for _, tip := range []Tip{middle, old} {
+ e.effectCache.decref(tip)
+ }
+
+ e.effectCache.drainReclaim()
+
+ incoming := Tip{8, 1}
+ putCutEffects(t, e, map[Tip]*pb.Effect{incoming: cutEffect(incoming, x)})
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ if tip != old {
+ t.Fatalf("reverse navigation fetched reclaimed parent %v", tip)
+ }
+
+ return nil, ErrEffectNotFound
+ }}
+
+ got, complete, err := e.closeIngestedFrontier("k", []Tip{incoming, current, separate}, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{incoming, current, separate})
+
+ if !complete || err != nil {
+ t.Fatalf("reclaimed join treated as unresolved: complete=%t error=%v", complete, err)
+ }
+}
+
+func TestFrontierCutSharedBindNavigationStaysOnItsKey(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ kRoot, otherRoot, bind := Tip{7, 1}, Tip{8, 1}, Tip{9, 1}
+ kTip, otherTip, separate := Tip{7, 2}, Tip{8, 2}, Tip{10, 1}
+ shared := cutEffect(bind)
+ shared.Kind = &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{
+ Keys: []*pb.TransactionalBindEffect_KeyBind{
+ {Key: []byte("k"), NewTip: toPbRef(kRoot)},
+ {Key: []byte("other"), NewTip: toPbRef(otherRoot)},
+ },
+ }}
+ otherBase, otherParent := cutEffect(otherRoot), cutEffect(otherTip, bind)
+ otherBase.Key, otherParent.Key = []byte("other"), []byte("other")
+ putCutEffects(t, e, map[Tip]*pb.Effect{
+ kRoot: cutEffect(kRoot), otherRoot: otherBase, bind: shared,
+ kTip: cutEffect(kTip, bind), otherTip: otherParent, separate: cutEffect(separate),
+ })
+ // Establish the other key first, so its reverse link is stored first.
+ e.installIngestedTips("other", otherTip, otherRoot)
+ e.installIngestedTips("k", kTip, separate)
+ e.effectCache.decref(otherTip)
+ e.effectCache.drainReclaim()
+
+ incoming := Tip{11, 1}
+ putCutEffects(t, e, map[Tip]*pb.Effect{incoming: cutEffect(incoming, bind)})
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ t.Fatalf("key-scoped resident closure unexpectedly fetched %v", tip)
+
+ return nil, ErrEffectNotFound
+ }}
+ got, complete, err := e.closeIngestedFrontier("k", []Tip{incoming, separate}, make(map[Tip]error))
+ requireCutTips(t, got, []Tip{incoming, kTip, separate})
+
+ if !complete || err != nil {
+ t.Fatalf("shared-bind closure: complete=%t error=%v", complete, err)
+ }
+}
+
+func TestFrontierCutSnapshotLabelsPreserveDirectedClosure(t *testing.T) {
+ for mask := range 8 {
+ for _, reverse := range []bool{false, true} {
+ t.Run(fmt.Sprintf("snapshots=%03b/reverse=%t", mask, reverse), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ x, a, middle, b := Tip{7, 1}, Tip{8, 1}, Tip{9, 1}, Tip{9, 2}
+
+ nodes := map[Tip]*pb.Effect{
+ x: cutEffect(x), a: cutEffect(a, x), middle: cutEffect(middle, x), b: cutEffect(b, middle),
+ }
+ for bit, tip := range []Tip{x, middle, b} {
+ if mask&(1< 30*length || loads != attempt+1 {
+ t.Fatalf("attempt=%d lookups=%d loads=%d", attempt, after-before, loads)
+ }
+ }
+}
+
+func TestFrontierCutIngestionMatchesReachability(t *testing.T) {
+ // Each graph is ingested in two phases. The first publication establishes
+ // reverse links naturally; the second reconciles another advertised region.
+ // The oracle uses only immutable forward edges and candidate reachability.
+ const n = 5
+ for graph := range 1 << (n * (n - 1) / 2) {
+ var (
+ reach [n][n]bool
+ deps [n][]Tip
+ tips [n]Tip
+ )
+
+ bit := 0
+
+ for i := range n {
+ // Distinct origins avoid asserting any cross-vertex ordering from
+ // one origin's monotonic counters. Only the edges prove ancestry.
+ tips[i] = Tip{uint64(i + 7), 1}
+ for j := range i {
+ if graph&(1< %v traversed %d times, want 1", tip, r(ref), got)
+ }
+ }
+ }
+}
+
+func TestClosurePublicationCASRetriesReuseDiscovery(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ base, middle, incoming := Tip{7, 1}, Tip{7, 2}, Tip{8, 1}
+ records := map[Tip]*pb.Effect{
+ base: cutEffect(base), middle: cutEffect(middle, base), incoming: cutEffect(incoming, middle),
+ }
+ putCutEffects(t, e, records)
+ e.installTips("k", []Tip{base})
+
+ op := newClosureOperation(e, make(map[Tip]error))
+ defer op.release()
+
+ vertices, edges := countClosureWork(op)
+
+ const failures = 8
+
+ attempts := 0
+
+ op.cas = func(key string, old, next *keytrie.TipSet) (*keytrie.TipSet, bool) {
+ attempts++
+ if attempts <= failures {
+ // Replace the pointer without changing causal history. Force the
+ // actual CAS to fail after closure, without scheduler timing.
+ if _, ok := e.index.Insert(key, old, keytrie.NewTipSet(old.Tips()...)); !ok {
+ t.Fatal("could not force competing publication")
+ }
+ }
+
+ return e.index.Insert(key, old, next)
+ }
+ if _, complete, err := op.publish("k", []Tip{incoming}, false); err != nil || !complete {
+ t.Fatalf("publication: complete=%t err=%v", complete, err)
+ }
+
+ if attempts != failures+1 {
+ t.Fatalf("CAS attempts = %d, want %d", attempts, failures+1)
+ }
+
+ requireFrontier(t, e, incoming)
+ requireClosureWorkOnce(t, records, vertices, edges)
+}
+
+func TestClosurePublicationConcurrentTipExtendsCuts(t *testing.T) {
+ for _, consumesIncoming := range []bool{false, true} {
+ name := "concurrent sibling"
+ if consumesIncoming {
+ name = "consumes discovered tip"
+ }
+
+ t.Run(name, func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ defer func() { _ = e.Close() }()
+
+ base, left, incoming, right, concurrent := Tip{7, 1}, Tip{8, 1}, Tip{8, 2}, Tip{9, 1}, Tip{9, 2}
+
+ rightDep := base
+ if consumesIncoming {
+ rightDep = incoming
+ }
+
+ records := map[Tip]*pb.Effect{
+ base: cutEffect(base), left: cutEffect(left, base), incoming: cutEffect(incoming, left),
+ right: cutEffect(right, rightDep), concurrent: cutEffect(concurrent, right),
+ }
+ putCutEffects(t, e, records)
+ e.installTips("k", []Tip{base})
+
+ op := newClosureOperation(e, make(map[Tip]error))
+ defer op.release()
+
+ vertices, edges := countClosureWork(op)
+ attempts := 0
+
+ op.cas = func(key string, old, next *keytrie.TipSet) (*keytrie.TipSet, bool) {
+ attempts++
+ if attempts == 1 {
+ // This local publication consumes the former indexed tip.
+ // Its branch was not supplied to the initial closure walk.
+ e.updateIndex(key, old, concurrent)
+ }
+
+ return e.index.Insert(key, old, next)
+ }
+ if _, complete, err := op.publish("k", []Tip{incoming}, false); err != nil || !complete {
+ t.Fatalf("publication: complete=%t err=%v", complete, err)
+ }
+
+ if consumesIncoming {
+ requireFrontier(t, e, concurrent)
+ } else {
+ requireFrontier(t, e, incoming, concurrent)
+ }
+
+ wantAttempts := 2
+ if consumesIncoming {
+ // The competing publication already installed the final set.
+ wantAttempts = 1
+ }
+
+ if attempts != wantAttempts {
+ t.Fatalf("CAS attempts = %d, want %d", attempts, wantAttempts)
+ }
+
+ requireClosureWorkOnce(t, records, vertices, edges)
+ })
+ }
+}
+
+// installTips merges a whole tip frontier into the key's index in one CAS
+// transition, so a concurrent reader observes either the prior frontier or the
+// full merged one — never a partial install. Same accounting as updateIndex:
+// Insert fires the trie's refDelta hook for the diff, and NewTipSet dedupes
+// tips the index already holds.
+//
+//nolint:unparam // Keep the fixture helper usable for any key; current callers use "k".
+func (e *Engine) installTips(key string, tips []Tip) {
+ for attempt := 0; ; attempt++ {
+ if attempt > 0 {
+ slog.Debug("installTips: CAS retry", "key", key, "attempt", attempt)
+ }
+
+ current := e.index.Contains(key)
+ cur := current.Tips()
+ offsets := make([]Tip, 0, len(cur)+len(tips))
+ offsets = append(offsets, cur...)
+ offsets = append(offsets, tips...)
+
+ newTips := keytrie.NewTipSet(offsets...)
+ if _, ok := e.index.Insert(key, current, newTips); ok {
+ if current.Len() != newTips.Len() || !current.ContainsAll(newTips.Tips()) {
+ e.notifyKeyChanged(key)
+ }
+
+ return
+ }
+
+ if e.index.Closed() {
+ // Not a CAS race: Insert fails unconditionally on a closed trie,
+ // so retrying would spin forever during shutdown.
+ return
+ }
+ }
+}
diff --git a/effects/historical.go b/effects/historical.go
index 248a41a..761737c 100644
--- a/effects/historical.go
+++ b/effects/historical.go
@@ -93,14 +93,15 @@ func (e *Engine) historicalFrontier(ctx context.Context, key string, tips []Tip,
}
// historicalReadView applies the existing bind closure and fork choice to
-// selected historical frontiers. The caller supplies the same fixed cut for
-// every key. It must establish completeness before exposing interactive reads;
+// selected historical frontiers, retaining live competition as evidence only.
+// The caller supplies the same fixed cut for every key. It must establish
+// completeness before exposing interactive reads;
// this helper does not claim that currently available tips are complete.
-func (e *Engine) historicalReadView(ctx context.Context, keys []string, frontier func(string) ([]Tip, error)) (*snapshotReadView, error) {
+func (e *Engine) historicalReadView(ctx context.Context, keys []string, frontier func(string) ([]Tip, error), live *snapshotReadView) (*snapshotReadView, error) {
v := newSnapshotReadView(e, ctx)
v.historical = true
- err := v.discover(keys, func(key string) ([]Tip, error) {
+ err := v.discoverWithEvidence(keys, func(key string) ([]Tip, error) {
if captured, ok := v.frontiers[key]; ok {
return captured.resolved, nil
}
@@ -111,7 +112,7 @@ func (e *Engine) historicalReadView(ctx context.Context, keys []string, frontier
}
return tips, err
- })
+ }, live)
if err != nil {
return nil, err
}
diff --git a/effects/historical_evidence_test.go b/effects/historical_evidence_test.go
new file mode 100644
index 0000000..2edbae9
--- /dev/null
+++ b/effects/historical_evidence_test.go
@@ -0,0 +1,253 @@
+package effects
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "testing"
+ "time"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/keytrie"
+ "github.com/swytchdb/engine/stn"
+)
+
+// A retained proof can account for a bind without retaining the old value
+// ancestry. Cached labels and proofs outside the frozen cut cannot do so.
+func TestHistoricalMembershipConsultsFrozenEvidence(t *testing.T) {
+ for _, evidence := range []string{"frozen", "cached-label", "live-only"} {
+ t.Run(evidence, func(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+
+ t.Cleanup(func() { _ = e.Close() })
+
+ bind := closureBind(log, "old", 1, []string{"a", "b"}, nil)
+
+ proof, err := e.bindProof(bind, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ owner := log.putEffect(bindEvidenceSnapshot("a", proof))
+ e.effectCache.releaseEvidence(evidenceTips(proof))
+ ownerEffect, _ := e.effectCache.Get(owner)
+
+ verdicts, err := e.proofVerdicts(owner, ownerEffect)
+ if err != nil || !verdicts["old"].proven {
+ t.Fatalf("expected evaluated proof: %v, %v", verdicts, err)
+ }
+
+ // Value reconstruction stops at this state snapshot. The missing
+ // connector models the already reclaimed path to the old bind.
+ missing := Tip{99, 99}
+ state := log.putEffect(&pb.Effect{
+ Key: []byte("a"), Deps: toPbRefs([]Tip{missing}),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{}}},
+ })
+ frontier := state
+
+ if evidence != "live-only" {
+ // Make the proof source part of the selected value history.
+ snapshot := bindEvidenceSnapshot("a", proof)
+ snapshot.Deps = toPbRefs([]Tip{state})
+ owner = log.putEffect(snapshot)
+ ownerEffect, _ = e.effectCache.Get(owner)
+
+ verdicts, err = e.proofVerdicts(owner, ownerEffect)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ frontier = owner
+ }
+
+ if evidence == "cached-label" {
+ entry := verdicts["old"]
+ entry.proven = false
+ verdicts["old"] = entry
+ }
+
+ view := newSnapshotReadView(e, context.Background())
+ view.frontiers["a"] = snapshotFrontier{resolved: []Tip{frontier}}
+ view.verdicts = verdicts
+ s := &transactionSnapshot{
+ engine: e, views: map[string]*snapshotReadView{"a": view},
+ decided: make(map[Tip]bool),
+ }
+ eff, _ := e.effectCache.Get(bind)
+
+ included, err := s.member(context.Background(), bind, eff, make(map[Tip]bool))
+ if evidence == "frozen" {
+ if err != nil || !included {
+ t.Fatalf("retained proof reopened reclaimed history: included=%t error=%v", included, err)
+ }
+ // Evidence must not reverse an earlier exclusion.
+ s.decided[bind] = false
+ if included, err := s.member(context.Background(), bind, eff, make(map[Tip]bool)); err != nil || included {
+ t.Fatalf("proof reversed frozen membership: included=%t error=%v", included, err)
+ }
+ } else if !errors.Is(err, errEffectNotResident) {
+ t.Fatalf("unusable proof bypassed the history check: included=%t error=%v", included, err)
+ }
+ })
+ }
+}
+
+// Historical membership can include an ambiguously timed remote bind while
+// excluding its local competitor. Value selection must not discard the live
+// evidence that determines which transaction survives.
+func TestHistoricalReadRetainsLiveCompetition(t *testing.T) {
+ for _, chain := range []bool{false, true} {
+ for _, cached := range []bool{false, true} {
+ t.Run(fmt.Sprintf("chain=%t/cached=%t", chain, cached), func(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ preSubscribe(e, "winner")
+ preSubscribe(e, "other")
+
+ root := scalarSet("winner", "base")
+ root.NodeId, root.Hlc = 1, sTs(0)
+ base := log.putEffect(root)
+ log.nodeID = 2
+ a := closureBind(log, "A", 3, []string{"winner"}, map[string]Tip{"winner": base})
+
+ var b, excluded Tip
+
+ frontiers := make(map[string][]Tip)
+ verdicts := map[string]pb.Verdict{"A": pb.Verdict_LOST, "B": pb.Verdict_WON}
+ loser, wantValue, verdictKey := "A", "base", "winner"
+ boundary := int64(450)
+
+ if chain {
+ // A conflicts with B, B conflicts with C, A and C are
+ // disjoint. C's higher priority eliminates B, allowing A.
+ log.nodeID = 3
+ b = closureBind(log, "B", 2, []string{"winner", "other"}, map[string]Tip{"winner": base})
+ log.nodeID = 1
+ excluded = closureBind(log, "C", 1, []string{"other"}, nil)
+ frontiers["other"] = []Tip{b, excluded}
+ verdicts = map[string]pb.Verdict{"B": pb.Verdict_LOST, "C": pb.Verdict_WON}
+ loser, wantValue, verdictKey, boundary = "B", "A", "other", 750
+ } else {
+ log.nodeID = 1
+ b = closureBind(log, "B", 1, []string{"winner"}, map[string]Tip{"winner": base})
+ excluded = b
+ }
+
+ frontiers["winner"] = []Tip{a, b}
+ if cached {
+ v := log.putEffect(&pb.Effect{
+ Key: []byte(verdictKey), NodeId: 1, Hlc: sTs(int64(log.nextOff)),
+ Deps: toPbRefs(frontiers[verdictKey]),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{TxnVerdicts: verdicts}},
+ })
+ frontiers[verdictKey] = []Tip{v}
+ }
+
+ for key, tips := range frontiers {
+ e.index.Insert(key, nil, keytrie.NewTipSet(tips...))
+ }
+
+ s := &transactionSnapshot{
+ engine: e, at: stn.Sample{Domain: 1, Tick: boundary},
+ expiryAt: time.Now(), clocks: e.clockEvidence(),
+ }
+ if err := s.prepare(context.Background(), []string{"winner"}); err != nil {
+ t.Fatal(err)
+ }
+
+ if !s.decided[a] || s.decided[excluded] {
+ t.Fatalf("fixture must select A and exclude its local competitor: A=%v excluded=%v", s.decided[a], s.decided[excluded])
+ }
+
+ v := s.views["winner"]
+ if _, lost := v.lost[loser]; !lost || len(v.lost) != 1 {
+ t.Errorf("lost=%v, want only %s", v.lost, loser)
+ }
+
+ for key := range v.anchors {
+ if slices.Contains(v.frontiers[key].resolved, excluded) || slices.Contains(v.anchors[key], excluded) {
+ t.Errorf("live competitor leaked into historical value history on %q", key)
+ }
+ }
+
+ tips := slices.Concat(v.frontiers["winner"].resolved, v.anchors["winner"])
+
+ result, _, err := e.reconstructWithView("winner", tips, "reader", true, v)
+ if err != nil || string(result.GetScalar().GetRaw()) != wantValue {
+ t.Fatalf("value=%q error=%v, want %q", result.GetScalar().GetRaw(), err, wantValue)
+ }
+ })
+ }
+ }
+}
+
+// Equivalent live proof must not hide the source already present in the frozen
+// history: membership needs that source after value ancestry has been reclaimed.
+func TestHistoricalMembershipPreservesEquivalentFrozenProof(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+
+ t.Cleanup(func() { _ = e.Close() })
+
+ bind := closureBind(log, "old", 1, []string{"a", "b"}, nil)
+
+ proof, err := e.bindProof(bind, nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ frozenEffect := bindEvidenceSnapshot("a", proof)
+ frozenEffect.GetSnapshot().State = &pb.ReducedEffect{}
+ frozenEffect.Deps = toPbRefs([]Tip{{99, 99}}) // Reclaimed value ancestry.
+ frozen := log.putEffect(frozenEffect)
+ liveEffect := bindEvidenceSnapshot("a", proof)
+ liveEffect.GetSnapshot().State = &pb.ReducedEffect{}
+ liveEffect.Deps = toPbRefs([]Tip{frozen})
+ newer := log.putEffect(liveEffect)
+
+ e.effectCache.releaseEvidence(evidenceTips(proof))
+
+ e.updateIndex("a", nil, newer)
+
+ live := newSnapshotReadView(e, context.Background())
+ if err := live.discover([]string{"a"}, live.capture); err != nil {
+ t.Fatal(err)
+ }
+
+ if entry := live.verdicts["old"]; !entry.proven || entry.snapshotTip != newer {
+ t.Fatalf("live fixture must use newer proof: %+v", entry)
+ }
+
+ view, err := e.historicalReadView(context.Background(), []string{"a"}, func(key string) ([]Tip, error) {
+ if key != "a" {
+ t.Fatalf("unexpected value history requested: %s", key)
+ }
+
+ return []Tip{frozen}, nil
+ }, live)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if entry := view.verdicts["old"]; !entry.proven || entry.snapshotTip != frozen {
+ t.Errorf("equivalent live proof displaced frozen source: %+v, want %v", entry, frozen)
+ }
+
+ if entry := live.verdicts["old"]; entry.snapshotTip != newer {
+ t.Error("historical merge mutated the live view")
+ }
+
+ s := &transactionSnapshot{
+ engine: e, views: map[string]*snapshotReadView{"a": view},
+ decided: make(map[Tip]bool),
+ }
+ eff, _ := e.effectCache.Get(bind)
+
+ included, err := s.member(context.Background(), bind, eff, make(map[Tip]bool))
+ if err != nil || !included {
+ t.Fatalf("equivalent proof reopened reclaimed history: included=%t error=%v", included, err)
+ }
+}
diff --git a/effects/historical_test.go b/effects/historical_test.go
index f0580af..89e0e50 100644
--- a/effects/historical_test.go
+++ b/effects/historical_test.go
@@ -68,7 +68,7 @@ func historicalTestRead(t *testing.T, e *Engine, at time.Time, keys ...string) [
})
}
- v, err := e.historicalReadView(ctx, keys, frontier)
+ v, err := e.historicalReadView(ctx, keys, frontier, nil)
if err != nil {
t.Fatal(err)
}
diff --git a/effects/history_watermark.go b/effects/history_watermark.go
new file mode 100644
index 0000000..5905bc0
--- /dev/null
+++ b/effects/history_watermark.go
@@ -0,0 +1,214 @@
+package effects
+
+import (
+ "slices"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+// HistoryProgress describes a peer's own publications and one local effect
+// whose ancestry it processed. Processed is an exact witness, not a contiguous
+// acknowledgement prefix: delivery on independent keys can finish out of order.
+type HistoryProgress struct {
+ Node, Minted, Processed uint64
+ Received uint64 // our processed witness for this origin
+}
+
+type historyProgressProvider interface {
+ HistoryProgress() []HistoryProgress
+ RecordHistorySent(pb.NodeID, uint64)
+ RecordHistoryProcessed(pb.NodeID, uint64)
+}
+
+// An overlay freezes reclamation evidence, not the live index or snapshot
+// production. The intrusive list has one entry per active context, no effect
+// references, and is protected by Engine.historyMu.
+type historyOverlay struct {
+ progress []HistoryProgress
+ prev, next *historyOverlay
+}
+
+func (e *Engine) historyProgress() []HistoryProgress {
+ var progress []HistoryProgress
+ if provider, ok := e.broadcaster.(historyProgressProvider); ok {
+ progress = provider.HistoryProgress()
+ }
+
+ return append(progress, HistoryProgress{Node: uint64(e.nodeID), Minted: e.nextOff.Load()})
+}
+
+func (e *Engine) pinHistory(overlay *historyOverlay) {
+ if overlay.progress != nil { // only the owning context changes this field
+ return
+ }
+
+ e.historyMu.Lock()
+ defer e.historyMu.Unlock()
+
+ overlay.progress = e.historyProgress()
+
+ overlay.next = e.historyOverlays
+ if overlay.next != nil {
+ overlay.next.prev = overlay
+ }
+
+ e.historyOverlays = overlay
+}
+
+func (e *Engine) unpinHistory(overlay *historyOverlay) {
+ if overlay.progress == nil {
+ return
+ }
+
+ e.historyMu.Lock()
+ defer e.historyMu.Unlock()
+
+ if overlay.prev != nil {
+ overlay.prev.next = overlay.next
+ } else {
+ e.historyOverlays = overlay.next
+ }
+
+ if overlay.next != nil {
+ overlay.next.prev = overlay.prev
+ }
+
+ *overlay = historyOverlay{}
+}
+
+// ConstrainHistoryProgress applies active transaction overlays to outgoing
+// evidence too: peers must not reclaim history our own transactions still need.
+func (e *Engine) ConstrainHistoryProgress(peer pb.NodeID, my, your uint64) (uint64, uint64) {
+ e.historyMu.Lock()
+ defer e.historyMu.Unlock()
+
+ for overlay := e.historyOverlays; overlay != nil; overlay = overlay.next {
+ var received uint64
+
+ for _, p := range overlay.progress {
+ if p.Node == uint64(e.nodeID) {
+ my = min(my, p.Minted)
+ }
+
+ if p.Node == uint64(peer) {
+ received = p.Received
+ }
+ }
+
+ your = min(your, received)
+ }
+
+ return my, your
+}
+
+func (e *Engine) recordHistorySent(tip Tip, eff *pb.Effect) {
+ provider, ok := e.broadcaster.(historyProgressProvider)
+ if !ok {
+ return
+ }
+
+ keys := [][]byte{eff.GetKey()}
+ for _, kb := range eff.GetTxnBind().GetKeys() {
+ keys = append(keys, kb.GetKey())
+ }
+
+ for _, key := range keys {
+ // Subscribers remain obligations during partitions. The provider drops
+ // departed membership entries, not temporarily unreachable peers.
+ for _, peer := range e.PeerSubscribers(string(key)) {
+ provider.RecordHistorySent(peer, tip[1])
+ }
+ }
+}
+
+func (d *dag) recordHistoryProcessed() {
+ if d.lcaTip != (Tip{}) {
+ // The common snapshot covers every branch of this read. An arbitrary
+ // visited effect could be on just one branch of an unresolved fork.
+ d.engine.recordHistoryProcessed(d.lcaTip, d.nodes[d.lcaTip])
+ }
+}
+
+func (e *Engine) recordHistoryProcessed(tip Tip, eff *pb.Effect) {
+ provider, ok := e.broadcaster.(historyProgressProvider)
+ if !ok {
+ return
+ }
+ // A key-scoped walk does not certify every key in a bind. Neither
+ // uncommitted data nor subscription metadata certify a data closure.
+ if eff != nil && eff.GetTxnId() == "" && eff.GetTxnBind() == nil && eff.GetSubscription() == nil {
+ provider.RecordHistoryProcessed(pb.NodeID(tip[0]), tip[1])
+ }
+}
+
+// historyBoundary is scratch state for the existing LCA trimming traversal.
+// historyMu stays held while sampling overlays and releasing creation refs;
+// this traversal reads resident effects only and never waits for peer/CDN I/O.
+type historyBoundary struct {
+ engine *Engine
+ peers []HistoryProgress
+}
+
+func (e *Engine) historyBoundary(key string) *historyBoundary {
+ h := &historyBoundary{engine: e}
+
+ var members []pb.NodeID
+ if e.broadcaster != nil {
+ members = e.broadcaster.PeerIDs()
+ }
+
+ progress := e.historyProgress()
+ for _, peer := range e.PeerSubscribers(key) {
+ if !slices.Contains(members, peer) {
+ continue
+ }
+
+ p := HistoryProgress{Node: uint64(peer)}
+ for _, known := range progress {
+ if known.Node == p.Node {
+ p = known
+
+ break
+ }
+ }
+
+ for overlay := e.historyOverlays; overlay != nil; overlay = overlay.next {
+ pinned := HistoryProgress{}
+
+ for _, known := range overlay.progress {
+ if known.Node == p.Node {
+ pinned = known
+
+ break
+ }
+ }
+
+ p.Minted = min(p.Minted, pinned.Minted)
+ p.Processed = min(p.Processed, pinned.Processed)
+ }
+
+ h.peers = append(h.peers, p)
+ }
+
+ return h
+}
+
+func (h *historyBoundary) beforeOverlays(tip Tip) bool {
+ for overlay := h.engine.historyOverlays; overlay != nil; overlay = overlay.next {
+ known := false
+
+ for _, p := range overlay.progress {
+ if p.Node == tip[0] {
+ known = tip[1] <= p.Minted
+
+ break
+ }
+ }
+
+ if !known {
+ return false
+ }
+ }
+
+ return true
+}
diff --git a/effects/interfaces.go b/effects/interfaces.go
index 68aa2c9..a0eec6a 100644
--- a/effects/interfaces.go
+++ b/effects/interfaces.go
@@ -104,29 +104,13 @@ type CloudReader interface {
// the read through the subscribe + consult path, whose per-leaf
// cloudConsulted marker caps the WAN round-trips.
MayHold(key string) bool
- // CloudTips returns the candidate tip frontier Cloud's index holds for
- // key, or nil (with nil error) if Cloud holds nothing for it. These are
- // candidates, never authoritative: Cloud's tips directory is an index
- // that can lag or hole, and the engine's own DAG walk is what decides
- // what is actually installable. sidecar is the closure — every effect
- // reachable from those tips down to the LCA snapshot — delivered inline
- // by GetTips and installed into the effect cache before the tip walk, so
- // the walk runs locally instead of one WAN fetch per dep. The sidecar may
- // be partial (capped, or missing a blob the cloud is still fetching
- // back) — the walk stays the authority and pulls anything missing on
- // demand via FetchFromAny. missing names refs GetTips' own closure walk
- // found reachable but does not hold the blob for — a fact declared by
- // the storage side, letting the engine skip a redundant CDN round trip to
- // re-learn the same hole; it is a hint, not a complete inventory, so the
- // engine's walk still treats anything not listed here as normal. Content-
- // blind on the wire: the implementation maps key to its Cloud PRF image
- // and calls GetTips.
+ // CloudTips returns the marker candidates and inline effects held by Cloud.
+ // The engine supplies these to its ingestion closure walk; the sidecar may
+ // be partial and missing describes Cloud's storage, not the peers' history.
+ // Neither field chooses a reconstruction boundary or filters the frontier.
CloudTips(ctx context.Context, key string) (tips []Tip, sidecar []CloudEffect, missing []Tip, err error)
- // MarkPending reports that key's Cloud frontier has a durable hole this
- // node could not walk past — see effects.ErrCDNBlobMissing. Cloud's own
- // reconcile loop owns retrying and eventually healing or pruning the
- // affected markers; this is not a request the engine expects synchronous
- // action on.
+ // MarkPending schedules another Cloud consult after an incomplete response,
+ // such as serving an outbox frontier while GetTips is unavailable.
MarkPending(key string)
}
diff --git a/effects/mixed_transaction_test.go b/effects/mixed_transaction_test.go
new file mode 100644
index 0000000..c5f4609
--- /dev/null
+++ b/effects/mixed_transaction_test.go
@@ -0,0 +1,177 @@
+package effects
+
+import (
+ "errors"
+ "fmt"
+ "testing"
+ "testing/synctest"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+type mixedWriteBroadcaster struct {
+ bindPublicationBroadcaster
+
+ beforeBind func(*pb.OffsetNotify) error
+ responses []*pb.NackNotify
+}
+
+func (b *mixedWriteBroadcaster) Replicate(n *pb.OffsetNotify, _ []byte) error {
+ _, err := b.peer.HandleRemote(n)
+
+ return err
+}
+
+func (b *mixedWriteBroadcaster) ReplicateMarshalled(n *pb.OffsetNotify, _ []byte, _ pb.NodeID) ([]*pb.NackNotify, error) {
+ if b.beforeBind != nil {
+ if err := b.beforeBind(n); err != nil {
+ return nil, err
+ }
+ }
+
+ nacks, err := b.peer.HandleRemote(n)
+ b.responses = nacks
+
+ return nacks, err
+}
+
+// Control the delivery order between two engines: A reads x and writes y;
+// B writes x and reads the old y before receiving A's bind. Mixed operations
+// have no isolation guarantee, but the transaction result must match its reads.
+func TestNonTransactionalWriteAtBindPublication(t *testing.T) {
+ for _, withHorizon := range []bool{false, true} {
+ for _, afterPublication := range []bool{false, true} {
+ t.Run(fmt.Sprintf("horizon=%t/after-publication=%t", withHorizon, afterPublication), func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ var (
+ engines [2]*Engine
+ links [2]*mixedWriteBroadcaster
+ )
+ for i := range engines {
+ links[i] = &mixedWriteBroadcaster{}
+ links[i].peerIDs = []pb.NodeID{pb.NodeID(2 - i)}
+ links[i].allRegionPeersReachable = true
+ engines[i] = newTxnTestEngine(links[i])
+ engines[i].nodeID = pb.NodeID(i + 1)
+ engines[i].UpdateSafetyRules(SafeMode, nil)
+
+ if withHorizon {
+ engines[i].horizon = newHorizonSet(engines[i])
+ }
+
+ for j, key := range []string{"x", "y"} {
+ base := Tip{99, uint64(j + 1)}
+ if err := engines[i].effectCache.Put(base, scalarSet(key, "base")); err != nil {
+ t.Fatal(err)
+ }
+
+ engines[i].updateIndex(key, nil, base)
+ preSubscribe(engines[i], key)
+ engines[i].addPeerSubscriber(key, pb.NodeID(2-i))
+ }
+ }
+
+ links[0].peer, links[1].peer = engines[1], engines[0]
+ deliverQueued := func(i int) error {
+ link := links[i]
+ link.mu.Lock()
+ queued := link.broadcasts
+ link.broadcasts = nil
+ link.mu.Unlock()
+
+ for _, n := range queued {
+ if _, err := engines[1-i].HandleRemote(n); err != nil {
+ return err
+ }
+ }
+
+ return nil
+ }
+ tx := engines[0].NewContext()
+ tx.BeginTx()
+
+ initial, _, err := tx.GetSnapshot("x")
+ if err != nil || string(initial.GetScalar().GetRaw()) != "base" {
+ t.Fatalf("initial read: value=%v err=%v", initial, err)
+ }
+
+ if err := tx.Emit(scalarSet("y", "committed")); err != nil {
+ t.Fatal(err)
+ }
+
+ writeOutside := func() error {
+ other := engines[1].NewContext()
+ if err := other.Emit(scalarSet("x", "outside")); err != nil {
+ return err
+ }
+
+ if err := other.Flush(); err != nil {
+ return err
+ }
+
+ if err := deliverQueued(1); err != nil {
+ return err
+ }
+
+ reader := engines[1].NewContext()
+
+ value, _, err := reader.GetSnapshot("y")
+ if err != nil || string(value.GetScalar().GetRaw()) != "base" {
+ return fmt.Errorf("read before bind arrival: value=%v err=%v", value, err)
+ }
+
+ return nil
+ }
+
+ var bindTip Tip
+
+ links[0].beforeBind = func(n *pb.OffsetNotify) error {
+ bindTip = r(n.GetOrigin())
+ if !afterPublication || !engines[0].index.Contains("y").Contains(bindTip) {
+ return errors.New("unexpected bind publication")
+ }
+
+ return writeOutside()
+ }
+
+ if !afterPublication {
+ if err := writeOutside(); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ err = tx.Flush()
+ wantY := "base"
+
+ if afterPublication {
+ if err != nil || bindTip == (Tip{}) || len(links[0].responses) == 0 {
+ t.Fatalf("published bind must commit despite non-transactional NACK: bind=%v nacks=%v err=%v", bindTip, links[0].responses, err)
+ }
+
+ wantY = "committed"
+ } else if !errors.Is(err, ErrTxnAborted) || bindTip != (Tip{}) {
+ t.Fatalf("pre-publication conflict must abort without a bind: bind=%v err=%v", bindTip, err)
+ }
+
+ // Drain asynchronous effects and verdict snapshots before reads.
+ for i := range links {
+ if err := deliverQueued(i); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ for i, e := range engines {
+ for key, want := range map[string]string{"x": "outside", "y": wantY} {
+ reader := e.NewContext()
+
+ value, _, err := reader.GetSnapshot(key)
+ if err != nil || string(value.GetScalar().GetRaw()) != want {
+ t.Errorf("node%d key=%s: value=%v err=%v, want %q", i, key, value, err, want)
+ }
+ }
+ }
+ })
+ })
+ }
+ }
+}
diff --git a/effects/nack_frontier_test.go b/effects/nack_frontier_test.go
index b65ff5e..3af1a26 100644
--- a/effects/nack_frontier_test.go
+++ b/effects/nack_frontier_test.go
@@ -40,14 +40,15 @@ type nackFrontierNode struct {
}
type nackFrontierCase struct {
- name string
- nodes []nackFrontierNode
- tips []string
- lca string
- value int64
- missing bool
- required []string // refs needed even when another branch cannot be resolved
- skipped []string // history behind a valid common snapshot, or another key
+ name string
+ nodes []nackFrontierNode
+ tips []string
+ lca string
+ value int64
+ missing bool
+ required []string // refs needed even when another branch cannot be resolved
+ skipped []string // references outside this key's dependency graph
+ exhausted []string // absent ancestry below the reconstruction boundary
}
type frontierBroadcaster struct {
@@ -56,10 +57,14 @@ type frontierBroadcaster struct {
fetched map[Tip]bool
}
-func (b *frontierBroadcaster) FetchFromAny(ref *pb.EffectRef, hint FetchHint) ([]byte, error) {
+func (b *frontierBroadcaster) FetchFromAny(ref *pb.EffectRef, _ FetchHint) ([]byte, error) {
b.fetched[r(ref)] = true
- return b.selectiveBroadcaster.FetchFromAny(ref, hint)
+ if data := b.fetchable[r(ref)]; data != nil {
+ return data, nil
+ }
+
+ return nil, ErrEffectNotFound
}
// Replay answers what state is visible; collection advertises fetchable ancestry;
@@ -135,7 +140,7 @@ func TestNackWalkersDependencyFrontier(t *testing.T) {
{"r", "data", []string{"s"}, 1},
{"m", "data", []string{"l", "r"}, 1},
},
- tips: []string{"s", "m"}, lca: "s", value: 13, skipped: []string{"pruned"},
+ tips: []string{"s", "m"}, lca: "s", value: 13, exhausted: []string{"pruned"},
},
{
name: "verdict-only-snapshot",
@@ -160,7 +165,7 @@ func TestNackWalkersDependencyFrontier(t *testing.T) {
nodes: []nackFrontierNode{
{"sub", "subscription", nil, 0}, {"s", "snapshot", []string{"pruned"}, 10},
},
- tips: []string{"sub", "s"}, lca: "s", value: 10, skipped: []string{"pruned"},
+ tips: []string{"sub", "s"}, lca: "s", value: 10, exhausted: []string{"pruned"},
},
{
name: "referenced-subscription",
@@ -335,7 +340,20 @@ func checkNackFrontier(t *testing.T, tc nackFrontierCase, reverseTips, reverseDe
}
}
- // Disable fetching so replay cannot silently repair incomplete ingestion.
+ for _, name := range tc.exhausted {
+ tip := ref(name)
+ if advertised[tip] || !bc.fetched[tip] {
+ t.Errorf("closure boundary %s: advertised=%t fetched=%t", name, advertised[tip], bc.fetched[tip])
+ }
+
+ if _, resident := remote.effectCache.Get(tip); resident {
+ t.Errorf("absent boundary %s unexpectedly became resident", name)
+ }
+ }
+
+ // Disable fetching so reconstruction cannot silently repair ingestion or
+ // depend on history below the common snapshot. Closure has exhausted those
+ // boundaries; their payloads deliberately remain unavailable.
remote.broadcaster = nil
for _, engine := range []*Engine{local, remote} {
d := newDag(engine, "k", "")
diff --git a/effects/observation_causality_test.go b/effects/observation_causality_test.go
new file mode 100644
index 0000000..c3d9ac2
--- /dev/null
+++ b/effects/observation_causality_test.go
@@ -0,0 +1,254 @@
+package effects
+
+import (
+ "errors"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+func ingestObservation(t *testing.T, e *Engine, tip Tip, deps []Tip) {
+ t.Helper()
+
+ eff := &pb.Effect{
+ Key: []byte("k"), NodeId: tip[0], Hlc: sTs(int64(tip[1])),
+ ForkChoiceHash: ComputeForkChoiceHash(pb.NodeID(tip[0]), sTs(int64(tip[1]))),
+ TxnId: "remote-reader", Deps: toPbRefs(deps),
+ Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}},
+ }
+
+ data, err := MarshalEffect(eff)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := e.HandleRemote(BuildOffsetNotify(pb.NodeID(tip[0]), tip, eff, data, nil)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestObservationsRemainCausalDependencies(t *testing.T) {
+ for _, kind := range []string{"noop", "snapshot", "write", "verdict"} {
+ t.Run(kind, func(t *testing.T) {
+ e := newTxnTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "k")
+ historicalTestWrite(t, e, map[string]string{"k": "value"}, false)
+
+ observation := Tip{7, 100}
+ ingestObservation(t, e, observation, e.index.Contains("k").Tips())
+
+ var emitted *pb.Effect
+
+ e.SetOnLocalEffect(func(_ Tip, eff *pb.Effect) { emitted = eff })
+
+ c := e.NewContext()
+ defer c.Abort()
+
+ var err error
+
+ switch kind {
+ case "noop":
+ err = c.Emit(&pb.Effect{Key: []byte("k"), TxnId: "next-reader", Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}}})
+ case "snapshot":
+ value, _, _, readErr := e.GetSnapshot("k")
+ if readErr != nil {
+ t.Fatal(readErr)
+ }
+
+ err = c.Emit(&pb.Effect{Key: []byte("k"), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: value}}})
+ case "write":
+ c.BeginTx()
+ err = c.Emit(scalarSet("k", "next"))
+ case "verdict":
+ err = e.emitSnapshot("k", nil)
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if emitted == nil || !refsContain(emitted.GetDeps(), observation) {
+ t.Fatalf("%s bypassed observed NOOP: %v", kind, emitted)
+ }
+ })
+ }
+}
+
+func TestObservationDoesNotInvalidateTransaction(t *testing.T) {
+ for _, watched := range []bool{false, true} {
+ t.Run(map[bool]string{false: "read", true: "watch"}[watched], func(t *testing.T) {
+ e := newTxnTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "k")
+ historicalTestWrite(t, e, map[string]string{"k": "value"}, false)
+
+ c := e.NewContext()
+ defer c.Abort()
+
+ if watched {
+ if err := c.Watch("k"); err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ c.BeginTx()
+
+ if _, _, err := c.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ ingestObservation(t, e, Tip{7, 100}, e.index.Contains("k").Tips())
+
+ if watched {
+ c.BeginTx()
+
+ if !c.CheckWatches() {
+ t.Fatal("another observation invalidated WATCH")
+ }
+ }
+
+ if err := c.Emit(scalarSet("other", "committed")); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := c.Flush(); err != nil {
+ t.Fatalf("observation prevented commit: %v", err)
+ }
+ })
+ }
+}
+
+func TestCompactionConsumesRemoteObservations(t *testing.T) {
+ e := newTxnTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "k")
+ historicalTestWrite(t, e, map[string]string{"k": "value"}, false)
+
+ first := Tip{7, 100}
+
+ for i := range 12 {
+ ingestObservation(t, e, Tip{7, uint64(100 + i)}, e.index.Contains("k").Tips())
+ }
+
+ c := e.NewContext()
+ defer c.Abort()
+
+ if _, _, err := c.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ if c.keys["k"] == nil {
+ t.Fatal("remote observation history did not trigger compaction")
+ }
+
+ snapshot := c.keys["k"].lastOffset
+ if err := c.Flush(); err != nil {
+ t.Fatal(err)
+ }
+ // Reconciliation must recognize the old observation as consumed, rather
+ // than restoring it as a competing branch beside the snapshot.
+ e.installIngestedTips("k", first)
+
+ tips := e.index.Contains("k").Tips()
+ if len(tips) != 1 || tips[0] != snapshot {
+ t.Fatalf("compacted observations returned to the frontier: %v", tips)
+ }
+
+ value, _, walked, err := e.GetSnapshot("k")
+ if err != nil || string(value.GetScalar().GetRaw()) != "value" || walked != 1 {
+ t.Fatalf("compaction did not bound reconstruction: value=%v walked=%d err=%v", value, walked, err)
+ }
+}
+
+func TestWriteBehindObservationInvalidatesTransaction(t *testing.T) {
+ e := newTxnTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "k")
+ historicalTestWrite(t, e, map[string]string{"k": "before"}, false)
+
+ c := e.NewContext()
+ defer c.Abort()
+
+ c.BeginTx()
+
+ if _, _, err := c.GetSnapshot("k"); err != nil {
+ t.Fatal(err)
+ }
+
+ historicalTestWrite(t, e, map[string]string{"k": "after"}, false)
+ ingestObservation(t, e, Tip{7, 100}, e.index.Contains("k").Tips())
+
+ if err := c.Emit(scalarSet("other", "must abort")); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := c.Flush(); !errors.Is(err, ErrTxnAborted) {
+ t.Fatalf("write hidden by NOOP did not abort: %v", err)
+ }
+}
+
+func TestObservingNoopDoesNotOrderBinds(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ base := log.putEffect(scalarSet("k", "base"))
+ observation := log.putEffect(&pb.Effect{
+ Key: []byte("k"), TxnId: "reader", Deps: toPbRefs([]Tip{base}),
+ Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}},
+ })
+ write := scalarSet("k", "writer")
+ write.TxnId, write.Deps = "writer", toPbRefs([]Tip{observation})
+ writerTip := log.putEffect(write)
+ makeBind := func(txn string, tip Tip, consumed Tip, hash byte) (*pb.Effect, Tip) {
+ eff := &pb.Effect{
+ Key: []byte("k"), TxnId: txn, Deps: toPbRefs([]Tip{tip}), ForkChoiceHash: []byte{hash},
+ Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{Keys: []*pb.TransactionalBindEffect_KeyBind{
+ {Key: []byte("k"), NewTip: toPbRef(tip), ConsumedTips: toPbRefs([]Tip{consumed})},
+ }}},
+ }
+
+ return eff, log.putEffect(eff)
+ }
+ reader, readerBind := makeBind("reader", observation, base, 2)
+ writer, writerBind := makeBind("writer", writerTip, observation, 1)
+
+ e.updateIndex("k", nil, readerBind)
+ e.updateIndex("k", nil, writerBind)
+
+ if got := e.checkCompetingBinds(writer.GetTxnBind(), "writer"); got != "reader" {
+ t.Errorf("observing reader's NOOP excused its competing bind: %q", got)
+ }
+
+ ptxn := &pendingTxn{txnID: "reader", bindOffset: readerBind, forkChoiceHash: reader.GetForkChoiceHash(), keys: []pendingTxnKey{{key: "k", newTip: observation, readOnly: true}}}
+ detail := nackTipDetail(writerBind, writer)
+
+ detail.BindForkChoiceHash = writer.GetForkChoiceHash()
+ if !e.isRealConflict(ptxn, "k", detail) {
+ t.Error("writer observed our NOOP, not our bind; its winning bind must conflict")
+ }
+
+ losers, _ := e.losersOnKeyAt("k", []Tip{readerBind, writerBind}, nil, nil, nil, false)
+ if _, lost := losers["reader"]; !lost {
+ t.Errorf("reconstruction treated observation as committed ordering: losers=%v", losers)
+ }
+}
+
+func TestBindAncestryIsKeyScoped(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ first := closureBind(log, "first", 1, []string{"k", "other"}, nil)
+ second := closureBind(log, "second", 2, []string{"k", "other"}, map[string]Tip{"other": first})
+
+ ancestry := newCausalIndex(e, "k")
+ if ancestry.reaches(second, first) {
+ t.Fatal("observing a bind on another key ordered concurrent branches on k")
+ }
+
+ if !newCausalIndex(e, "other").reaches(second, first) {
+ t.Fatal("lost the bind dependency on the key where it was observed")
+ }
+}
diff --git a/effects/predicate.go b/effects/predicate.go
index 411c1b3..338b3e4 100644
--- a/effects/predicate.go
+++ b/effects/predicate.go
@@ -50,6 +50,12 @@ import (
// tx's entire reachable history.
func (e *Engine) collectTxnEffectsOnKey(txnID, key string, startTips ...Tip) (
obs []*pb.ObservationEffect, rw []*pb.RowWriteEffect,
+) {
+ return collectTxnEffects(txnID, key, func(t Tip) (*pb.Effect, error) { return e.getEffect(key, t) }, startTips...)
+}
+
+func collectTxnEffects(txnID, key string, load func(Tip) (*pb.Effect, error), startTips ...Tip) (
+ obs []*pb.ObservationEffect, rw []*pb.RowWriteEffect,
) {
var zero Tip
@@ -72,7 +78,7 @@ func (e *Engine) collectTxnEffectsOnKey(txnID, key string, startTips ...Tip) (
visited[off] = true
- eff, err := e.getEffect(key, off)
+ eff, err := load(off)
if err != nil {
continue
}
diff --git a/effects/repair_test.go b/effects/repair_test.go
index 650ffb2..6844f8c 100644
--- a/effects/repair_test.go
+++ b/effects/repair_test.go
@@ -24,6 +24,7 @@
package effects
import (
+ "errors"
"fmt"
"slices"
"testing"
@@ -99,11 +100,8 @@ func (b *holedFetchBroadcaster) FetchFromAny(ref *pb.EffectRef, _ FetchHint) ([]
return nil, fmt.Errorf("cdn fetch %v: %w", r(ref), ErrCDNBlobMissing)
}
-// TestInstallCloudTipsNeverMints: a cloud frontier tip whose walk fails with
-// the CDN-404 sentinel must not be repaired with a synthesized snapshot —
-// InstallCloudTips never mints and never touches local ancestry. The holed
-// tip is reported back as pending for the caller to hand to Cloud's reconcile
-// loop, and the engine's own local state is left exactly as it was.
+// Cloud ingestion retains unresolved tips without minting a replacement value
+// or discarding existing local history. Reconstruction reports missing data.
func TestInstallCloudTipsNeverMints(t *testing.T) {
e := newTxnTestEngine(&holedFetchBroadcaster{})
defer func() { _ = e.Close() }()
@@ -118,39 +116,38 @@ func TestInstallCloudTipsNeverMints(t *testing.T) {
}
before := e.index.Contains("k").Tips()
+ beforeOffset := e.nextOff.Load()
holedTip := Tip{12345, 7}
- pending, err := e.InstallCloudTips("k", []Tip{holedTip}, nil, nil)
+ err := e.InstallCloudTips("k", []Tip{holedTip}, nil)
if err != nil {
t.Fatalf("install over holed frontier: %v", err)
}
- if !slices.Equal(pending, []Tip{holedTip}) {
- t.Fatalf("pending = %v, want [%v]", pending, holedTip)
+ if e.nextOff.Load() != beforeOffset {
+ t.Fatal("cloud ingestion minted a local effect")
}
after := e.index.Contains("k").Tips()
- if !slices.Equal(before, after) {
- t.Fatalf("local tips changed from %v to %v: install must never mint over a hole", before, after)
+ if len(after) != len(before)+1 || !slices.Contains(after, holedTip) {
+ t.Fatalf("frontier = %v, want local tips %v plus %v", after, before, holedTip)
}
- result, _, _, err := e.GetSnapshot("k")
- if err != nil {
- t.Fatalf("post-install read: %v", err)
+ for _, tip := range before {
+ if !slices.Contains(after, tip) {
+ t.Fatalf("lost local tip %v from %v", tip, after)
+ }
}
- if result == nil || result.GetScalar() == nil || string(result.GetScalar().GetRaw()) != "local" {
- t.Fatalf("post-install state = %v, want scalar %q", result, "local")
+ if _, _, _, err := e.GetSnapshot("k"); !errors.Is(err, errEffectNotResident) {
+ t.Fatalf("reconstruction must report the unavailable branch, got %v", err)
}
}
-// TestInstallCloudTipsPreservesReadableSiblings: when only part of Cloud's
-// frontier sits above a hole, the readable tip must still install and serve
-// on its own — a hole in one candidate is not grounds to withhold the rest of
-// the frontier — while the holed tip is reported pending instead of blocking
-// the install.
-func TestInstallCloudTipsPreservesReadableSiblings(t *testing.T) {
+// A readable sibling cannot justify dropping an unavailable branch from the
+// closure frontier. Value reconstruction must see both.
+func TestInstallCloudTipsPreservesAllSiblings(t *testing.T) {
e := newTxnTestEngine(&holedFetchBroadcaster{})
defer func() { _ = e.Close() }()
@@ -171,30 +168,21 @@ func TestInstallCloudTipsPreservesReadableSiblings(t *testing.T) {
sidecar := []CloudEffect{{Tip: readableTip, Eff: readableEff, ProtoLen: proto.Size(readableEff)}}
- pending, err := e.InstallCloudTips("k", []Tip{readableTip, holedTip}, sidecar, nil)
+ err := e.InstallCloudTips("k", []Tip{readableTip, holedTip}, sidecar)
if err != nil {
t.Fatalf("install over partially holed frontier: %v", err)
}
- if !slices.Equal(pending, []Tip{holedTip}) {
- t.Fatalf("pending = %v, want [%v]", pending, holedTip)
- }
-
ts := e.index.Contains("k")
if ts == nil || !slices.Contains(ts.Tips(), readableTip) {
t.Fatalf("post-install tips = %v, want the readable tip installed", ts.Tips())
}
- if slices.Contains(ts.Tips(), holedTip) {
- t.Fatalf("post-install tips = %v, holed tip must never be installed", ts.Tips())
- }
-
- result, _, _, err := e.GetSnapshot("k")
- if err != nil {
- t.Fatalf("post-install read: %v", err)
+ if !slices.Contains(ts.Tips(), holedTip) {
+ t.Fatalf("post-install tips = %v, missing branch must remain installed", ts.Tips())
}
- if result == nil || result.GetScalar() == nil || string(result.GetScalar().GetRaw()) != "cloud" {
- t.Fatalf("post-install state = %v, want the readable sibling's scalar %q", result, "cloud")
+ if _, _, _, err := e.GetSnapshot("k"); !errors.Is(err, errEffectNotResident) {
+ t.Fatalf("reconstruction must not serve just the readable sibling: %v", err)
}
}
diff --git a/effects/snapshot.go b/effects/snapshot.go
index a32e903..d4cafac 100644
--- a/effects/snapshot.go
+++ b/effects/snapshot.go
@@ -52,6 +52,11 @@ import (
// post-resolution state.
var errHorizonRetryNeeded = errors.New("reconstruct: horizon ancestor resolved, retry")
+const compactionInterval = 5 * time.Second
+
+// Elapsed time is used only to pace local maintenance, never to order effects.
+var compactionEpoch = time.Now()
+
// leafState is the per-key payload attached to each critbit leaf (the trie's
// T). It carries the cached subdag for the key so reads reuse the prepared
// structure instead of rebuilding it. It is created once per key and mutated
@@ -59,6 +64,9 @@ var errHorizonRetryNeeded = errors.New("reconstruct: horizon ancestor resolved,
// the subdag, which is why this is a struct rather than a bare subdag pointer.
type leafState struct {
subdag atomic.Pointer[cachedDag]
+ // Monotonic nanoseconds since compactionEpoch (+1 reserves zero for an
+ // unused leaf). Claiming before emission also paces aborted attempts.
+ lastCompaction atomic.Int64
// cloudConsulted marks that the Cloud read-miss backstop has received one
// complete answer for this key during this leaf's residency. It turns a
// nil reduction from "not known — ask Cloud" into "authoritatively
@@ -67,25 +75,10 @@ type leafState struct {
// reclaims the leaf and the marker with it, re-arming the rehydrate.
cloudConsulted atomic.Bool
- // cloudPending marks that a Cloud consult for this key found a durable
- // hole (ErrCDNBlobMissing) it handed to the cluster's reconcile loop via
- // MarkPending, rather than a complete answer. It is mutually exclusive
- // with cloudConsulted for a given consult: unlike cloudConsulted it does
- // NOT make a nil reduction authoritative — Cloud provably holds ancestry
- // this node could not fetch, so GetSnapshot must keep returning
- // ErrCloudUnavailable instead of a free miss. What it does share with
- // cloudConsulted is suppressing repeat WAN GetTips round-trips for the
- // rest of this leaf's residency: the reconcile loop owns retrying the
- // hole on Cloud's own schedule, and re-polling here on every read of a
- // hot pending key would just pay the same round-trip for the same
- // not-yet-healed answer. Eviction reclaims the leaf and the marker with
- // it, re-arming the consult.
- cloudPending atomic.Bool
-
// cloudMu single-flights the Cloud consult for this leaf: concurrent
// missers of the same key queue here instead of each paying the WAN
- // GetTips round-trip, then re-check cloudConsulted/cloudPending and serve
- // the winner's install. A failed consult leaves both markers false, so
+ // GetTips round-trip, then re-check cloudConsulted and serve
+ // the winner's install. A failed consult leaves the marker false, so
// the next miss retries.
cloudMu sync.Mutex
@@ -102,6 +95,15 @@ type leafState struct {
subscriberBytes int64
}
+func (ls *leafState) claimCompaction(now int64) bool {
+ previous := ls.lastCompaction.Load()
+ if previous != 0 && now-previous < int64(compactionInterval) {
+ return false
+ }
+
+ return ls.lastCompaction.CompareAndSwap(previous, now)
+}
+
// cachedDag is the reusable output of dag.prepare for a specific tip set: the
// fork-choice-sorted topological order and the LCA. It is immutable once
// published; a tip change produces a new one. Fork-choice adjudication
@@ -173,6 +175,7 @@ func (e *Engine) prepareDag(d *dag, key string, tips []Tip) error {
d.nodes = nodes
d.topoOrder = cs.topoOrder
d.lcaTip = cs.lcaTip
+ d.recordHistoryProcessed()
if d.activeTxnID == "" {
e.installBoundSubdags(d)
@@ -182,8 +185,17 @@ func (e *Engine) prepareDag(d *dag, key string, tips []Tip) error {
}
}
+ d.compact = d.activeTxnID == ""
if err := d.prepare(tips); err != nil {
- return err
+ if !errors.Is(err, errEffectNotResident) {
+ return err
+ }
+ // Failed ingestion can leave an unresolved frontier installed. Retry
+ // closure only for an actually missing reconstruction dependency;
+ // normal reads and reclaimed history never enter the fetch path.
+ if err := d.ingest(tips); err != nil {
+ return err
+ }
}
// A transaction reads its captured frontier. Installing historical tips
@@ -408,10 +420,12 @@ func (e *Engine) publishSubdag(ls *leafState, d *dag, tips []Tip) {
}
// verdictEntry holds a snapshot's verdict for a txnID; snapshotTip is the
-// offset of the snapshot that supplied it, used for debug log provenance.
+// offset of the snapshot that supplied it. Historical membership requires this
+// proof source to belong to the selected history.
type verdictEntry struct {
verdict pb.Verdict
snapshotTip Tip
+ proven bool // derived from retained proof, not merely a cached label
}
// verdictMap accumulates snapshot-supplied verdicts during a walk. Recorded
@@ -427,28 +441,44 @@ func (v verdictMap) harvest(snapshotTip Tip, snap *pb.SnapshotEffect) {
}
for txnID, newVerdict := range snap.GetTxnVerdicts() {
- if newVerdict == pb.Verdict_VERDICT_UNSPECIFIED {
- continue
- }
+ v.retain(txnID, verdictEntry{verdict: newVerdict, snapshotTip: snapshotTip})
+ }
+}
- existing, ok := v[txnID]
- if !ok {
- v[txnID] = verdictEntry{verdict: newVerdict, snapshotTip: snapshotTip}
+// retain keeps the first observed verdict, including when combining evidence
+// from a live walk and a historical walk.
+func (v verdictMap) retain(txnID string, entry verdictEntry) {
+ if entry.verdict == pb.Verdict_VERDICT_UNSPECIFIED {
+ return
+ }
- continue
- }
+ existing, ok := v[txnID]
+ if entry.proven && !existing.proven {
+ v[txnID] = entry
- if existing.verdict == newVerdict {
- continue
- }
+ return
+ }
+
+ if existing.proven && !entry.proven {
+ return
+ }
+
+ if !ok {
+ v[txnID] = entry
+
+ return
+ }
- slog.Warn("snapshot verdict disagreement",
- "txn_id", txnID,
- "existing_verdict", existing.verdict,
- "existing_snapshot", existing.snapshotTip,
- "new_verdict", newVerdict,
- "new_snapshot", snapshotTip)
+ if existing.verdict == entry.verdict {
+ return
}
+
+ slog.Warn("snapshot verdict disagreement",
+ "txn_id", txnID,
+ "existing_verdict", existing.verdict,
+ "existing_snapshot", existing.snapshotTip,
+ "new_verdict", entry.verdict,
+ "new_snapshot", entry.snapshotTip)
}
// isJSONType reports whether the type tag marks a RedisJSON node (key root or a
@@ -556,44 +586,22 @@ func (e *Engine) GetSnapshot(key string) (*pb.ReducedEffect, []Tip, int, error)
return nil, nil, 0, err
}
- // Tiered storage: a nil result can mean three things and only two warrant
- // a Cloud consult. "Not-known nil" — no local history, or the key was
- // evicted — is the rehydrate case: ask Cloud for the frontier and install
- // whatever portion of it is locally walkable (blobs pulled via
- // FetchFromAny → CDN), so reconstructing again returns the value and the
- // cluster owns the key thereafter. "Authoritative nil" — we hold the
- // chain and it reduces to nothing (DEL tombstone, emptied collection) —
- // must not re-consult: such keys stay filter-positive in Cloud forever,
- // and without the distinction every GET of them pays a WAN GetTips
- // round-trip. "Holed nil" — Cloud's frontier hit a durable
- // ErrCDNBlobMissing and nothing readable resolved locally — must not be
- // reported as a miss either, since Cloud provably holds ancestry this
- // node could not fetch; it returns ErrCloudUnavailable instead, and only
- // Cloud's own reconcile loop (via MarkPending) resolves it, on its own
- // schedule.
- //
- // A data-bearing tip marks the nil authoritative; subscription tips don't
- // count (a subscribed reader's own SubscriptionEffect sits in the index
- // as a tip), and a free-missed key has no tips at all — vacuously
- // subscription-only. The leaf's cloudConsulted and cloudPending markers
- // are mutually exclusive per consult and both cap further WAN round-trips
- // for the rest of this leaf's residency — one distinguishes "nil is
- // authoritative" from "nil is a retryable hole" — and eviction reclaims
- // both with the leaf, re-arming the consult. Gated on the majority
- // partition: a minority node can't trust its cluster view
- // (ensureSubscribed already errored out above for it). UnsafeMode has no
- // majority to trust — it rehydrates on whatever view it has.
+ // Consult Cloud once for a key whose local tips contain only subscription
+ // metadata. Ingest the available closure, then reconstruct from the complete
+ // frontier: required missing history is an error, never a readable-subset
+ // success or a fabricated miss. A successful empty reduction is authoritative
+ // for this leaf's residency; subscriptions reconcile subsequent changes.
if result == nil && e.cloudReader != nil && e.subscriptionOnlyTips(tips) &&
(e.inMajorityPartition() || e.modeForKey(key) == UnsafeMode) {
ls, _ := e.index.LoadOrStoreData(key, &leafState{})
- if ls == nil || (!ls.cloudConsulted.Load() && !ls.cloudPending.Load()) {
+ if ls == nil || !ls.cloudConsulted.Load() {
if ls != nil {
ls.cloudMu.Lock()
defer ls.cloudMu.Unlock()
}
- if ls == nil || (!ls.cloudConsulted.Load() && !ls.cloudPending.Load()) {
- installed, consulted, hydrateErr := e.hydrateFromCloud(key)
+ if ls == nil || !ls.cloudConsulted.Load() {
+ installed, hydrateErr := e.hydrateFromCloud(key)
if hydrateErr != nil {
// Cloud may hold this key and we couldn't get a complete
// answer: the read fails (client retries) rather than
@@ -622,30 +630,18 @@ func (e *Engine) GetSnapshot(key string) (*pb.ReducedEffect, []Tip, int, error)
}
if ls != nil {
- if consulted {
- ls.cloudConsulted.Store(true)
- } else {
- // hydrateFromCloud reports !consulted with a nil error
- // exactly when it found a durable hole and handed it
- // to MarkPending — never for a transport failure,
- // which surfaced as hydrateErr above.
- ls.cloudPending.Store(true)
- }
+ ls.cloudConsulted.Store(true)
}
} else {
// Lost the single-flight race: a concurrent reader already
- // resolved (or found pending) this leaf's Cloud consult while
- // we waited on cloudMu. Serve its result.
+ // resolved this leaf's Cloud consult while we waited on cloudMu.
+ // Serve its result.
result, tips, chainLen, err = e.reconstructLocal(key)
if err != nil {
return nil, nil, 0, err
}
}
}
-
- if result == nil && ls != nil && ls.cloudPending.Load() && !ls.cloudConsulted.Load() {
- return nil, nil, 0, fmt.Errorf("%w: frontier for %q partially holed, reconcile pending", ErrCloudUnavailable, key)
- }
}
return result, tips, chainLen, nil
@@ -658,7 +654,7 @@ func (e *Engine) GetSnapshot(key string) (*pb.ReducedEffect, []Tip, int, error)
// by a failure path (markFailed) rather than a successful bootstrap — the
// closeOnce + Store-before-close in markFailed ensures the re-load sees the
// up-to-date value.
-func (e *Engine) awaitSubscription(existing *subscriptionState, unsafe bool) error {
+func (e *Engine) awaitSubscription(existing *subscriptionState, unsafe, allowAsync bool) error {
if existing.incomplete.Load() {
if unsafe {
return nil
@@ -667,7 +663,27 @@ func (e *Engine) awaitSubscription(existing *subscriptionState, unsafe bool) err
return ErrBootstrapIncomplete
}
- <-existing.ready
+ select {
+ case <-existing.ready:
+ default:
+ // An async announcement installs local authority, but does not prove
+ // discovery complete. Upgrade it once when a transaction needs peers.
+ existing.bootstrapMu.Lock()
+ if notify := existing.announcement; notify != nil {
+ if allowAsync {
+ existing.bootstrapMu.Unlock()
+
+ return nil
+ }
+
+ existing.announcement = nil
+ existing.bootstrapMu.Unlock()
+ // Retry work may outlive this caller's borrowed key buffer.
+ return e.bootstrapSubscription(string(notify.GetKey()), existing, notify, unsafe)
+ }
+ existing.bootstrapMu.Unlock()
+ <-existing.ready
+ }
if existing.incomplete.Load() {
if unsafe {
@@ -834,7 +850,7 @@ func (e *Engine) ensureSubscribedMode(key string, allowAsync bool) error {
// already-subscribed case must not pay the state+channel allocation the
// LoadOrStore below needs.
if existing, ok := e.subscriptions.Load(key); ok {
- return e.awaitSubscription(existing, isUnsafe)
+ return e.awaitSubscription(existing, isUnsafe, allowAsync)
}
// Callers on the Redis hot path may borrow key from a pooled command
// buffer. A new subscription outlives that command, so own the map/trie key
@@ -843,23 +859,18 @@ func (e *Engine) ensureSubscribedMode(key string, allowAsync bool) error {
ownedKey := e.retainKey(key)
state := &subscriptionState{ready: make(chan struct{}), ownedKey: true}
+ state.bootstrapMu.Lock()
if existing, loaded := e.subscriptions.LoadOrStore(ownedKey, state); loaded {
+ state.bootstrapMu.Unlock()
e.releaseKey(ownedKey)
- return e.awaitSubscription(existing, isUnsafe)
+ return e.awaitSubscription(existing, isUnsafe, allowAsync)
}
if slog.Default().Enabled(context.Background(), slog.LevelDebug) {
slog.Debug("ensureSubscribed: bootstrapping", "key", ownedKey)
}
- succeeded := false
- defer func() {
- if succeeded {
- state.markReady()
- }
- }()
-
hlc := timestamppb.New(e.clock.Now())
eff := &pb.Effect{
Key: []byte(ownedKey),
@@ -871,12 +882,13 @@ func (e *Engine) ensureSubscribedMode(key string, allowAsync bool) error {
}},
}
- data, err := MarshalEffect(eff)
+ data, err := e.effectCache.MarshalEffect(eff)
if err != nil {
// Marshal failure is a code bug; remove state so a fresh call
// retries from scratch and concurrent waiters re-bootstrap.
e.dropSubscription(ownedKey)
state.markFailed()
+ state.bootstrapMu.Unlock()
return err
}
@@ -887,6 +899,7 @@ func (e *Engine) ensureSubscribedMode(key string, allowAsync bool) error {
if err := e.effectCache.PutSized(offset, eff, len(data)); err != nil {
e.dropSubscription(ownedKey)
state.markFailed()
+ state.bootstrapMu.Unlock()
return err
}
@@ -896,35 +909,46 @@ func (e *Engine) ensureSubscribedMode(key string, allowAsync bool) error {
e.fireLocalEffect(offset, eff)
if e.broadcaster == nil {
- succeeded = true
+ state.markReady()
+ state.bootstrapMu.Unlock()
return nil
}
notify := BuildOffsetNotify(e.nodeID, offset, eff, data, nil)
- // Async-announce fast path: when no peer in the cluster holds this key
- // (authoritative while we're in the majority partition) there is no prior
- // state to bootstrap — the blocking rounds below would collect nothing. We
- // still MUST announce the subscription so future peer writes route to us
- // (skipping it splits brain), but we can fire-and-forget: QUIC delivers
- // reliably on a live connection, and peer-recovery re-announce
- // (pubsub_router.OnPeerAdded) heals any peer whose connection was down.
- // Never skip the announce; only skip the wait. The local install above is
- // synchronous (so a subsequent Emit dep-references our SubscriptionEffect),
- // but the network send is off the critical path: fire it in the background
- // so the calling SET/GET returns immediately. A connected peer whose bulk
- // filter hasn't arrived yet forces clusterMaybeHasKey true, so a fresh
- // node always takes the blocking bootstrap below.
+ // Ordinary accesses can announce asynchronously when peer filters report
+ // no holder. Concurrent peers may still subscribe: the announcement is
+ // not completed discovery, and a later transaction must run the blocking
+ // rounds. Local installation stays synchronous so Emit can depend on our
+ // SubscriptionEffect. Missing peer filters force the blocking path below.
if allowAsync &&
(e.inMajorityPartition() || isUnsafe) && !e.clusterMaybeHasKey(ownedKey) {
+ // Keep ready open: a later blocking caller must perform discovery,
+ // even if this announcement has already been delivered.
+ state.announcement = notify
go e.broadcaster.BroadcastWithData(notify, notify.GetEffectData())
-
- succeeded = true
+ state.bootstrapMu.Unlock()
return nil
}
+ state.bootstrapMu.Unlock()
+
+ return e.bootstrapSubscription(ownedKey, state, notify, isUnsafe)
+}
+
+// bootstrapSubscription runs the same discovery for a new blocking subscription
+// and an upgrade from an async announcement. Other callers wait on state.ready;
+// no subscription lock is held during network or ancestry work.
+func (e *Engine) bootstrapSubscription(ownedKey string, state *subscriptionState, notify *pb.OffsetNotify, isUnsafe bool) error {
+ succeeded := false
+ defer func() {
+ if succeeded {
+ state.markReady()
+ }
+ }()
+
// Register bootstrap collector before broadcasting so NACKs aren't missed
collector := &bootstrapCollector{
nacks: make(chan *pb.NackNotify, 64),
@@ -1129,7 +1153,7 @@ func (e *Engine) seedSubscribersFromDag(key string) {
func (e *Engine) walkAndInstall(key string, tips []Tip) (installed, skipped int) {
for _, tipOff := range tips {
rd := newDag(e, key, "")
- if walkErr := rd.walk([]Tip{tipOff}, func(*pb.Effect) error { return nil }); walkErr != nil {
+ if walkErr := rd.ingest([]Tip{tipOff}); walkErr != nil {
slog.Debug("walkAndInstall: skipping unreachable tip",
"key", key, "tip", tipOff, "error", walkErr)
@@ -1253,23 +1277,20 @@ func (e *Engine) reconstructWithView(key string, tips []Tip, txID string, waitFo
return nil, 0, err
}
- atomicallyLost = make(map[string]struct{})
-
- for txnID, entry := range snapshotVerdicts {
- if entry.verdict == pb.Verdict_LOST {
- atomicallyLost[txnID] = struct{}{}
- }
+ var invisible func(string) bool
+ if e.horizon != nil && !e.horizon.Empty() {
+ invisible = e.horizon.IsInvisible
}
- for k := range bindsByKey {
- losers, walked := e.losersOnKey(k, bindsByKey[k], snapshotVerdicts)
+ var walkedByKey map[string]int
+
+ atomicallyLost, walkedByKey = e.losersOnKeysAt(bindsByKey, func(k string) []Tip {
+ return e.index.Contains(k).Tips()
+ }, snapshotVerdicts, invisible, true)
+ for k, walked := range walkedByKey {
if k != key {
crossKeyWalked += walked
}
-
- for txnID := range losers {
- atomicallyLost[txnID] = struct{}{}
- }
}
}
@@ -1521,7 +1542,136 @@ func (e *Engine) reconstructWithView(key string, tips []Tip, txID string, waitFo
"count", count, "cross_key_walked", crossKeyWalked,
"result_nil", result == nil)
- return result, count + crossKeyWalked, nil
+ // Compaction is driven by causal history size, including transaction
+ // observations filtered out of value reconstruction by shouldEmit.
+ return result, len(d.topoOrder) + crossKeyWalked, nil
+}
+
+// walkBindEvidence follows the same common-snapshot boundary as reconstruction.
+// The starting set includes cross-key bind anchors: a snapshot that covered the
+// local frontier alone may not cover those anchors. PrevSnapshot is a separate
+// non-owning verdict shortcut, never a required dependency or a reason to
+// traverse data below the common boundary. Required proof uses Evidence refs.
+func (e *Engine) walkBindEvidence(key string, tips []Tip, visit func(Tip, *pb.Effect)) error {
+ d := newDag(e, key, "")
+ failed := make(map[Tip]error)
+ load := func(t Tip) *pb.Effect {
+ if eff := d.nodes[t]; eff != nil {
+ return eff
+ }
+
+ if _, attempted := failed[t]; attempted {
+ return nil
+ }
+
+ eff, err := e.getEffect(key, t)
+ if err != nil {
+ failed[t] = err
+ }
+
+ return eff
+ }
+
+ for {
+ d.nodes = make(map[Tip]*pb.Effect)
+ d.topoOrder = nil
+ d.lcaTip = Tip{}
+
+ paths, err := d.topoCollect(tips, true)
+ if err != nil {
+ // Tolerant callers historically inspect cyclic evidence. Without
+ // a proven common snapshot, no snapshot can stop the walk.
+ d.lcaTip = Tip{}
+
+ break
+ }
+ // Ingestion normally supplies the resident graph. If required effects
+ // must be fetched, establish the boundary again before interpreting it.
+ // The post-order prefix below the LCA is folded, possibly reclaimed.
+ start := 0
+ if d.lcaTip != (Tip{}) {
+ start = paths[d.lcaTip].order
+ }
+
+ loaded := false
+
+ for _, t := range d.topoOrder[start:] {
+ if d.nodes[t] == nil && load(t) != nil {
+ loaded = true
+ }
+ }
+
+ if !loaded {
+ break
+ }
+ }
+
+ seen := make(map[Tip]bool)
+ stack := slices.Clone(tips)
+
+ var snapshots []Tip
+
+ for len(stack) > 0 {
+ t := stack[len(stack)-1]
+ stack = stack[:len(stack)-1]
+
+ if seen[t] {
+ continue
+ }
+
+ seen[t] = true
+
+ eff := load(t)
+ if eff == nil {
+ continue
+ }
+
+ visit(t, eff)
+
+ if prev := eff.GetSnapshot().GetPrevSnapshot(); prev != nil {
+ snapshots = append(snapshots, r(prev))
+ }
+
+ if t != d.lcaTip {
+ for dep := range keyDeps(key, eff) {
+ stack = append(stack, r(dep))
+ }
+ }
+ }
+
+ for len(snapshots) > 0 {
+ t := snapshots[len(snapshots)-1]
+ snapshots = snapshots[:len(snapshots)-1]
+
+ if seen[t] {
+ continue
+ }
+
+ seen[t] = true
+
+ // Compaction may reclaim a shortcut target. Keep any available
+ // verdicts, but do not turn this optional fetch into a read failure.
+ // If the same vertex was required by the value DAG, its load failure
+ // is already recorded above and remains an error.
+ eff, _ := e.getEffect(key, t)
+ if snap := eff.GetSnapshot(); snap != nil {
+ visit(t, eff)
+
+ if prev := snap.GetPrevSnapshot(); prev != nil {
+ snapshots = append(snapshots, r(prev))
+ }
+ }
+ }
+
+ var failures []error
+
+ for t, err := range failed {
+ if seen[t] {
+ failures = append(failures, err)
+ }
+ }
+
+ return errors.Join(failures...)
}
// bindKeyClosure widens outward from startKey along bind cross-key links
@@ -1569,16 +1719,14 @@ func (e *Engine) bindKeyClosureFrom(startKeys []string, tipsFor func(string) ([]
}
}
- // A tip can be discovered as another key's anchor before being visited.
- // Keep both facts together, with one ledger for the entire closure.
+ // Only anchor discovery survives a key revisit. The walk's visited set is
+ // local to its current frontier, since a new anchor can move the LCA.
type location struct {
key string
tip Tip
}
- type progress struct{ visited, anchored bool }
-
- seen := make(map[location]progress)
+ anchored := make(map[location]bool)
verdicts := make(verdictMap)
for len(queue) > 0 {
@@ -1591,47 +1739,38 @@ func (e *Engine) bindKeyClosureFrom(startKeys []string, tipsFor func(string) ([]
return nil, nil, err
}
- stack := slices.Concat(frontier, bindsByKey[k])
- // Tolerant traversal: dag.walk errors on the first unfetchable
- // dep, which would truncate closure expansion. For closure
- // discovery we only need to find binds reachable from tips,
- // not a complete DAG — skip missing refs and keep going.
- for len(stack) > 0 {
- t := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
+ var (
+ discovered []Tip
+ proofErr error
+ )
- at := location{k, t}
-
- state := seen[at]
- if state.visited {
- continue
+ err = e.walkBindEvidence(k, slices.Concat(frontier, bindsByKey[k]), func(t Tip, eff *pb.Effect) {
+ if eff.GetTxnBind() != nil {
+ discovered = append(discovered, t)
}
- state.visited = true
- seen[at] = state
-
- eff, err := e.getEffect(k, t)
- if err != nil {
- if strict {
- return nil, nil, err
- }
-
- continue
+ if eff.GetSnapshot() != nil {
+ proofErr = errors.Join(proofErr, e.harvestProofVerdicts(t, eff, verdicts))
}
+ })
- bind := eff.GetTxnBind()
- if bind != nil {
+ if proofErr != nil {
+ return nil, nil, proofErr
+ }
+
+ for _, t := range discovered {
+ eff, _ := e.effectCache.Get(t)
+ // An owned proof supplies the historical cross-key outcome without
+ // reopening another key's compacted value DAG. Bare cached labels
+ // retain the legacy discovery path.
+ if bind := eff.GetTxnBind(); bind != nil && !verdicts[eff.GetTxnId()].proven {
for _, kb := range bind.GetKeys() {
other := string(kb.GetKey())
- anchor := location{other, t}
- state := seen[anchor]
- if other != k && !state.anchored {
- // Start at the envelope: NewTip is its ancestor
- // and cannot lead a backwards walk to this bind.
- // A new anchor also requeues an already visited key.
- state.anchored = true
- seen[anchor] = state
+ anchor := location{other, t}
+ if other != k && !anchored[anchor] {
+ // Start at the envelope, not its NewTip ancestor.
+ anchored[anchor] = true
bindsByKey[other] = append(bindsByKey[other], t)
if !queued[other] {
@@ -1640,33 +1779,11 @@ func (e *Engine) bindKeyClosureFrom(startKeys []string, tipsFor func(string) ([]
}
}
}
- // Bind dep: same-key NewTip.
- for ref := range keyDeps(k, eff) {
- stack = append(stack, r(ref))
- }
-
- continue
- }
-
- if snap := eff.GetSnapshot(); snap != nil {
- verdicts.harvest(t, snap)
-
- if snap.GetPrevSnapshot() != nil {
- stack = append(stack, r(snap.GetPrevSnapshot()))
- }
-
- if snap.GetState() == nil {
- for _, dep := range eff.GetDeps() {
- stack = append(stack, r(dep))
- }
- }
-
- continue
}
+ }
- for _, dep := range eff.GetDeps() {
- stack = append(stack, r(dep))
- }
+ if err != nil && strict {
+ return nil, nil, err
}
}
@@ -1690,6 +1807,79 @@ func (e *Engine) losersOnKey(key string, extraTips []Tip, snapshotVerdicts verdi
}
func (e *Engine) losersOnKeyAt(key string, tips, extraTips []Tip, snapshotVerdicts verdictMap, isInvisible func(string) bool, reportConflicts bool) (map[string]struct{}, int) {
+ losers, walked, _ := e.forkChoicesOnKeyAt(key, tips, extraTips, snapshotVerdicts, isInvisible, reportConflicts, nil)
+
+ return losers, walked
+}
+
+// losersOnKeysAt uses the existing per-key walks to discover conflict edges,
+// then resolves them in hash priority across the whole transaction closure.
+// A bind that loses on one key cannot eliminate another bind on a different
+// key. Cached verdicts skip work; they do not change this ordering.
+func (e *Engine) losersOnKeysAt(anchors map[string][]Tip, tipsFor func(string) []Tip, verdicts verdictMap, isInvisible func(string) bool, report bool, packaged ...map[Tip]*pb.Effect) (map[string]struct{}, map[string]int) {
+ competitors := make(map[string][]string)
+ walked := make(map[string]int, len(anchors))
+ losses := make(map[string]bool)
+
+ for key, extra := range anchors {
+ _, count, observed := e.forkChoicesOnKeyAt(key, tipsFor(key), extra, verdicts, isInvisible, report, func(winner, loser string) {
+ competitors[loser] = append(competitors[loser], winner)
+ }, packaged...)
+ walked[key] = count
+ // Indexes may advance after closure discovery. Preserve cached losses
+ // found by this walk, without retaining its provisional pairwise losers.
+ for txn, entry := range observed {
+ if entry.verdict == pb.Verdict_LOST {
+ losses[txn] = true
+ }
+ }
+ }
+
+ for txn, entry := range verdicts {
+ if entry.verdict == pb.Verdict_LOST {
+ losses[txn] = true
+ }
+ }
+
+ var loses func(string) bool
+
+ loses = func(txn string) bool {
+ if lost, known := losses[txn]; known {
+ return lost
+ }
+ // Edges always point to strictly smaller fork-choice hashes, so this
+ // recursion is acyclic. Each transaction is evaluated once, after all
+ // higher-priority competitors it needs have been evaluated.
+ lost := false
+
+ for _, other := range competitors[txn] {
+ if !loses(other) {
+ lost = true
+
+ break
+ }
+ }
+
+ losses[txn] = lost
+
+ return lost
+ }
+ losers := make(map[string]struct{})
+
+ for txn := range competitors {
+ loses(txn)
+ }
+
+ for txn, lost := range losses {
+ if lost {
+ losers[txn] = struct{}{}
+ }
+ }
+
+ return losers, walked
+}
+
+func (e *Engine) forkChoicesOnKeyAt(key string, tips, extraTips []Tip, snapshotVerdicts verdictMap, isInvisible func(string) bool, reportConflicts bool, compete func(winner, loser string), packaged ...map[Tip]*pb.Effect) (map[string]struct{}, int, verdictMap) {
losers := make(map[string]struct{})
startTips := append([]Tip(nil), tips...)
@@ -1711,7 +1901,7 @@ func (e *Engine) losersOnKeyAt(key string, tips, extraTips []Tip, snapshotVerdic
}
if len(startTips) == 0 {
- return losers, 0
+ return losers, 0, snapshotVerdicts
}
// Local verdict map, seeded from any verdicts the caller harvested via
@@ -1729,81 +1919,46 @@ func (e *Engine) losersOnKeyAt(key string, tips, extraTips []Tip, snapshotVerdic
var binds []seenBind
- visited := make(map[Tip]bool)
-
- stack := append([]Tip(nil), startTips...)
walked := 0
-
- for len(stack) > 0 {
- t := stack[len(stack)-1]
- stack = stack[:len(stack)-1]
-
- if visited[t] {
- continue
- }
-
- visited[t] = true
-
- eff, err := e.getEffect(key, t)
- if err != nil {
- continue
- }
-
+ // Preserve this caller's tolerant handling of unavailable evidence.
+ visit := func(t Tip, eff *pb.Effect) {
walked++
- if snap := eff.GetSnapshot(); snap != nil {
- verdicts.harvest(t, snap)
-
- if snap.GetPrevSnapshot() != nil {
- stack = append(stack, r(snap.GetPrevSnapshot()))
- }
-
- if snap.GetState() == nil {
- for _, dep := range eff.GetDeps() {
- stack = append(stack, r(dep))
- }
- }
-
- continue
+ if snap := eff.GetSnapshot(); snap != nil && len(packaged) == 0 {
+ _ = e.harvestProofVerdicts(t, eff, verdicts)
}
- if bind := eff.GetTxnBind(); bind != nil {
- if isInvisible == nil || !isInvisible(eff.GetTxnId()) {
- var newTip Tip
-
- for _, kb := range bind.GetKeys() {
- if string(kb.GetKey()) == key {
- newTip = r(kb.GetNewTip())
-
- break
- }
- }
-
- var zero Tip
- if newTip != zero {
+ if bind := eff.GetTxnBind(); bind != nil && (isInvisible == nil || !isInvisible(eff.GetTxnId())) {
+ for _, kb := range bind.GetKeys() {
+ if string(kb.GetKey()) == key && r(kb.GetNewTip()) != (Tip{}) {
binds = append(binds, seenBind{
txnID: eff.GetTxnId(),
bindOffset: t,
- newTip: newTip,
+ newTip: r(kb.GetNewTip()),
hash: eff.GetForkChoiceHash(),
})
- }
- }
-
- for _, kb := range bind.GetKeys() {
- if string(kb.GetKey()) == key {
- stack = append(stack, r(kb.GetNewTip()))
break
}
}
+ }
+ }
- continue
+ load := func(t Tip) (*pb.Effect, error) {
+ if len(packaged) != 0 {
+ return packaged[0][t], nil
}
- for _, dep := range eff.GetDeps() {
- stack = append(stack, r(dep))
+ return e.getEffect(key, t)
+ }
+ if len(packaged) != 0 {
+ // Explicit proof membership, not a value frontier. Never expand data
+ // ancestry or trust the cached labels when recomputing this proof.
+ for _, t := range startTips {
+ visit(t, packaged[0][t])
}
+ } else {
+ _ = e.walkBindEvidence(key, startTips, visit)
}
type bindWithSource struct {
@@ -1834,20 +1989,21 @@ func (e *Engine) losersOnKeyAt(key string, tips, extraTips []Tip, snapshotVerdic
}
if len(candidates) < 2 {
- return losers, walked
+ return losers, walked, verdicts
}
- // Discover stronger competitors first so unreported losers can stop
- // after the first real conflict. Losing candidates remain competitors:
- // removing them would change the existing fork-choice semantics.
+ // Discover higher-priority competitors first. A single-key evaluation
+ // skips eliminated binds; a cross-key evaluation collects the conflict
+ // edges and resolves them after every key's evidence has been gathered.
slices.SortFunc(candidates, func(a, b seenBind) int { return bytes.Compare(a.hash, b.hash) })
candidateTips := make([]Tip, len(candidates))
for i, b := range candidates {
- candidateTips[i] = b.newTip
+ candidateTips[i] = b.bindOffset
}
ancestry := newCausalIndex(e, key)
+ ancestry.load = load
ancestry.add(candidateTips...)
// Check reporting eligibility once per envelope for this scan, and defer
@@ -1882,7 +2038,7 @@ func (e *Engine) losersOnKeyAt(key string, tips, extraTips []Tip, snapshotVerdic
proof := func(i int) *evidence {
if proofs[i] == nil {
b := candidates[i]
- obs, rw := e.collectTxnEffectsOnKey(b.txnID, key, b.newTip)
+ obs, rw := collectTxnEffects(b.txnID, key, load, b.newTip)
proofs[i] = &evidence{obs: obs, rw: rw}
}
@@ -1897,14 +2053,22 @@ candidateLoop:
for i := range candidates {
b := &candidates[i]
for _, chain := range chains {
- first, last := ancestry.concurrentRange(b.newTip, chain, candidateTips)
+ first, last := ancestry.concurrentRange(b.bindOffset, chain, candidateTips)
for _, j := range chain[first:last] {
if i == j {
continue
}
other := &candidates[j]
- if ancestry.cyclic && (ancestry.reaches(b.newTip, other.newTip) || ancestry.reaches(other.newTip, b.newTip)) {
+ _, otherLost := losers[other.txnID]
+ otherLost = otherLost && compete == nil
+ // Conflict diagnostics still report every incompatible pair
+ // with cached WON evidence, even if one is now eliminated.
+ if otherLost && (!reportable[b.bindOffset] || !reportable[other.bindOffset]) {
+ continue
+ }
+
+ if ancestry.cyclic && (ancestry.reaches(b.bindOffset, other.bindOffset) || ancestry.reaches(other.bindOffset, b.bindOffset)) {
continue
}
@@ -1921,7 +2085,13 @@ candidateLoop:
}
}
- losers[b.txnID] = struct{}{}
+ if compete != nil {
+ compete(other.txnID, b.txnID)
+ }
+
+ if !otherLost {
+ losers[b.txnID] = struct{}{}
+ }
canReport := reportable[b.bindOffset]
if canReport && reportable[other.bindOffset] {
@@ -1942,7 +2112,7 @@ candidateLoop:
// Reporting needs every competing WON pair, even after this
// candidate has already lost to another transaction.
- if !canReport {
+ if !canReport && compete == nil {
continue candidateLoop
}
}
@@ -1987,7 +2157,7 @@ candidateLoop:
"pairwise_candidates", len(candidates))
}
- return losers, walked
+ return losers, walked, verdicts
}
// on the given key. Walks from the bind's NewTip following deps until it
diff --git a/effects/snapshot_batch.go b/effects/snapshot_batch.go
index 8bd10ac..ee660da 100644
--- a/effects/snapshot_batch.go
+++ b/effects/snapshot_batch.go
@@ -127,6 +127,10 @@ func (v *snapshotReadView) valid() bool {
// discover applies the same closure and arbitration rules to live and historical
// frontiers. Historical views never report conflicts or consult live visibility.
func (v *snapshotReadView) discover(keys []string, frontier func(string) ([]Tip, error)) error {
+ return v.discoverWithEvidence(keys, frontier, nil)
+}
+
+func (v *snapshotReadView) discoverWithEvidence(keys []string, frontier func(string) ([]Tip, error), live *snapshotReadView) error {
var err error
v.anchors, v.verdicts, err = v.engine.bindKeyClosureFrom(keys, frontier, true)
@@ -134,25 +138,49 @@ func (v *snapshotReadView) discover(keys []string, frontier func(string) ([]Tip,
return err
}
- for txn, entry := range v.verdicts {
- if entry.verdict == pb.Verdict_LOST {
- if v.lost == nil {
- v.lost = make(map[string]struct{})
+ anchors, verdicts := v.anchors, v.verdicts
+ tipsFor := func(key string) []Tip {
+ return v.frontiers[key].resolved
+ }
+
+ if live != nil {
+ // Temporal selection limits values, not the evidence needed to decide
+ // whether an included bind survives. Keep the live competition across
+ // all closure keys, including competitors of competitors. These tips
+ // must never enter v.frontiers or v.anchors: reconstruction uses those.
+ anchors = make(map[string][]Tip, len(v.anchors)+len(live.anchors))
+ maps.Copy(anchors, v.anchors)
+
+ for key, tips := range live.anchors {
+ anchors[key] = slices.Concat(anchors[key], tips)
+ }
+
+ tipsFor = func(key string) []Tip {
+ return slices.Concat(v.frontiers[key].resolved, live.frontiers[key].resolved)
+ }
+ verdicts = make(verdictMap, len(live.verdicts)+len(v.verdicts))
+ maps.Copy(verdicts, live.verdicts)
+
+ for txn, entry := range v.verdicts {
+ // Equivalent proofs must keep the historical source: membership
+ // checks require it to belong to the selected value history.
+ if existing := verdicts[txn]; entry.proven && existing.proven && entry.verdict == existing.verdict {
+ verdicts[txn] = entry
+
+ continue
}
- v.lost[txn] = struct{}{}
+ verdicts.retain(txn, entry)
}
+
+ v.verdicts = verdicts
}
- for key := range v.anchors {
- losers, walked := v.engine.losersOnKeyAt(key, v.frontiers[key].resolved, v.anchors[key], v.verdicts, v.isInvisible, !v.historical)
+ var walkedByKey map[string]int
+ v.lost, walkedByKey = v.engine.losersOnKeysAt(anchors, tipsFor, verdicts, v.isInvisible, !v.historical)
+ for _, walked := range walkedByKey {
v.walked += walked
- if v.lost == nil {
- v.lost = losers
- } else {
- maps.Copy(v.lost, losers)
- }
}
return nil
diff --git a/effects/snapshot_evidence_boundary_test.go b/effects/snapshot_evidence_boundary_test.go
new file mode 100644
index 0000000..7092bc4
--- /dev/null
+++ b/effects/snapshot_evidence_boundary_test.go
@@ -0,0 +1,340 @@
+package effects
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "google.golang.org/protobuf/proto"
+)
+
+// A snapshot covers its own frontier, not a competing transaction discovered
+// through another key. Adding that bind as an anchor moves the common boundary
+// into the snapshot's ancestry. No cached verdict is required for fork choice.
+func TestBindEvidence_CompetitorBelowSnapshot(t *testing.T) {
+ for _, snapshotFirst := range []bool{false, true} {
+ name := "competitor-first"
+ if snapshotFirst {
+ name = "snapshot-first"
+ }
+
+ t.Run(name, func(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ base := log.putEffect(&pb.Effect{
+ Key: []byte("winner"),
+ // History below the actual common snapshot may be reclaimed.
+ Deps: toPbRefs([]Tip{{99, 1}}),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ State: &pb.ReducedEffect{Scalar: scalarInsertRaw([]byte("base"))},
+ }},
+ })
+ alice := closureBind(log, "alice", 1, []string{"winner", "payout:alice"}, map[string]Tip{"winner": base})
+ bob := closureBind(log, "bob", 2, []string{"winner", "payout:bob"}, map[string]Tip{"winner": base})
+ snapshot := log.putEffect(&pb.Effect{
+ Key: []byte("winner"), Deps: toPbRefs([]Tip{alice}),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ State: &pb.ReducedEffect{Scalar: scalarInsertRaw([]byte("alice"))},
+ }},
+ })
+
+ // Prove the distinction independently of the evidence walks.
+ for _, tc := range []struct {
+ tips []Tip
+ lca Tip
+ }{{[]Tip{snapshot}, snapshot}, {[]Tip{snapshot, bob}, base}} {
+ d := newDag(e, "winner", "")
+ if err := d.prepare(tc.tips); err != nil {
+ t.Fatal(err)
+ }
+
+ if d.lcaTip != tc.lca {
+ t.Fatalf("tips %v: LCA = %v, want %v", tc.tips, d.lcaTip, tc.lca)
+ }
+ }
+
+ frontiers := map[string][]Tip{
+ "winner": {snapshot},
+ "payout:alice": {alice},
+ "payout:bob": {bob},
+ }
+
+ keys := []string{"winner", "payout:bob"}
+ if snapshotFirst {
+ // bindKeyClosureFrom processes its key queue from the end.
+ slices.Reverse(keys)
+ }
+
+ anchors, verdicts, err := e.bindKeyClosureFrom(keys, func(key string) ([]Tip, error) {
+ return frontiers[key], nil
+ }, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !slices.Contains(anchors["payout:alice"], alice) {
+ t.Errorf("did not discover Alice below the former LCA: anchors=%v", anchors)
+ }
+
+ if len(verdicts) != 0 {
+ t.Fatalf("fixture unexpectedly supplies cached verdicts: %v", verdicts)
+ }
+
+ lost, _ := e.losersOnKeysAt(anchors, func(key string) []Tip {
+ return frontiers[key]
+ }, verdicts, nil, false)
+ if _, ok := lost["bob"]; !ok || len(lost) != 1 {
+ t.Errorf("fork choice = %v, want only Bob to lose", lost)
+ }
+
+ view := &snapshotReadView{anchors: anchors, verdicts: verdicts, lost: lost, historical: true}
+
+ result, _, err := e.reconstructWithView("payout:bob", []Tip{bob}, "", false, view)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if result != nil {
+ t.Errorf("reconstructed losing transaction's payout: %v", result)
+ }
+ })
+ }
+}
+
+func TestBindEvidence_ConcurrentSnapshots(t *testing.T) {
+ for _, cached := range []bool{false, true} {
+ t.Run(fmt.Sprintf("cached-verdicts=%t", cached), func(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ state := func(deps ...Tip) Tip {
+ return log.putEffect(&pb.Effect{
+ Key: []byte("winner"), Deps: toPbRefs(deps),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ State: &pb.ReducedEffect{Scalar: scalarInsertRaw([]byte("alice"))},
+ }},
+ })
+ }
+ base := state(Tip{99, 1}) // reclaimed history
+ alice := closureBind(log, "alice", 1, []string{"winner"}, map[string]Tip{"winner": base})
+ bob := closureBind(log, "bob", 2, []string{"winner"}, map[string]Tip{"winner": base})
+
+ branch := state(alice, bob)
+ if cached {
+ branch = log.putEffect(&pb.Effect{
+ Key: []byte("winner"), Deps: toPbRefs([]Tip{branch}),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"alice": pb.Verdict_WON, "bob": pb.Verdict_LOST},
+ PrevSnapshot: toPbRef(branch),
+ }},
+ })
+ }
+
+ noop := log.putEffect(&pb.Effect{
+ Key: []byte("winner"), Deps: toPbRefs([]Tip{branch}),
+ Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}},
+ })
+ tips := []Tip{state(noop), state(base)}
+
+ anchors, verdicts, err := e.bindKeyClosureFrom([]string{"winner"}, func(string) ([]Tip, error) {
+ return tips, nil
+ }, true)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ lost, _ := e.losersOnKeysAt(anchors, func(string) []Tip { return tips }, verdicts, nil, false)
+ if _, ok := lost["bob"]; !ok || len(lost) != 1 {
+ t.Fatalf("losers=%v, want only Bob", lost)
+ }
+
+ view := &snapshotReadView{anchors: anchors, verdicts: verdicts, lost: lost, historical: true}
+
+ result, _, err := e.reconstructWithView("winner", tips, "", false, view)
+ if err != nil || string(result.GetScalar().GetRaw()) != "alice" {
+ t.Fatalf("reconstructed %v, error=%v; want Alice", result, err)
+ }
+ })
+ }
+}
+
+func TestBindEvidence_VerdictShortcutDoesNotExpandFoldedData(t *testing.T) {
+ for _, resident := range []bool{false, true} {
+ t.Run(fmt.Sprintf("resident=%t", resident), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ old, verdict, snapshot := Tip{7, 1}, Tip{7, 2}, Tip{7, 3}
+ metadata := &pb.Effect{
+ Key: []byte("k"), Deps: toPbRefs([]Tip{old}),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"lost": pb.Verdict_LOST},
+ }},
+ }
+ state := &pb.Effect{
+ Key: []byte("k"), Deps: toPbRefs([]Tip{verdict}),
+ Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ State: &pb.ReducedEffect{}, PrevSnapshot: toPbRef(verdict),
+ }},
+ }
+ putCutEffects(t, e, map[Tip]*pb.Effect{verdict: metadata})
+
+ if resident {
+ putCutEffects(t, e, map[Tip]*pb.Effect{snapshot: state})
+ }
+
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ if tip != snapshot {
+ t.Fatalf("fetched folded data at %v", tip)
+ }
+
+ return wireEffect(state), nil
+ }}
+
+ _, verdicts, err := e.bindKeyClosureFrom([]string{"k"}, func(string) ([]Tip, error) {
+ return []Tip{snapshot}, nil
+ }, true)
+ if err != nil || verdicts["lost"].verdict != pb.Verdict_LOST {
+ t.Fatalf("verdicts=%v, error=%v; want cached loss without folded data", verdicts, err)
+ }
+ })
+ }
+}
+
+func TestBindEvidence_ReclaimedPredecessorDoesNotBreakBatch(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "k")
+
+ winner := closureBind(log, "winner", 1, []string{"k"}, nil)
+ loser := closureBind(log, "loser", 2, []string{"k"}, nil)
+
+ proof, err := e.bindProof(winner, map[string]Tip{"loser": loser}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ var previous Tip
+
+ snapshots := make([]Tip, 0, 3)
+
+ for range 3 {
+ eff := bindEvidenceSnapshot("k", proof)
+
+ eff.GetSnapshot().State = &pb.ReducedEffect{Scalar: scalarInsertRaw([]byte("100"))}
+ if previous != (Tip{}) {
+ eff.Deps = toPbRefs([]Tip{previous})
+ eff.GetSnapshot().PrevSnapshot = toPbRef(previous)
+ }
+
+ previous = log.putEffect(eff)
+ snapshots = append(snapshots, previous)
+ }
+
+ e.effectCache.releaseEvidence(evidenceTips(proof))
+ e.updateIndex("k", nil, previous)
+
+ before, err := e.GetSnapshots(context.Background(), []string{"k"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ d := newDag(e, "k", "")
+
+ d.compact = true
+ if err := d.prepare([]Tip{previous}); err != nil {
+ t.Fatal(err)
+ }
+
+ e.reclaimUnreferenced()
+
+ if _, ok := e.effectCache.Get(snapshots[0]); ok {
+ t.Fatal("oldest predecessor was not reclaimed")
+ }
+
+ for _, tip := range snapshots[1:] {
+ if _, ok := e.effectCache.Get(tip); !ok {
+ t.Fatalf("required snapshot was reclaimed: %v", tip)
+ }
+ }
+
+ after, err := e.GetSnapshots(context.Background(), []string{"k"})
+ if err != nil {
+ t.Fatalf("reclaimed predecessor broke readable batch: %v", err)
+ }
+
+ if len(after) != 1 || !proto.Equal(before[0], after[0]) {
+ t.Fatalf("compaction changed value: before=%v after=%v", before, after)
+ }
+
+ _, verdicts, err := e.bindKeyClosureFrom([]string{"k"}, func(string) ([]Tip, error) {
+ return []Tip{previous}, nil
+ }, true)
+ if err != nil || !verdicts["winner"].proven || !verdicts["loser"].proven ||
+ verdicts["winner"].verdict != pb.Verdict_WON || verdicts["loser"].verdict != pb.Verdict_LOST {
+ t.Fatalf("compaction lost retained bind proof: verdicts=%v err=%v", verdicts, err)
+ }
+}
+
+func TestBindEvidence_PredecessorFetchFailure(t *testing.T) {
+ transport := errors.New("peer unavailable")
+ for _, failure := range []error{nil, ErrEffectNotFound, transport} {
+ for _, required := range []bool{false, true} {
+ t.Run(fmt.Sprintf("failure=%v/required=%t", failure, required), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+
+ old, current := Tip{7, 1}, Tip{7, 2}
+ metadata := cutEffect(old)
+ metadata.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"lost": pb.Verdict_LOST},
+ }}
+ snapshot := cutEffect(current, old)
+
+ snapshot.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ PrevSnapshot: toPbRef(old), State: &pb.ReducedEffect{},
+ }}
+ if required {
+ // Without a state boundary, old is a required value dependency
+ // as well as the shortcut target. Its failure must survive.
+ snapshot.GetSnapshot().State = nil
+ }
+
+ putCutEffects(t, e, map[Tip]*pb.Effect{current: snapshot})
+ e.broadcaster = &cutBroadcaster{load: func(tip Tip) ([]byte, error) {
+ if tip != old {
+ t.Fatalf("unexpected fetch: %v", tip)
+ }
+
+ if failure != nil {
+ return nil, failure
+ }
+
+ return wireEffect(metadata), nil
+ }}
+
+ _, verdicts, err := e.bindKeyClosureFrom([]string{"k"}, func(string) ([]Tip, error) {
+ return []Tip{current}, nil
+ }, true)
+ if required && failure != nil {
+ if !errors.Is(err, failure) {
+ t.Fatalf("required dependency failure lost: got %v, want %v", err, failure)
+ }
+
+ return
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if failure == nil && verdicts["lost"].verdict != pb.Verdict_LOST {
+ t.Fatalf("available predecessor verdict discarded: %v", verdicts)
+ }
+ })
+ }
+ }
+}
diff --git a/effects/subscription_frontier_test.go b/effects/subscription_frontier_test.go
new file mode 100644
index 0000000..44f55c9
--- /dev/null
+++ b/effects/subscription_frontier_test.go
@@ -0,0 +1,154 @@
+package effects
+
+import (
+ "fmt"
+ "slices"
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+ "github.com/swytchdb/engine/keytrie"
+)
+
+// Observing a subscription does not add a data branch beside the snapshot.
+// Its folded history is deliberately absent from both nodes and the transport.
+func TestSubscriptionObservationSnapshotBoundary(t *testing.T) {
+ for _, tips := range [][]string{{"d"}, {"sub", "c", "d"}, {"s", "d", "sub"}} {
+ for _, reverseTips := range []bool{false, true} {
+ for _, reverseDeps := range []bool{false, true} {
+ for _, duplicates := range []bool{false, true} {
+ t.Run(fmt.Sprintf("tips-%v/reverse-%t/deps-%t/duplicates-%t", tips, reverseTips, reverseDeps, duplicates), func(t *testing.T) {
+ checkNackFrontier(t, nackFrontierCase{
+ nodes: []nackFrontierNode{
+ {"sub", "subscription", nil, 0},
+ {"s", "snapshot", []string{"pruned"}, 10},
+ {"c", "data", []string{"s"}, 1},
+ {"d", "data", []string{"c", "sub"}, 1},
+ },
+ tips: tips, lca: "s", value: 12,
+ required: []string{"sub", "c"}, exhausted: []string{"pruned"},
+ }, reverseTips, reverseDeps, duplicates)
+ })
+ }
+ }
+ }
+ }
+}
+
+func TestSubscriptionMetadataChainPreservesDataRoot(t *testing.T) {
+ for _, reverseTips := range []bool{false, true} {
+ for _, reverseDeps := range []bool{false, true} {
+ t.Run(fmt.Sprintf("reverse-%t/deps-%t", reverseTips, reverseDeps), func(t *testing.T) {
+ checkNackFrontier(t, nackFrontierCase{
+ nodes: []nackFrontierNode{
+ {"sub", "subscription", nil, 0},
+ {"observed", "subscription", []string{"sub"}, 0},
+ {"a", "data", []string{"observed"}, 1},
+ {"b", "data", nil, 2},
+ {"s", "snapshot", []string{"b"}, 2},
+ },
+ tips: []string{"a", "s"}, value: 3, required: []string{"b"},
+ }, reverseTips, reverseDeps, false)
+ })
+ }
+ }
+}
+
+func TestSubscriptionCarriesObservedData(t *testing.T) {
+ for _, tips := range [][]string{{"d"}, {"sub", "d"}, {"sub"}} {
+ for _, reverseTips := range []bool{false, true} {
+ t.Run(fmt.Sprintf("tips-%v/reverse-%t", tips, reverseTips), func(t *testing.T) {
+ value := int64(11)
+ if len(tips) == 1 && tips[0] == "sub" {
+ value = 10
+ }
+
+ checkNackFrontier(t, nackFrontierCase{
+ nodes: []nackFrontierNode{
+ {"s", "snapshot", []string{"pruned"}, 10},
+ {"sub", "subscription", []string{"s"}, 0},
+ {"d", "data", []string{"sub"}, 1},
+ },
+ tips: tips, lca: "s", value: value,
+ required: []string{"s", "sub"}, exhausted: []string{"pruned"},
+ }, reverseTips, false, false)
+ })
+ }
+ }
+}
+
+func TestSubscriptionObservationNormalization(t *testing.T) {
+ for _, unsubscribe := range []bool{false, true} {
+ for _, reverseDeps := range []bool{false, true} {
+ for _, arrival := range []string{"normalize", "remote", "backfill"} {
+ t.Run(fmt.Sprintf("unsubscribe-%t/reverse-%t/%s", unsubscribe, reverseDeps, arrival), func(t *testing.T) {
+ e := newSnapshotEngine(nil)
+ preSubscribe(e, "k")
+
+ bc := &frontierBroadcaster{fetchable: make(map[Tip][]byte), fetched: make(map[Tip]bool)}
+ e.broadcaster = bc
+ u, s, c, d, pruned := Tip{7, 1}, Tip{7, 2}, Tip{7, 3}, Tip{7, 4}, Tip{7, 99}
+ sub := frontierEffect(u[1])
+
+ sub.Kind = &pb.Effect_Subscription{Subscription: &pb.SubscriptionEffect{SubscriberNodeId: 7, Unsubscribe: unsubscribe}}
+ if unsubscribe {
+ // Unsubscribes remain transparent to the data walk, but
+ // their observed data dependencies must still be followed.
+ sub.Deps = toPbRefs([]Tip{c})
+ }
+
+ snap := frontierEffect(s[1], pruned[1])
+ snap.Kind = &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{State: &pb.ReducedEffect{
+ Scalar: scalarInsertInt(pb.MergeRule_ADDITIVE_INT, 10),
+ }}}
+
+ child := frontierEffect(d[1], c[1], u[1])
+ if unsubscribe {
+ child.Deps = toPbRefs([]Tip{u})
+ }
+
+ if reverseDeps {
+ slices.Reverse(child.GetDeps())
+ }
+
+ for tip, eff := range map[Tip]*pb.Effect{u: sub, s: snap, c: frontierEffect(c[1], s[1]), d: child} {
+ if err := e.effectCache.Put(tip, eff); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ e.index.Insert("k", nil, keytrie.NewTipSet(u, c))
+
+ for range 2 { // retries must preserve the same frontier
+ switch arrival {
+ case "normalize":
+ tips, _, err := NormalizeCloudTipCandidates("k", []Tip{u, c, d}, nil, func(tip Tip) (*pb.Effect, error) {
+ return e.getEffect("k", tip)
+ })
+ if err != nil || !slices.Equal(tips, []Tip{d}) {
+ t.Fatalf("normalization = %v, %v; want [%v], nil", tips, err, d)
+ }
+ case "remote":
+ if _, err := e.HandleRemote(&pb.OffsetNotify{Origin: toPbRef(d), EffectData: wireEffect(child)}); err != nil {
+ t.Fatal(err)
+ }
+
+ requireFrontier(t, e, d)
+ case "backfill":
+ if err := e.handleBackfill(&pb.OffsetNotify{Origin: toPbRef(d), EffectData: wireEffect(child)}); err != nil {
+ t.Fatal(err)
+ }
+
+ requireFrontier(t, e, d)
+ }
+ }
+
+ // Ingestion exhausts the missing boundary. Cloud normalization
+ // still uses the reconstruction walk and stops at its LCA.
+ if want := arrival != "normalize"; bc.fetched[pruned] != want {
+ t.Fatalf("boundary fetch=%t, want %t for %s", bc.fetched[pruned], want, arrival)
+ }
+ })
+ }
+ }
+ }
+}
diff --git a/effects/subscription_readiness_test.go b/effects/subscription_readiness_test.go
new file mode 100644
index 0000000..f4bf058
--- /dev/null
+++ b/effects/subscription_readiness_test.go
@@ -0,0 +1,188 @@
+/*
+ * Copyright 2026 Swytch Labs BV
+ *
+ * This file is part of Swytch.
+ *
+ * Swytch is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Swytch is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Swytch. If not, see .
+ *
+ * Additional permissions under GNU AGPL version 3 section 7 apply to
+ * this file. See the NOTICE.md file distributed with this source code
+ * for the current set of additional permissions.
+ */
+
+package effects
+
+import (
+ "errors"
+ "sync/atomic"
+ "testing"
+ "testing/synctest"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+type gatedBootstrapBroadcaster struct {
+ mockBroadcaster
+
+ rounds atomic.Int32
+ binds atomic.Int32
+ gates [2]chan struct{}
+}
+
+func (b *gatedBootstrapBroadcaster) ReplicateTo(n *pb.OffsetNotify, _ []byte, _ pb.NodeID) ([]*pb.NackNotify, error) {
+ var effect pb.Effect
+ if err := UnmarshalEffect(n.GetEffectData()[4+len(n.GetKey()):], &effect); err != nil {
+ return nil, err
+ }
+
+ if effect.GetSubscription() != nil {
+ round := b.rounds.Add(1)
+ if round <= 2 {
+ <-b.gates[round-1]
+ }
+ // An empty NACK still identifies a peer subscriber. No ancestry is needed.
+ return []*pb.NackNotify{{Key: n.GetKey()}}, nil
+ }
+
+ if effect.GetTxnBind() != nil {
+ b.binds.Add(1)
+ }
+
+ return nil, nil
+}
+
+func (b *gatedBootstrapBroadcaster) ReplicateMarshalled(n *pb.OffsetNotify, _ []byte, target pb.NodeID) ([]*pb.NackNotify, error) {
+ return b.ReplicateTo(n, n.GetEffectData(), target)
+}
+
+func TestAsyncSubscriptionRequiresDiscoveryBeforeCommit(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ b := &gatedBootstrapBroadcaster{peerIDs: []pb.NodeID{2}, allRegionPeersReachable: true, gates: [2]chan struct{}{make(chan struct{}), make(chan struct{})}}
+ e := newTestEngine(b)
+ e.safety.Store(&safetyMap{defaultMode: SafeMode})
+
+ c := e.NewContext()
+ if err := c.Watch("fresh"); err != nil {
+ t.Fatal(err)
+ }
+
+ synctest.Wait()
+
+ state, _ := e.subscriptions.Load("fresh")
+ select {
+ case <-state.ready:
+ t.Error("async announcement claimed completed discovery")
+ default:
+ }
+ // Another ordinary caller may still use the async subscription.
+ if err := e.ensureSubscribed("fresh"); err != nil {
+ t.Fatal(err)
+ }
+
+ c.BeginTx()
+
+ done := make(chan error, 1)
+
+ go func() {
+ if err := c.Emit(dataEffect("fresh")); err != nil {
+ done <- err
+
+ return
+ }
+
+ done <- c.Flush()
+ }()
+ // Concurrent blocking callers must share the same discovery, not redo it.
+ waiter := make(chan error, 1)
+ go func() { waiter <- e.ensureSubscribedBlocking("fresh") }()
+
+ for round := range 2 {
+ synctest.Wait()
+
+ if got := b.rounds.Load(); got != int32(round+1) {
+ t.Errorf("rounds=%d, want %d", got, round+1)
+ }
+
+ if len(done) != 0 || len(waiter) != 0 || b.binds.Load() != 0 {
+ t.Error("transaction or blocking caller escaped unfinished bootstrap")
+ }
+
+ close(b.gates[round])
+ }
+
+ synctest.Wait()
+
+ if err := <-done; err != nil {
+ t.Fatal(err)
+ }
+
+ if err := <-waiter; err != nil {
+ t.Fatal(err)
+ }
+
+ if b.rounds.Load() != 2 || b.binds.Load() != 1 {
+ t.Errorf("rounds=%d binds=%d, want 2 and 1", b.rounds.Load(), b.binds.Load())
+ }
+
+ select {
+ case <-state.ready:
+ default:
+ t.Error("successful discovery did not mark subscription ready")
+ }
+ })
+}
+
+func TestAsyncSubscriptionUpgradePropagatesPartition(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ b := &gatedBootstrapBroadcaster{peerIDs: []pb.NodeID{2}, allRegionPeersReachable: true, gates: [2]chan struct{}{make(chan struct{}), make(chan struct{})}}
+ e := newTestEngine(b)
+ e.safety.Store(&safetyMap{defaultMode: SafeMode})
+
+ if err := e.ensureSubscribed("fresh"); err != nil {
+ t.Fatal(err)
+ }
+
+ synctest.Wait()
+
+ state, _ := e.subscriptions.Load("fresh")
+
+ done := make(chan error, 1)
+ go func() { done <- e.ensureSubscribedBlocking("fresh") }()
+
+ synctest.Wait()
+
+ b.allRegionPeersReachable = false
+ close(b.gates[0])
+ close(b.gates[1])
+ synctest.Wait()
+
+ if err := <-done; !errors.Is(err, ErrRegionPartitioned) {
+ t.Errorf("upgrade error=%v, want partition", err)
+ }
+
+ if _, ok := e.subscriptions.Load("fresh"); ok {
+ t.Error("failed upgrade retained subscription")
+ }
+
+ select {
+ case <-state.ready:
+ default:
+ t.Error("failed upgrade left waiters blocked")
+ }
+
+ if !state.incomplete.Load() {
+ t.Error("failed upgrade marked ready without recording failure")
+ }
+ })
+}
diff --git a/effects/temporal_ingestion_test.go b/effects/temporal_ingestion_test.go
index d6824e7..9470343 100644
--- a/effects/temporal_ingestion_test.go
+++ b/effects/temporal_ingestion_test.go
@@ -118,10 +118,10 @@ func TestTemporalFetchedHistoryObservedOnce(t *testing.T) {
}
d := newDag(reader, key, "")
- if err := d.prepare([]Tip{newest}); err != nil {
+ if err := d.ingest([]Tip{newest}); err != nil {
t.Fatal(err)
}
- // Preparation fetches the absent parents. The explicit dependency pass
+ // Ingestion fetches the absent parents. The explicit dependency pass
// must finish the child's evidence without sampling another receipt.
beforeCompletion := clockReads.Load()
diff --git a/effects/transaction_snapshot.go b/effects/transaction_snapshot.go
index 7bda5fa..238105d 100644
--- a/effects/transaction_snapshot.go
+++ b/effects/transaction_snapshot.go
@@ -73,9 +73,9 @@ func (s *transactionSnapshot) tips(ctx context.Context, key string) (*keytrie.Ti
return keytrie.NewTipSet(tips...), nil
}
-// contains uses the existing DAG walker. Only when a later-discovered effect
-// needs proof of membership behind a retained state snapshot do we continue
-// behind that snapshot. An unavailable required ancestor is an error, not nil.
+// contains first consults evaluated evidence carried by the frozen history.
+// Only when it does not account for the bind do we search older value ancestry.
+// An unavailable required ancestor is an error, not nil.
func (s *transactionSnapshot) contains(ctx context.Context, key string, target Tip) (bool, error) {
v := s.views[key]
if v == nil {
@@ -85,6 +85,11 @@ func (s *transactionSnapshot) contains(ctx context.Context, key string, target T
frontier := slices.Concat(v.frontiers[key].resolved, v.anchors[key])
seen := make(map[Tip]bool)
+ var proof verdictEntry
+ if eff, ok := s.engine.effectCache.Get(target); ok && eff.GetTxnBind() != nil {
+ proof = v.verdicts[eff.GetTxnId()]
+ }
+
for len(frontier) != 0 {
if err := ctx.Err(); err != nil {
return false, err
@@ -98,6 +103,13 @@ func (s *transactionSnapshot) contains(ctx context.Context, key string, target T
if _, ok := d.nodes[target]; ok {
return true, nil
}
+ // This history already accounts for the bind through retained proof;
+ // its old value-ancestry path need not still exist. A cached label is
+ // insufficient. Nor may live-only evidence extend a frozen view: its
+ // source snapshot must actually belong to the selected history.
+ if proof.proven && d.nodes[proof.snapshotTip] != nil {
+ return true, nil
+ }
frontier = nil
@@ -169,7 +181,7 @@ func (s *transactionSnapshot) member(ctx context.Context, tip Tip, eff *pb.Effec
}
if knownPast {
- return choose(true, "ancestor of frozen frontier"), nil
+ return choose(true, "accounted for by frozen history"), nil
}
lower, upper, err := s.clocks.Difference(s.at, effectSample(eff))
@@ -298,7 +310,7 @@ func (s *transactionSnapshot) prepare(ctx context.Context, requested []string) e
return s.engine.historicalFrontier(ctx, key, tips, func(t Tip, eff *pb.Effect) (bool, error) {
return s.member(ctx, t, eff, visiting)
})
- })
+ }, live)
if err != nil {
return err
}
diff --git a/effects/txn.go b/effects/txn.go
index b052414..cbb0c45 100644
--- a/effects/txn.go
+++ b/effects/txn.go
@@ -24,12 +24,12 @@
package effects
import (
- "bytes"
"errors"
"sync/atomic"
"time"
pb "github.com/swytchdb/engine/cluster/proto"
+ "google.golang.org/protobuf/proto"
)
// ErrTxnAborted is returned by Flush when a transaction loses FWW or
@@ -76,11 +76,14 @@ type pendingTxn struct {
txnID string // transaction ID (matches TxnId on emitted effects)
txnHLC time.Time
originNode pb.NodeID
- forkChoiceHash []byte // precomputed ForkChoiceHash(originNode, txnHLC)
- bindOffset Tip // offset of the TransactionalBindEffect
- keys []pendingTxnKey // read-only after creation
- state atomic.Uint32 // txnState*
- done chan struct{} // closed on decision
+ forkChoiceHash []byte // precomputed ForkChoiceHash(originNode, txnHLC)
+ bindOffset Tip // offset of the TransactionalBindEffect
+ keys []pendingTxnKey // read-only after creation
+ state atomic.Uint32 // txnState*
+ done chan struct{} // closed on decision
+ resolved chan struct{} // closed after evaluation and evidence publication
+ responses []*pb.NackNotify // immutable once resolved closes
+ proofSources []Tip // predecessor snapshots actually used in resolution
}
type pendingTxnKey struct {
@@ -92,13 +95,86 @@ type pendingTxnKey struct {
elementIDs [][]byte // element IDs our tx touches on this key
}
+// hasWritesSince validates an observation against the current key history.
+// NOOPs and snapshots can replace the observed tips without changing the value.
+// Tentative effects do not write a value until their bind arrives. Follow these
+// wrappers to find actual writes, stopping at the observed causal boundary.
+func (e *Engine) hasWritesSince(key string, observed []Tip, txnID string) bool {
+ seen := make(map[Tip]bool, len(observed))
+ base := &pendingTxn{}
+
+ for _, tip := range observed {
+ seen[tip] = true
+ base.keys = append(base.keys, pendingTxnKey{key: key, newTip: tip})
+ }
+
+ current := e.index.Contains(key).Tips()
+ if len(current) == 0 && len(observed) != 0 {
+ return true
+ }
+
+ stack := append([]Tip(nil), current...)
+ for len(stack) > 0 {
+ tip := stack[len(stack)-1]
+ stack = stack[:len(stack)-1]
+
+ if seen[tip] {
+ continue
+ }
+
+ seen[tip] = true
+
+ eff, err := e.getEffect(key, tip)
+ if err != nil {
+ if e.reachesFromTxOnKey(base, &key, tip) {
+ continue
+ }
+
+ return true
+ }
+
+ if eff.GetTxnId() != txnID || txnID == "" {
+ writes := eff.GetTxnId() == "" && (eff.GetData() != nil || eff.GetMeta() != nil)
+ if eff.GetTxnBind() != nil {
+ // The bind, not its pre-bind effects, must be in our past.
+ if e.reachesFromTxOnKey(base, &key, tip) {
+ continue
+ }
+
+ effects, err := e.collectBindEffects(eff, key)
+ if err != nil {
+ return true
+ }
+
+ for _, bound := range effects {
+ writes = writes || bound.GetData() != nil || bound.GetMeta() != nil || bound.GetRowWrite() != nil
+ }
+ }
+
+ if writes {
+ if !e.reachesFromTxOnKey(base, &key, tip) {
+ return true
+ }
+
+ continue
+ }
+ }
+
+ for dep := range keyDeps(key, eff) {
+ stack = append(stack, r(dep))
+ }
+ }
+
+ return false
+}
+
// isRealConflict determines whether a NACK Tip represents a real conflict
// with our pending transaction on the given key.
//
// NACK itself only says "peer's tips diverged from your bind's
// ConsumedTips". Whether that divergence is a real conflict is a
// deterministic function of (our effects, their effects, predicate
-// rules) — the same function evaluateBindForkChoice runs on every
+// rules) — the same predicate and ancestry rules used on every
// node. All call sites must reach the same answer so the client's
// abort/commit signal matches the cluster's view of the bind.
func (e *Engine) isRealConflict(ptxn *pendingTxn, key string, detail *pb.NackTipDetail) bool {
@@ -113,16 +189,49 @@ func (e *Engine) isRealConflict(ptxn *pendingTxn, key string, detail *pb.NackTip
// cannot authorize a commit. Keep the defeated bind anchors from the same pass
// so flushTx publishes the same decisions for wrapped and unwrapped competitors.
func (e *Engine) checkNackConflict(ptxn *pendingTxn, key string, detail *pb.NackTipDetail) (bool, map[string]Tip) {
- queue := []*pb.NackTipDetail{detail}
- visited := make(map[Tip]bool)
+ candidates, _, err := e.nackCandidates(ptxn, key, []*pb.NackTipDetail{detail})
+ if err != nil {
+ return true, nil
+ }
var defeated map[string]Tip
+ for _, cur := range candidates {
+ conflict, txn := e.classifyNackTip(ptxn, key, cur)
+ if conflict {
+ return true, nil
+ }
+
+ if txn != "" {
+ if defeated == nil {
+ defeated = make(map[string]Tip)
+ }
+
+ defeated[txn] = r(cur.GetRef())
+ }
+ }
+
+ return false, defeated
+}
+
+// nackCandidates discovers competing envelopes and cached fork-choice evidence.
+// Discovery does not decide winners. Unrelated keys can supply evidence without
+// subscribing to them or treating their effects as conflicts with our transaction.
+func (e *Engine) nackCandidates(ptxn *pendingTxn, key string, details []*pb.NackTipDetail) ([]*pb.NackTipDetail, verdictMap, error) {
+ queue := append([]*pb.NackTipDetail(nil), details...)
+ visited := make(map[Tip]bool)
+ verdicts := make(verdictMap)
+
+ var (
+ candidates []*pb.NackTipDetail
+ snapshots []Tip
+ )
+
for len(queue) > 0 {
- cur := queue[0]
- queue = queue[1:]
- tip := r(cur.GetRef())
+ cur := queue[len(queue)-1]
+ queue = queue[:len(queue)-1]
+ tip := r(cur.GetRef())
if visited[tip] {
continue
}
@@ -130,27 +239,34 @@ func (e *Engine) checkNackConflict(ptxn *pendingTxn, key string, detail *pb.Nack
visited[tip] = true
if cur.GetIsData() || cur.GetIsBind() {
- conflict, txn := e.classifyNackTip(ptxn, key, cur)
- if conflict {
- return true, nil
- }
+ candidates = append(candidates, cur)
- if txn != "" {
- if defeated == nil {
- defeated = make(map[string]Tip)
+ continue
+ }
+
+ deps := cur.GetDeps()
+
+ var predecessor *pb.EffectRef
+
+ if eff, err := e.getEffect(key, tip); err == nil {
+ if snap := eff.GetSnapshot(); snap != nil {
+ if err := e.harvestProofVerdicts(tip, eff, verdicts); err != nil {
+ return nil, nil, err
}
- defeated[txn] = tip
+ predecessor = snap.GetPrevSnapshot()
}
-
- continue
}
if tip != (Tip{}) && e.reachesFromTxOnKey(ptxn, &key, tip) {
continue
}
- for _, dep := range cur.GetDeps() {
+ if predecessor != nil {
+ snapshots = append(snapshots, r(predecessor))
+ }
+
+ for _, dep := range deps {
dt := r(dep)
if visited[dt] || e.reachesFromTxOnKey(ptxn, &key, dt) {
continue
@@ -158,14 +274,197 @@ func (e *Engine) checkNackConflict(ptxn *pendingTxn, key string, detail *pb.Nack
eff, err := e.getEffect(key, dt)
if err != nil {
- return true, nil
+ if findPendingKey(ptxn, key) != nil {
+ return nil, nil, ErrTxnAmbiguous
+ }
+
+ continue
}
queue = append(queue, nackTipDetail(dt, eff))
}
}
- return false, defeated
+ // PrevSnapshot is a non-owning verdict shortcut. Visit required ancestry
+ // first so a shortcut cannot hide a missing ordinary dependency. Available
+ // predecessors supply verdicts, not another requirement for folded value history.
+ for len(snapshots) > 0 {
+ tip := snapshots[len(snapshots)-1]
+ snapshots = snapshots[:len(snapshots)-1]
+
+ if visited[tip] || e.reachesFromTxOnKey(ptxn, &key, tip) {
+ continue
+ }
+
+ visited[tip] = true
+
+ eff, err := e.getEffect(key, tip)
+ if err != nil || eff.GetSnapshot() == nil {
+ continue
+ }
+
+ // A cache-only predecessor can be reclaimed between loading it and
+ // pinning its proof. Its unavailability is still optional; malformed
+ // proofs and failures on the required-ancestry path remain errors.
+ if err := e.harvestProofVerdicts(tip, eff, verdicts); err != nil && !errors.Is(err, errEffectNotResident) {
+ return nil, nil, err
+ }
+
+ if prev := eff.GetSnapshot().GetPrevSnapshot(); prev != nil {
+ snapshots = append(snapshots, r(prev))
+ }
+ }
+
+ return candidates, verdicts, nil
+}
+
+// evaluateNacks runs only after every subscriber has responded. First collect
+// ALL evidence: a later response can resolve a bind named by an earlier NACK.
+// A better hash grants priority, not victory. Without its resolution the
+// published transaction remains ambiguous, rather than reporting a false abort.
+func (e *Engine) evaluateNacks(ptxn *pendingTxn, nacks []*pb.NackNotify) (map[string]Tip, error) {
+ ptxn.proofSources = nil
+
+ type candidate struct {
+ key string
+ detail *pb.NackTipDetail
+ txn string
+ }
+
+ var candidates []candidate
+
+ verdicts := make(verdictMap)
+
+ for _, nack := range nacks {
+ key := string(nack.GetKey())
+
+ tips, evidence, err := e.nackCandidates(ptxn, key, nack.GetTipDetails())
+ if err != nil {
+ return nil, err
+ }
+
+ for txn, entry := range evidence {
+ if prior, ok := verdicts[txn]; ok && prior.verdict != entry.verdict {
+ return nil, ErrTxnAmbiguous
+ }
+
+ verdicts[txn] = entry
+ }
+
+ if findPendingKey(ptxn, key) == nil {
+ continue
+ }
+
+ for _, tip := range tips {
+ c := candidate{key: key, detail: tip}
+ if tip.GetIsBind() {
+ eff, err := e.getEffect(key, r(tip.GetRef()))
+ if err != nil || eff.GetTxnBind() == nil {
+ return nil, ErrTxnAmbiguous
+ }
+
+ c.txn = eff.GetTxnId()
+ }
+
+ candidates = append(candidates, c)
+ }
+ }
+ // Establish priority before evaluating competing candidates. Predicate-
+ // disjoint binds can coexist regardless of the other transaction's outcome.
+ for _, c := range candidates {
+ if c.detail.GetIsBind() && c.txn != ptxn.txnID &&
+ ForkChoiceLess(c.detail.GetBindForkChoiceHash(), ptxn.forkChoiceHash) &&
+ !e.nackTipOrdered(ptxn, c.key, c.detail) && verdicts[c.txn].verdict == pb.Verdict_VERDICT_UNSPECIFIED {
+ if e.bindPredicatesDisjoint(ptxn, c.key, c.txn, r(c.detail.GetRef())) {
+ continue
+ }
+
+ return nil, ErrTxnAmbiguous
+ }
+ }
+
+ defeated := make(map[string]Tip)
+
+ for _, c := range candidates {
+ if verdicts[c.txn].verdict == pb.Verdict_LOST {
+ ptxn.proofSources = append(ptxn.proofSources, verdicts[c.txn].snapshotTip)
+
+ continue
+ }
+
+ conflict, txn := e.classifyNackTip(ptxn, c.key, c.detail)
+ if conflict {
+ return nil, ErrTxnAborted
+ }
+
+ if txn != "" {
+ defeated[txn] = r(c.detail.GetRef())
+ }
+ }
+
+ return defeated, nil
+}
+
+// priorityPredecessors is called at the bind ingestion/publication boundary.
+// Only local origins defer responses: every origin is a subscriber of its own
+// keys, so the sender must receive their response before evaluating. Other
+// subscribers can answer immediately. All waits point toward a smaller hash.
+func (e *Engine) priorityPredecessors(ref Tip, eff *pb.Effect) []*pendingTxn {
+ var prior []*pendingTxn
+
+ detail := nackTipDetail(ref, eff)
+
+ e.pendingTxns.Range(func(_ Tip, txn *pendingTxn) bool {
+ if txn.resolved == nil || !ForkChoiceLess(txn.forkChoiceHash, detail.GetBindForkChoiceHash()) {
+ return true
+ }
+
+ for _, kb := range eff.GetTxnBind().GetKeys() {
+ key := string(kb.GetKey())
+ if findPendingKey(txn, key) != nil && !e.nackTipOrdered(txn, key, detail) {
+ prior = append(prior, txn)
+
+ break
+ }
+ }
+
+ return true
+ })
+
+ return prior
+}
+
+// finishPriorityResponse waits without nodeMu, then forwards the evidence the
+// predecessor used, including keys the requester need not subscribe to. A
+// bounded wait without evidence leaves the original unresolved NACK intact;
+// evaluateNacks will return an ambiguous result, never infer an abort.
+func (e *Engine) finishPriorityResponse(ref *pb.EffectRef, nacks []*pb.NackNotify, prior []*pendingTxn) []*pb.NackNotify {
+ if len(prior) == 0 {
+ return nacks
+ }
+
+ timer := time.NewTimer(horizonFallbackWait)
+ defer timer.Stop()
+
+ for _, txn := range prior {
+ select {
+ case <-txn.resolved:
+ case <-timer.C:
+ return nacks
+ }
+
+ for _, response := range txn.responses {
+ forwarded := proto.Clone(response).(*pb.NackNotify)
+ forwarded.Conflicting = ref
+ nacks = append(nacks, forwarded)
+ }
+
+ for _, pk := range txn.keys {
+ nacks = append(nacks, e.buildEnrichedNack(pk.key, ref, e.index.Contains(pk.key).Tips()))
+ }
+ }
+
+ return nacks
}
// nackTipOrdered proves that a data effect or bind is causally ordered with our
@@ -177,12 +476,6 @@ func (e *Engine) nackTipOrdered(ptxn *pendingTxn, key string, detail *pb.NackTip
if r(dep) == ptxn.bindOffset {
return true
}
-
- for _, pk := range ptxn.keys {
- if pk.key == key && r(dep) == pk.newTip {
- return true
- }
- }
}
if detail.GetRef() != nil {
@@ -201,17 +494,7 @@ func (e *Engine) nackTipOrdered(ptxn *pendingTxn, key string, detail *pb.NackTip
theirs.keys = append(theirs.keys, pendingTxnKey{key: key, newTip: r(dep)})
}
- if e.reachesFromTxOnKey(&theirs, &key, ptxn.bindOffset) {
- return true
- }
-
- for _, pk := range ptxn.keys {
- if pk.key == key && e.reachesFromTxOnKey(&theirs, &key, pk.newTip) {
- return true
- }
- }
-
- return false
+ return e.reachesFromTxOnKey(&theirs, &key, ptxn.bindOffset)
}
// classifyNackTip decides both sides of fork choice in one pass. Ordered or
@@ -248,16 +531,8 @@ func (e *Engine) classifyNackTip(ptxn *pendingTxn, key string, detail *pb.NackTi
return false, "" // our own bind
}
- if ptxn.txnID != "" && theirTxnID != "" {
- ourStart := collectOurBindTips(ptxn, key)
-
- conflict, bothHadEvidence := e.hasPredicateConflict(
- ptxn.txnID, theirTxnID, key,
- ourStart,
- []Tip{theirBindOffset})
- if bothHadEvidence && !conflict {
- return false, "" // disjoint predicates — can coexist
- }
+ if e.bindPredicatesDisjoint(ptxn, key, theirTxnID, theirBindOffset) {
+ return false, "" // disjoint predicates — can coexist
}
if ForkChoiceLess(detail.GetBindForkChoiceHash(), ptxn.forkChoiceHash) {
@@ -277,10 +552,25 @@ func (e *Engine) classifyNackTip(ptxn *pendingTxn, key string, detail *pb.NackTi
return false, ""
}
- // Non-transactional DataEffect on same key
- pk := findPendingKey(ptxn, key)
+ // Mixing transactional and non-transactional operations has no isolation
+ // guarantee. An unordered ordinary write does not defeat a published bind:
+ // reconstruction cannot derive that transaction loss from competing binds.
+ // Writes observed before publication are still checked by hasWritesSince.
+ return false, ""
+}
+
+// bindPredicatesDisjoint shares the per-key exemption between priority gating
+// and classification. Missing predicate evidence keeps the conservative fallback.
+func (e *Engine) bindPredicatesDisjoint(ptxn *pendingTxn, key, theirTxnID string, theirBindOffset Tip) bool {
+ if ptxn.txnID == "" || theirTxnID == "" {
+ return false
+ }
- return pk != nil && affectsSameData(pk, detail), ""
+ conflict, bothHadEvidence := e.hasPredicateConflict(
+ ptxn.txnID, theirTxnID, key,
+ collectOurBindTips(ptxn, key), []Tip{theirBindOffset})
+
+ return bothHadEvidence && !conflict
}
// reachesFromTx returns true if target is in the DAG ancestry of our
@@ -387,42 +677,6 @@ func collectOurBindTips(ptxn *pendingTxn, key string) []Tip {
return tips
}
-// affectsSameData checks if a competing Tip affects the same data as our
-// transaction's key entry.
-func affectsSameData(pk *pendingTxnKey, detail *pb.NackTipDetail) bool {
- // If our key is read-only (NoopEffect), any SCALAR write is a phantom write conflict
- if pk.readOnly {
- // For KEYED/ORDERED, a noop read doesn't conflict with writes to
- // different elements, but we conservatively treat it as conflict
- // because we can't distinguish which elements were "read".
- return true
- }
-
- // SCALAR: always conflicts (single value per key)
- if pk.collection == pb.CollectionKind_SCALAR || detail.GetCollection() == pb.CollectionKind_SCALAR {
- return true
- }
-
- // ORDERED: any concurrent insert conflicts — position depends on seeing
- // all prior inserts, so concurrent inserts create incompatible orderings.
- if pk.collection == pb.CollectionKind_ORDERED || detail.GetCollection() == pb.CollectionKind_ORDERED {
- return true
- }
-
- // KEYED: compare element IDs — concurrent writes to different fields are fine
- if len(detail.GetElementId()) == 0 {
- return true // no element ID → assume conflict
- }
-
- for _, eid := range pk.elementIDs {
- if bytes.Equal(eid, detail.GetElementId()) {
- return true
- }
- }
-
- return false
-}
-
// findPendingKey finds the pendingTxnKey for a given key name.
func findPendingKey(ptxn *pendingTxn, key string) *pendingTxnKey {
for i := range ptxn.keys {
diff --git a/effects/txn_predicate_priority_test.go b/effects/txn_predicate_priority_test.go
new file mode 100644
index 0000000..3765a5b
--- /dev/null
+++ b/effects/txn_predicate_priority_test.go
@@ -0,0 +1,165 @@
+package effects
+
+import (
+ "errors"
+ "testing"
+ "testing/synctest"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+type predicatePriorityBroadcaster struct {
+ bindPublicationBroadcaster
+
+ sent chan *pb.OffsetNotify
+ allow chan bool
+ responses []*pb.NackNotify
+}
+
+func (b *predicatePriorityBroadcaster) ReplicateMarshalled(n *pb.OffsetNotify, _ []byte, _ pb.NodeID) ([]*pb.NackNotify, error) {
+ b.sent <- n
+
+ if !<-b.allow {
+ return nil, errors.New("injected bind delivery failure")
+ }
+
+ ns, err := b.peer.HandleRemote(n)
+ b.responses = ns
+
+ return ns, err
+}
+
+// Two real Flush calls publish binds before either request is delivered.
+// The stronger transaction's request fails in transport; the weaker one's
+// request is delivered afterwards and receives the unresolved stronger bind.
+func TestPredicateDisjointBindDoesNotRequirePriorityVerdict(t *testing.T) {
+ for _, mode := range []string{"disjoint", "overlap", "missing-evidence", "other-key-overlap"} {
+ t.Run(mode, func(t *testing.T) {
+ synctest.Test(t, func(t *testing.T) {
+ var (
+ es [2]*Engine
+ links [2]*predicatePriorityBroadcaster
+ tx [2]*Context
+ done [2]chan error
+ )
+
+ for i := range es {
+ links[i] = &predicatePriorityBroadcaster{sent: make(chan *pb.OffsetNotify, 1), allow: make(chan bool, 1)}
+ links[i].peerIDs = []pb.NodeID{pb.NodeID(2 - i)}
+ links[i].allRegionPeersReachable = true
+ es[i] = newTxnTestEngine(links[i])
+ es[i].nodeID = pb.NodeID(i + 1)
+ es[i].UpdateSafetyRules(SafeMode, nil)
+
+ for j, key := range []string{"k", "other"} {
+ base := Tip{99, uint64(j + 1)}
+ if err := es[i].effectCache.Put(base, scalarSet(key, "base")); err != nil {
+ t.Fatal(err)
+ }
+
+ es[i].updateIndex(key, nil, base)
+ preSubscribe(es[i], key)
+ es[i].addPeerSubscriber(key, pb.NodeID(2-i))
+ }
+
+ tx[i] = es[i].NewContext()
+ tx[i].BeginTx()
+ done[i] = make(chan error, 1)
+ }
+
+ links[0].peer = es[1]
+ links[1].peer = es[0]
+
+ keys := []string{"k"}
+ if mode == "other-key-overlap" {
+ keys = append(keys, "other")
+ }
+
+ for i, c := range tx {
+ for _, key := range keys {
+ if mode == "missing-evidence" && i == 0 {
+ if err := c.Emit(scalarSet(key, "plain")); err != nil {
+ t.Fatal(err)
+ }
+
+ continue
+ }
+
+ row := int64(i + 1)
+ if mode == "overlap" || key == "other" {
+ row = 1
+ }
+
+ if err := c.Emit(&pb.Effect{Key: []byte(key), Kind: &pb.Effect_Observation{Observation: &pb.ObservationEffect{
+ Predicate: &pb.Predicate{Kind: pb.Predicate_CMP, Col: 0, Op: pb.PredicateCmpOp_CMP_EQ, Literal: &pb.TypedValue{Kind: pb.TypedValue_INT, IntVal: row}},
+ }}}); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := c.Emit(&pb.Effect{Key: []byte(key), Kind: &pb.Effect_RowWrite{RowWrite: &pb.RowWriteEffect{
+ Kind: pb.RowWriteEffect_INSERT, Columns: []*pb.TypedValue{{Kind: pb.TypedValue_INT, IntVal: row}},
+ }}}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+
+ for i := range tx {
+ go func() { done[i] <- tx[i].Flush() }()
+ }
+
+ ns := [2]*pb.OffsetNotify{<-links[0].sent, <-links[1].sent}
+
+ var binds [2]*pb.Effect
+ for i, n := range ns {
+ binds[i], _ = es[i].effectCache.Get(r(n.GetOrigin()))
+ }
+
+ strong := 0
+ if ForkChoiceLess(binds[1].GetForkChoiceHash(), binds[0].GetForkChoiceHash()) {
+ strong = 1
+ }
+
+ weak := 1 - strong
+ links[strong].allow <- false
+
+ if err := <-done[strong]; !errors.Is(err, ErrTxnAmbiguous) {
+ t.Fatalf("transport failure should be ambiguous: %v", err)
+ }
+
+ links[weak].allow <- true
+
+ result := <-done[weak]
+ if mode == "disjoint" {
+ if result != nil {
+ t.Fatalf("disjoint transaction required unrelated priority verdict: %v", result)
+ }
+ } else if !errors.Is(result, ErrTxnAmbiguous) {
+ t.Fatalf("unresolved overlapping or unknown competitor: got %v, want ambiguous", result)
+ }
+
+ if len(links[weak].responses) == 0 {
+ t.Fatal("no real NACK received")
+ }
+
+ ours := &pendingTxn{txnID: binds[weak].GetTxnId(), bindOffset: r(ns[weak].GetOrigin()), forkChoiceHash: binds[weak].GetForkChoiceHash()}
+ for _, kb := range binds[weak].GetTxnBind().GetKeys() {
+ ours.keys = append(ours.keys, pendingTxnKey{key: string(kb.GetKey()), newTip: r(kb.GetNewTip()), consumedTips: fromPbRefs(kb.GetConsumedTips())})
+ }
+
+ conflict, known := es[weak].hasPredicateConflict(ours.txnID, binds[strong].GetTxnId(), "k", collectOurBindTips(ours, "k"), []Tip{r(ns[strong].GetOrigin())})
+ if (mode == "disjoint" || mode == "other-key-overlap") && (conflict || !known) {
+ t.Fatalf("disjoint fixture failed: conflict=%t evidence=%t", conflict, known)
+ }
+
+ if mode == "overlap" && (!conflict || !known) {
+ t.Fatal("overlap fixture failed")
+ }
+
+ if mode == "missing-evidence" && known {
+ t.Fatal("missing evidence fixture failed")
+ }
+ })
+ })
+ }
+}
diff --git a/effects/txn_priority_test.go b/effects/txn_priority_test.go
new file mode 100644
index 0000000..8575e82
--- /dev/null
+++ b/effects/txn_priority_test.go
@@ -0,0 +1,451 @@
+/*
+ * Copyright 2026 Swytch Labs BV
+ *
+ * This file is part of Swytch.
+ *
+ * Swytch is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * Swytch is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with Swytch. If not, see .
+ *
+ * Additional permissions under GNU AGPL version 3 section 7 apply to
+ * this file. See the NOTICE.md file distributed with this source code
+ * for the current set of additional permissions.
+ */
+
+package effects
+
+import (
+ "context"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "slices"
+ "sync"
+ "testing"
+ "time"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+// A and C have no common key. Only B can pass C's evidence to A.
+func TestNackPriorityEvidenceAcrossKeys(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ base := log.putEffect(scalarSet("ab", "base"))
+ a := newTestPendingTxn(pendingTxnKey{key: "ab", newTip: base})
+ a.txnID = "A"
+
+ node := pb.NodeID(2)
+ for !ForkChoiceLess(ComputeForkChoiceHash(node, sTs(100)), a.forkChoiceHash) {
+ node++
+ }
+
+ b := &pb.Effect{
+ Key: []byte("ab"), TxnId: "B", NodeId: uint64(node), Hlc: sTs(100),
+ Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{
+ OriginatorNodeId: uint64(node), TxnHlc: sTs(100),
+ Keys: []*pb.TransactionalBindEffect_KeyBind{
+ {Key: []byte("ab"), NewTip: toPbRef(base)},
+ {Key: []byte("bc"), NewTip: toPbRef(base)},
+ },
+ }},
+ }
+ bt := log.putEffect(b)
+ early := e.buildEnrichedNack("ab", toPbRef(a.bindOffset), []Tip{bt})
+ // Knowing B's hash does not establish a loss for A.
+ if _, err := e.evaluateNacks(a, []*pb.NackNotify{early}); !errors.Is(err, ErrTxnAmbiguous) {
+ t.Fatalf("unresolved B: got %v, want ambiguous", err)
+ }
+
+ for _, outcome := range []pb.Verdict{pb.Verdict_LOST, pb.Verdict_WON} {
+ evidence := log.putEffect(&pb.Effect{Key: []byte("bc"), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"B": outcome, "C": pb.Verdict_WON},
+ }}})
+
+ late := e.buildEnrichedNack("bc", toPbRef(a.bindOffset), []Tip{evidence})
+ for _, responses := range [][]*pb.NackNotify{{early, late}, {late, early}} {
+ _, err := e.evaluateNacks(a, responses)
+ if outcome == pb.Verdict_LOST && err != nil {
+ t.Fatalf("B lost: %v", err)
+ }
+
+ if outcome == pb.Verdict_WON && !errors.Is(err, ErrTxnAborted) {
+ t.Fatalf("B won: %v", err)
+ }
+ }
+ }
+
+ if _, ok := e.subscriptions.Load("bc"); ok {
+ t.Fatal("evaluating forwarded evidence subscribed A to C's key")
+ }
+}
+
+func TestBindResponseWaitsForPriorityEvidence(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ base := log.putEffect(scalarSet("ab", "base"))
+ b := newTestPendingTxn(pendingTxnKey{key: "ab", newTip: base})
+ b.txnID = "B"
+ b.resolved = make(chan struct{})
+
+ node := pb.NodeID(2)
+ for !ForkChoiceLess(b.forkChoiceHash, ComputeForkChoiceHash(node, sTs(100))) {
+ node++
+ }
+
+ incoming := &pb.Effect{
+ Key: []byte("ab"), TxnId: "A", NodeId: uint64(node), Hlc: sTs(100),
+ ForkChoiceHash: ComputeForkChoiceHash(node, sTs(100)),
+ Kind: &pb.Effect_TxnBind{TxnBind: &pb.TransactionalBindEffect{
+ OriginatorNodeId: uint64(node), TxnHlc: sTs(100),
+ Keys: []*pb.TransactionalBindEffect_KeyBind{{Key: []byte("ab"), NewTip: toPbRef(base)}},
+ }},
+ }
+ ref := Tip{uint64(node), 700}
+
+ e.pendingTxns.Store(b.bindOffset, b)
+ e.subscriptions.Store("ab", &subscriptionState{})
+ e.updateIndex("ab", nil, base)
+
+ wire, err := MarshalEffect(incoming)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ result := make(chan []*pb.NackNotify, 1)
+ errs := make(chan error, 1)
+
+ go func() {
+ nacks, err := e.HandleRemote(&pb.OffsetNotify{Key: []byte("ab"), Origin: toPbRef(ref), EffectData: wire})
+ result <- nacks
+
+ errs <- err
+ }()
+
+ deadline := time.After(time.Second)
+
+ for !e.index.Contains("ab").Contains(ref) {
+ select {
+ case <-deadline:
+ t.Fatal("bind was not installed before waiting")
+ default:
+ time.Sleep(time.Millisecond)
+ }
+ }
+
+ select {
+ case <-result:
+ t.Fatal("response preceded B's evaluation")
+ default:
+ }
+ // A new arrival/publication must remain possible while B waits for C.
+ unlocked := make(chan struct{})
+
+ go func() { lock := e.takeNodeLock(); lock.release(); close(unlocked) }()
+
+ select {
+ case <-unlocked:
+ case <-time.After(time.Second):
+ t.Fatal("priority wait held nodeMu")
+ }
+
+ b.finish(txnStateAborted)
+
+ select {
+ case <-result:
+ t.Fatal("local finish is not evidence publication")
+ default:
+ }
+
+ proof := log.putEffect(&pb.Effect{Key: []byte("bc"), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"B": pb.Verdict_LOST, "C": pb.Verdict_WON},
+ }}})
+ b.responses = []*pb.NackNotify{e.buildEnrichedNack("bc", toPbRef(b.bindOffset), []Tip{proof})}
+ close(b.resolved)
+
+ select {
+ case nacks := <-result:
+ if err := <-errs; err != nil {
+ t.Fatal(err)
+ }
+
+ found := false
+
+ for _, n := range nacks {
+ if string(n.GetKey()) == "bc" {
+ found = true
+ }
+ }
+
+ if !found {
+ t.Fatal("B did not forward C's evidence on the unrelated key")
+ }
+ case <-time.After(time.Second):
+ t.Fatal("response did not resume after evidence publication")
+ }
+}
+
+// The transport holds bind delivery until all three origins have published.
+// B receives A before it learns C. A does not subscribe to C's key.
+type priorityTestNetwork struct {
+ engines map[pb.NodeID]*Engine
+ mu sync.Mutex
+ published map[pb.NodeID]Tip
+ a, b, c pb.NodeID
+ ready chan struct{}
+}
+type priorityTestBroadcaster struct {
+ mockBroadcaster
+
+ net *priorityTestNetwork
+ node pb.NodeID
+}
+
+func (b *priorityTestBroadcaster) BroadcastWithData(*pb.OffsetNotify, []byte) {}
+func (b *priorityTestBroadcaster) FetchFromAny(ref *pb.EffectRef, _ FetchHint) ([]byte, error) {
+ if e := b.net.engines[pb.NodeID(ref.GetNodeId())]; e != nil {
+ if eff, ok := e.effectCache.Get(r(ref)); ok {
+ data, err := e.effectCache.MarshalEffect(eff)
+ if err != nil {
+ return nil, err
+ }
+
+ wire := make([]byte, 4+len(eff.GetKey())+len(data))
+ binary.LittleEndian.PutUint32(wire, uint32(len(eff.GetKey())))
+ copy(wire[4:], eff.GetKey())
+ copy(wire[4+len(eff.GetKey()):], data)
+
+ return wire, nil
+ }
+ }
+
+ return nil, fmt.Errorf("missing effect %v", ref)
+}
+
+func (b *priorityTestBroadcaster) ReplicateMarshalled(n *pb.OffsetNotify, _ []byte, target pb.NodeID) ([]*pb.NackNotify, error) {
+ b.net.mu.Lock()
+
+ b.net.published[b.node] = r(n.GetOrigin())
+ if len(b.net.published) == 3 {
+ select {
+ case <-b.net.ready:
+ default:
+ close(b.net.ready)
+ }
+ }
+ b.net.mu.Unlock()
+
+ select {
+ case <-b.net.ready:
+ case <-time.After(time.Second):
+ return nil, errors.New("bind publication barrier timed out")
+ }
+
+ if b.node == b.net.c && target == b.net.b {
+ b.net.mu.Lock()
+ aRef := b.net.published[b.net.a]
+ b.net.mu.Unlock()
+
+ deadline := time.After(time.Second)
+
+ for !b.net.engines[target].index.Contains("ab").Contains(aRef) {
+ select {
+ case <-deadline:
+ return nil, errors.New("A did not reach B before C")
+ default:
+ time.Sleep(time.Millisecond)
+ }
+ }
+ }
+
+ return b.net.engines[target].HandleRemote(n)
+}
+
+func TestPriorityChainCommitsDisjointEnds(t *testing.T) {
+ nodes := []pb.NodeID{1, 2, 3}
+ stamp := sTs(100)
+
+ slices.SortFunc(nodes, func(a, b pb.NodeID) int {
+ if ForkChoiceLess(ComputeForkChoiceHash(a, stamp), ComputeForkChoiceHash(b, stamp)) {
+ return -1
+ }
+
+ return 1
+ })
+ // Priority C, B, A; conflict edges A--B--C.
+ cID, bID, aID := nodes[0], nodes[1], nodes[2]
+ net := &priorityTestNetwork{engines: make(map[pb.NodeID]*Engine), published: make(map[pb.NodeID]Tip), ready: make(chan struct{}), a: aID, b: bID, c: cID}
+ keys := map[pb.NodeID][]string{aID: {"ab"}, bID: {"ab", "bc"}, cID: {"bc"}}
+ contexts := make(map[pb.NodeID]*Context)
+
+ for _, id := range nodes {
+ e := newTxnTestEngine(nil)
+ e.nodeID = id
+ e.clock.SetClock(func() time.Time { return stamp.AsTime() })
+
+ net.engines[id] = e
+ for _, key := range keys[id] {
+ base := Tip{99, 1}
+ if key == "bc" {
+ base[1] = 2
+ }
+
+ if err := e.effectCache.Put(base, scalarSet(key, "base")); err != nil {
+ t.Fatal(err)
+ }
+
+ e.updateIndex(key, nil, base)
+
+ state := &subscriptionState{ready: make(chan struct{})}
+ state.markReady()
+ e.subscriptions.Store(key, state)
+
+ for _, peer := range nodes {
+ if peer != id && slices.Contains(keys[peer], key) {
+ e.addPeerSubscriber(key, peer)
+ }
+ }
+ }
+
+ bc := &priorityTestBroadcaster{net: net, node: id}
+ bc.peerIDs = nodes
+ bc.allRegionPeersReachable = true
+ e.SetBroadcaster(bc)
+ ctx := e.NewContext()
+ ctx.BeginTx()
+
+ for _, key := range keys[id] {
+ if err := ctx.Emit(scalarSet(key, fmt.Sprint(id))); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ contexts[id] = ctx
+ }
+
+ type outcome struct {
+ id pb.NodeID
+ err error
+ }
+
+ results := make(chan outcome, 3)
+
+ for _, id := range nodes {
+ go func() { results <- outcome{id, contexts[id].Flush()} }()
+ }
+
+ for range nodes {
+ select {
+ case result := <-results:
+ if result.id == bID {
+ if !errors.Is(result.err, ErrTxnAborted) {
+ t.Errorf("B: %v, want definite abort", result.err)
+ }
+ } else if result.err != nil {
+ t.Errorf("end node %d: %v, want commit", result.id, result.err)
+ }
+ case <-time.After(3 * time.Second):
+ t.Fatal("priority chain failed to drain")
+ }
+ }
+
+ if _, subscribed := net.engines[aID].subscriptions.Load("bc"); subscribed {
+ t.Fatal("A subscribed to C's key")
+ }
+}
+
+func TestPriorityChainReconstructionWithoutVerdicts(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ nodes := []pb.NodeID{1, 2, 3}
+ slices.SortFunc(nodes, func(a, b pb.NodeID) int {
+ if ForkChoiceLess(ComputeForkChoiceHash(a, sTs(100)), ComputeForkChoiceHash(b, sTs(100))) {
+ return -1
+ }
+
+ return 1
+ })
+
+ for i, keys := range [][]string{{"bc"}, {"ab", "bc"}, {"ab"}} {
+ id := nodes[i]
+ txn := fmt.Sprintf("txn-%d", i)
+ log.nodeID = uint64(id)
+ bind := &pb.TransactionalBindEffect{OriginatorNodeId: uint64(id), TxnHlc: sTs(100)}
+
+ for _, key := range keys {
+ data := scalarSet(key, txn)
+ data.NodeId = uint64(id)
+ data.TxnId = txn
+ data.Hlc = sTs(100)
+ tip := log.putEffect(data)
+ bind.Keys = append(bind.Keys, &pb.TransactionalBindEffect_KeyBind{Key: []byte(key), NewTip: toPbRef(tip)})
+ }
+
+ eff := &pb.Effect{Key: []byte(keys[0]), NodeId: uint64(id), Hlc: sTs(100), TxnId: txn, Kind: &pb.Effect_TxnBind{TxnBind: bind}}
+ for _, kb := range bind.GetKeys() {
+ eff.Deps = append(eff.Deps, kb.GetNewTip())
+ }
+
+ tip := log.putEffect(eff)
+ for _, key := range keys {
+ e.updateIndex(key, nil, tip)
+ }
+ }
+
+ for _, cached := range []bool{false, true} {
+ if cached {
+ for _, key := range []string{"ab", "bc"} {
+ tips := e.index.Contains(key)
+ snap := log.putEffect(&pb.Effect{Key: []byte(key), Deps: toPbRefs(tips.Tips()), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"txn-0": pb.Verdict_WON, "txn-1": pb.Verdict_LOST, "txn-2": pb.Verdict_WON},
+ }}})
+ e.updateIndex(key, tips, snap)
+ }
+ }
+
+ for key, want := range map[string]string{"ab": "txn-2", "bc": "txn-0"} {
+ value, _, _, err := e.GetSnapshot(key)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if got := string(value.GetScalar().GetRaw()); got != want {
+ t.Errorf("cached=%t %s=%q, want %q", cached, key, got, want)
+ }
+ }
+
+ values, err := e.GetSnapshots(context.Background(), []string{"ab", "bc"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if string(values[0].GetScalar().GetRaw()) != "txn-2" || string(values[1].GetScalar().GetRaw()) != "txn-0" {
+ t.Fatalf("cached=%t batch disagrees: %v", cached, values)
+ }
+ }
+}
+
+// Closure discovery and each key walk can observe different live frontiers.
+// A cached loss first encountered by a per-key walk must survive aggregation.
+func TestPriorityReconstructionRetainsPerKeyVerdicts(t *testing.T) {
+ log := newSnapshotLog()
+ e := newSnapshotEngine(log)
+ loser := closureBind(log, "loser", 2, []string{"k"}, nil)
+ snap := log.putEffect(&pb.Effect{Key: []byte("k"), Deps: toPbRefs([]Tip{loser}), Kind: &pb.Effect_Snapshot{Snapshot: &pb.SnapshotEffect{
+ TxnVerdicts: map[string]pb.Verdict{"loser": pb.Verdict_LOST},
+ }}})
+
+ lost, _ := e.losersOnKeysAt(map[string][]Tip{"k": nil}, func(string) []Tip { return []Tip{snap} }, nil, nil, false)
+ if _, ok := lost["loser"]; !ok {
+ t.Fatal("per-key cached loss disappeared from the combined result")
+ }
+}
diff --git a/effects/txn_snapshot_conflict_test.go b/effects/txn_snapshot_conflict_test.go
index 394baec..265a62d 100644
--- a/effects/txn_snapshot_conflict_test.go
+++ b/effects/txn_snapshot_conflict_test.go
@@ -299,7 +299,7 @@ func TestIsRealConflictWrappedData(t *testing.T) {
name, element, txn string
want bool
}{
- {"same-element", "a", "", true},
+ {"same-element", "a", "", false},
{"disjoint-element", "b", "", false},
{"in-progress", "a", "pending", false},
} {
diff --git a/effects/txn_test.go b/effects/txn_test.go
index 9196ea6..1fe0382 100644
--- a/effects/txn_test.go
+++ b/effects/txn_test.go
@@ -84,7 +84,7 @@ func TestIsRealConflict_MetaEffect_NotConflict(t *testing.T) {
}
}
-func TestIsRealConflict_SameElementID_Conflict(t *testing.T) {
+func TestIsRealConflict_NonTxSameElementID_NotConflict(t *testing.T) {
ptxn := newTestPendingTxn(pendingTxnKey{
key: "k",
newTip: Tip{1, 500},
@@ -100,8 +100,8 @@ func TestIsRealConflict_SameElementID_Conflict(t *testing.T) {
ElementId: []byte("field1"),
}
- if !testIsRealConflict(t, ptxn, detail) {
- t.Fatal("same element ID should be a real conflict")
+ if testIsRealConflict(t, ptxn, detail) {
+ t.Fatal("non-transactional writes do not defeat a published bind, even on the same element")
}
}
@@ -126,7 +126,7 @@ func TestIsRealConflict_DifferentElementID_NotConflict(t *testing.T) {
}
}
-func TestIsRealConflict_NoopKey_AnyWriteConflicts(t *testing.T) {
+func TestIsRealConflict_NoopKey_NonTxWriteNotConflict(t *testing.T) {
ptxn := newTestPendingTxn(pendingTxnKey{
key: "k",
newTip: Tip{1, 500},
@@ -141,19 +141,20 @@ func TestIsRealConflict_NoopKey_AnyWriteConflicts(t *testing.T) {
Collection: pb.CollectionKind_SCALAR,
}
- if !testIsRealConflict(t, ptxn, detail) {
- t.Fatal("any write on a read-only key should be a real conflict (phantom write)")
+ if testIsRealConflict(t, ptxn, detail) {
+ t.Fatal("a non-transactional write to a read key does not defeat a published bind")
}
}
-func TestIsRealConflict_DependentEffect_NotConflict(t *testing.T) {
+func TestIsRealConflict_NonTxDependsOnUnboundEffect_NotConflict(t *testing.T) {
ptxn := newTestPendingTxn(pendingTxnKey{
key: "k",
newTip: Tip{1, 500},
collection: pb.CollectionKind_SCALAR,
})
- // The competing effect depends on our tx tip → sequential, not concurrent
+ // Observing a pre-bind effect does not observe our commit, but an ordinary
+ // write still cannot establish a transaction loss after bind publication.
detail := &pb.NackTipDetail{
Ref: &pb.EffectRef{NodeId: 1, Offset: 600},
Hlc: tTs(110),
@@ -163,7 +164,7 @@ func TestIsRealConflict_DependentEffect_NotConflict(t *testing.T) {
}
if testIsRealConflict(t, ptxn, detail) {
- t.Fatal("effect that depends on our tx should not be a real conflict")
+ t.Fatal("non-transactional ancestry through a pre-bind tip must not cause an abort")
}
}
@@ -234,7 +235,7 @@ func TestIsRealConflict_CompetingBind_OurHashLower_WeWin(t *testing.T) {
}
}
-func TestIsRealConflict_NonTxCompetitor_Conflict(t *testing.T) {
+func TestIsRealConflict_NonTxCompetitor_NotConflict(t *testing.T) {
ptxn := newTestPendingTxn(pendingTxnKey{
key: "k",
newTip: Tip{1, 500},
@@ -250,8 +251,8 @@ func TestIsRealConflict_NonTxCompetitor_Conflict(t *testing.T) {
Collection: pb.CollectionKind_SCALAR,
}
- if !testIsRealConflict(t, ptxn, detail) {
- t.Fatal("non-tx data write on same scalar key should be a real conflict")
+ if testIsRealConflict(t, ptxn, detail) {
+ t.Fatal("non-transactional data on the same scalar key must not defeat a published bind")
}
}
@@ -446,11 +447,9 @@ func TestDefeatedCompetitor_TheirBindReachableViaDeps_NotDefeated(t *testing.T)
}
}
-// Symmetric reachability case mirroring TestIsRealConflict_DependentEffect_NotConflict:
-// the foreign bind's own Deps reference one of our pending tx's tips,
-// so the foreign bind is causally-after us. Not concurrent — must not
-// be marked defeated regardless of hash.
-func TestDefeatedCompetitor_DetailDepsReferenceOurNewTip_NotDefeated(t *testing.T) {
+// Referencing a pending transaction's NewTip observes its effects, not its
+// commit. The two bind envelopes are still concurrent and use fork choice.
+func TestDefeatedCompetitor_DetailDepsReferenceOurNewTip_StillCompetes(t *testing.T) {
e := newTxnTestEngine(nil)
theirTxnID := "their-tx"
@@ -498,8 +497,8 @@ func TestDefeatedCompetitor_DetailDepsReferenceOurNewTip_NotDefeated(t *testing.
},
}
- if _, defeated := e.classifyNackTip(ptxn, "k", detail); defeated != "" {
- t.Fatalf("bind whose deps reference our newTip is causally-after us, not defeated; got %q", defeated)
+ if _, defeated := e.classifyNackTip(ptxn, "k", detail); defeated != theirTxnID {
+ t.Fatalf("observing our pre-bind tip must not exempt the competing bind; defeated=%q", defeated)
}
}
diff --git a/effects/vertexpool.go b/effects/vertexpool.go
index efb1dab..9f10bdf 100644
--- a/effects/vertexpool.go
+++ b/effects/vertexpool.go
@@ -50,6 +50,10 @@ type vertex struct {
eff *pb.Effect
size int64
refs atomic.Int32
+ // isRoot means missing ancestry beyond this vertex need not be fetched.
+ // Resident dependencies still participate in traversal and consumption.
+ // Protected by creationMu.
+ isRoot bool
// Temporal completion belongs to this resident copy, so eviction also
// discards the bookkeeping. Bits are set only after evidence publication.
temporal atomic.Uint32
@@ -58,6 +62,13 @@ type vertex struct {
creationMu sync.Mutex
creationKeys []string // sorted, unique; usually just the effect's own key
creationBytes int64
+ // Reverse navigation is not ownership. Identities do not pin descendant
+ // payloads, and this storage disappears with the ancestor. creationMu
+ // protects the slice and its memory accounting, not descendant lifetimes.
+ parents []struct {
+ tip Tip
+ key string
+ }
}
// VertexPool is the engine's deserialized effect store: a map from Tip to
@@ -135,6 +146,71 @@ func (p *VertexPool) Get(tip Tip) (*pb.Effect, bool) {
return v.eff, true
}
+func (p *VertexPool) getVertex(tip Tip) *vertex {
+ v, ok := p.m.Load(tip)
+ if !ok || v.refs.Load() < 0 {
+ p.misses.Add(1)
+
+ return nil
+ }
+
+ p.hits.Add(1)
+
+ return v
+}
+
+func (v *vertex) root() bool {
+ v.creationMu.Lock()
+ defer v.creationMu.Unlock()
+
+ return v.isRoot
+}
+
+func (v *vertex) markRoot() {
+ v.creationMu.Lock()
+ defer v.creationMu.Unlock()
+
+ v.isRoot = true
+}
+
+func (p *VertexPool) addParent(v *vertex, parent Tip, key string) {
+ v.creationMu.Lock()
+ defer v.creationMu.Unlock()
+
+ if v.refs.Load() < 0 {
+ return
+ }
+
+ for _, link := range v.parents {
+ if link.tip == parent && link.key == key {
+ return
+ }
+ }
+ // Reclamation removes payloads, not evidence of an established join.
+ // Keep the key as well: a shared bind can have parents on different keys,
+ // and the parent's payload may be gone when this edge is traversed.
+ v.parents = append(v.parents, struct {
+ tip Tip
+ key string
+ }{parent, key})
+ p.accountCreation(v)
+}
+
+// firstParent follows one established chain; cut discovery never enumerates
+// all reverse paths. A reclaimed parent's identity remains valid join evidence.
+func (v *vertex) firstParent(key string) (Tip, bool) {
+ v.creationMu.Lock()
+ defer v.creationMu.Unlock()
+
+ for _, link := range v.parents {
+ if link.key == key {
+ return link.tip, true
+ }
+ }
+
+ return Tip{}, false
+}
+
// Put stores eff at tip and accounts for its resident protobuf allocations.
//
// A re-Put of an already-resident tip keeps the existing vertex so its
@@ -205,12 +281,31 @@ func missingCreationKeys(eff *pb.Effect, existing []string, owns func([]byte) bo
// put owns persistence, publication, and accounting for both owned and cache-only
// effects. Creation references are initialized before a vertex is published.
func (p *VertexPool) put(tip Tip, eff *pb.Effect, takeOwnership bool, owns func([]byte) bool) error {
- if p.mirror != nil {
+ return p.putWithPin(tip, eff, takeOwnership, owns, false)
+}
+
+// pin reserves an independent reference before a new vertex is visible. Proof
+// package installation uses it to protect cache-only records until their owner
+// has acquired its references, even with reclamation running concurrently.
+func (p *VertexPool) putWithPin(tip Tip, eff *pb.Effect, takeOwnership bool, owns func([]byte) bool, pin bool) error {
+ if len(eff.GetEvidencePackage()) != 0 {
+ clean, release, err := p.installEvidencePackage(tip, eff)
+ if err != nil {
+ return err
+ }
+ defer release()
+
+ eff = clean
+ }
+ // Package members are durable inside their owning snapshot. Writing them
+ // as independent records would promote proof-only history into key
+ // frontiers during restart. An ordinary ingestion persists them normally.
+ if p.mirror != nil && !pin {
lock := &p.mirrorLocks[(tip[0]^tip[1])%256]
lock.Lock()
defer lock.Unlock()
- data, err := MarshalEffect(eff)
+ data, err := p.MarshalEffect(eff)
if err != nil {
return err
}
@@ -222,12 +317,21 @@ func (p *VertexPool) put(tip Tip, eff *pb.Effect, takeOwnership bool, owns func(
v, loaded := p.m.Load(tip)
if !loaded {
+ evidence, err := p.pinEvidence(tip, eff)
+ if err != nil {
+ return err
+ }
+
nv := &vertex{eff: eff, size: vertexOverhead}
if takeOwnership {
nv.creationKeys = missingCreationKeys(eff, nil, owns)
}
owners := len(nv.creationKeys)
+ if pin {
+ owners++
+ }
+
nv.refs.Store(int32(owners))
p.owned.change(eff, 1)
p.vertexBytes.Add(nv.size)
@@ -235,6 +339,7 @@ func (p *VertexPool) put(tip Tip, eff *pb.Effect, takeOwnership bool, owns func(
v, loaded = p.m.LoadOrStore(tip, nv)
if loaded {
+ p.releaseEvidence(evidence)
p.owned.change(eff, -1)
p.vertexBytes.Add(-nv.size - nv.creationBytes)
} else if owners == 0 {
@@ -242,22 +347,35 @@ func (p *VertexPool) put(tip Tip, eff *pb.Effect, takeOwnership bool, owns func(
}
}
- if loaded && takeOwnership {
+ if loaded && (takeOwnership || pin) {
v.creationMu.Lock()
defer v.creationMu.Unlock()
- keys := missingCreationKeys(eff, v.creationKeys, owns)
- if len(keys) == 0 {
+ var keys []string
+ if takeOwnership {
+ keys = missingCreationKeys(eff, v.creationKeys, owns)
+ }
+
+ added := len(keys)
+ if pin {
+ added++
+ }
+
+ if added == 0 {
return nil
}
for {
n := v.refs.Load()
if n < 0 {
+ if pin {
+ return fmt.Errorf("pin effect %v: reclamation already claimed it", tip)
+ }
+
return nil // reclaim already claimed this vertex; do not resurrect it
}
- if v.refs.CompareAndSwap(n, n+int32(len(keys))) {
+ if v.refs.CompareAndSwap(n, n+int32(added)) {
break
}
}
@@ -320,6 +438,9 @@ func (p *VertexPool) delete(tip Tip) int64 {
// CAS therefore publishes the final accounting and prevents further writes.
p.vertexBytes.Add(-prev.size - prev.creationBytes)
p.owned.change(prev.eff, -1)
+ // Each snapshot vertex owns its proof once, independently of the number
+ // of keys, read DAGs, or outbox entries holding the snapshot itself.
+ p.releaseEvidence(uniqueEvidence(prev.eff))
return prev.size
}
@@ -382,6 +503,10 @@ func (p *VertexPool) claimAndDelete(tip Tip, v *vertex) int64 {
return 0
}
+ // Finish navigation/accounting updates before retiring their storage.
+ v.creationMu.Lock()
+ defer v.creationMu.Unlock()
+
if !v.refs.CompareAndSwap(0, vertexTombstone) {
return 0
}
@@ -482,11 +607,15 @@ func (p *VertexPool) Bytes() int64 {
// Call with creationMu held, or before publishing a new vertex.
func (p *VertexPool) accountCreation(v *vertex) {
- cost := memsize.Alloc(int64(cap(v.creationKeys)) * 16)
+ cost := memsize.Alloc(int64(cap(v.creationKeys))*16) + memsize.Alloc(int64(cap(v.parents))*int64(unsafe.Sizeof(v.parents[0])))
for _, key := range v.creationKeys {
cost += memsize.Alloc(int64(len(key)))
}
+ for _, link := range v.parents {
+ cost += memsize.Alloc(int64(len(link.key)))
+ }
+
p.vertexBytes.Add(cost - v.creationBytes)
v.creationBytes = cost
}
diff --git a/effects/watch_compaction_test.go b/effects/watch_compaction_test.go
new file mode 100644
index 0000000..55007c4
--- /dev/null
+++ b/effects/watch_compaction_test.go
@@ -0,0 +1,145 @@
+package effects
+
+import (
+ "testing"
+
+ pb "github.com/swytchdb/engine/cluster/proto"
+)
+
+func TestWatchCompactsObservationHistory(t *testing.T) {
+ for _, readOnly := range []bool{false, true} {
+ name := "normal-context"
+ if readOnly {
+ name = "read-only-context"
+ }
+
+ t.Run(name, func(t *testing.T) {
+ e := newTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "key")
+ historicalTestWrite(t, e, map[string]string{"key": "value"}, false)
+
+ c := e.NewContext()
+ if readOnly {
+ c = e.NewReadOnlyContext()
+ }
+ defer c.Abort()
+
+ var compacted bool
+
+ for range 12 {
+ if err := c.Watch("key"); err != nil {
+ t.Fatal(err)
+ }
+
+ tip := c.watchedKeys["key"].noopOffset
+
+ noop, ok := e.effectCache.Get(tip)
+ if !ok || noop.GetNoop() == nil {
+ t.Fatal("observation NOOP is missing")
+ }
+
+ for _, dep := range noop.GetDeps() {
+ eff, ok := e.effectCache.Get(r(dep))
+ if ok && eff.GetSnapshot().GetState() != nil {
+ compacted = true
+ }
+ }
+
+ got, _, _, err := e.GetSnapshot("key")
+ if err != nil || string(got.GetScalar().GetRaw()) != "value" {
+ t.Fatalf("read after observation: value=%v error=%v", got, err)
+ }
+
+ if compacted {
+ c.BeginTx()
+
+ if !c.CheckWatches() {
+ t.Fatal("own compaction invalidated the observation")
+ }
+
+ if err := c.Flush(); err != nil {
+ t.Fatal(err)
+ }
+
+ return
+ }
+
+ c.ClearWatches()
+ }
+
+ t.Fatal("observation history grew without producing a state snapshot")
+ })
+ }
+}
+
+func TestReconstructionCountsFilteredHistory(t *testing.T) {
+ e := newTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "key")
+ historicalTestWrite(t, e, map[string]string{"key": "value"}, false)
+
+ _, _, before, err := e.GetSnapshot("key")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ c := e.NewContext()
+ if err := c.Emit(&pb.Effect{Key: []byte("key"), TxnId: "observation", Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}}}); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := c.Flush(); err != nil {
+ t.Fatal(err)
+ }
+
+ got, _, after, err := e.GetSnapshot("key")
+ if err != nil || string(got.GetScalar().GetRaw()) != "value" {
+ t.Fatalf("value=%v error=%v", got, err)
+ }
+
+ if after != before+1 {
+ t.Fatalf("walked count before=%d after=%d: filtered NOOP was not counted", before, after)
+ }
+}
+
+func TestNoopEmissionMakesContextWritable(t *testing.T) {
+ e := newTestEngine(nil)
+
+ t.Cleanup(func() { _ = e.Close() })
+ preSubscribe(e, "key")
+ historicalTestWrite(t, e, map[string]string{"key": "value"}, false)
+
+ c := e.NewReadOnlyContext()
+ defer c.Abort()
+
+ for range 6 {
+ if err := c.Emit(&pb.Effect{Key: []byte("key"), Kind: &pb.Effect_Noop{Noop: &pb.NoopEffect{}}}); err != nil {
+ t.Fatal(err)
+ }
+
+ if err := c.Flush(); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ if _, _, err := c.GetSnapshot("key"); err != nil {
+ t.Fatal(err)
+ }
+
+ pending := c.keys["key"]
+ if pending == nil {
+ t.Fatal("context that emitted NOOPs still skips compaction")
+ }
+
+ eff, ok := e.effectCache.Get(pending.lastOffset)
+ if !ok || eff.GetSnapshot().GetState() == nil {
+ t.Fatal("compaction did not produce a state snapshot")
+ }
+
+ if err := c.Flush(); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/go.mod b/go.mod
index 4cdcd62..c226e8a 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.27
require (
github.com/cespare/xxhash/v2 v2.3.0
- github.com/klauspost/compress v1.20.3
+ github.com/klauspost/compress v1.20.4
github.com/prometheus/client_golang v1.25.0
github.com/prometheus/client_model v0.6.3
github.com/puzpuzpuz/xsync/v4 v4.5.0
@@ -14,9 +14,9 @@ require (
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0
go.opentelemetry.io/otel/sdk v1.47.0
go.opentelemetry.io/otel/trace v1.47.0
- golang.org/x/crypto v0.58.0
- golang.org/x/sync v0.24.0
- golang.org/x/sys v0.49.0
+ golang.org/x/crypto v0.59.0
+ golang.org/x/sync v0.25.0
+ golang.org/x/sys v0.50.0
google.golang.org/grpc v1.85.0
google.golang.org/protobuf v1.36.12
)
@@ -37,8 +37,8 @@ require (
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
- golang.org/x/net v0.61.1-0.20261013155913-7e562009b388 // indirect
- golang.org/x/text v0.43.0 // indirect
+ golang.org/x/net v0.62.0 // indirect
+ golang.org/x/text v0.44.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect
)
diff --git a/go.sum b/go.sum
index 25da2de..0e9f5ce 100644
--- a/go.sum
+++ b/go.sum
@@ -17,8 +17,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY=
-github.com/klauspost/compress v1.20.3 h1:zWNoFqq/lSuYQRim3dUXlXeA0IqwuZGicdG5ZTfY5j4=
-github.com/klauspost/compress v1.20.3/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
+github.com/klauspost/compress v1.20.4 h1:dD0v/bvxtjCBqQu6sx1TNWkalEY5lNTIe+HY9w8TeNI=
+github.com/klauspost/compress v1.20.4/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw=
github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU=
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
@@ -35,8 +35,6 @@ github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQ
github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo=
github.com/quic-go/go-ossfuzz-seeds v0.1.0 h1:APacT+iIaNF6fd8AGEiN3bT/Jtkd2jz4v4TzM7MFjy0=
github.com/quic-go/go-ossfuzz-seeds v0.1.0/go.mod h1:3IOHRbJIc+L6YKMwfDtJAM9Vj9k0YY4muhuyUYk5tbk=
-github.com/quic-go/quic-go v0.64.1 h1:BCNTP+6x6++OPKrGST2FR3f1tycx8O3QOA3QQv09g70=
-github.com/quic-go/quic-go v0.64.1/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
github.com/quic-go/quic-go v0.65.0 h1:BJQSjdTvLPLmWtaCupUphRbgxgpZhdO/GMpr9Eoe3B4=
github.com/quic-go/quic-go v0.65.0/go.mod h1:RAro2j2yN9a9EiPACLHT9IB2NXCvGQmmo/alT0yYI0w=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
@@ -73,18 +71,16 @@ go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ=
go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
-golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
-golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
-golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU=
-golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
-golang.org/x/net v0.61.1-0.20261013155913-7e562009b388 h1:oRvUlnTW8OBc0WHDWUGm6d6bRZCDVVvZ0ceM0kaKLNQ=
-golang.org/x/net v0.61.1-0.20261013155913-7e562009b388/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
-golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
-golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
-golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
-golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
-golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
-golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
+golang.org/x/crypto v0.59.0 h1:xS2jhqYbzNrpvkWP2YTDf4TJwBy3ykR03i8vFDxEc7c=
+golang.org/x/crypto v0.59.0/go.mod h1:veDyUwATGs3c+vN3VfTbzViMGXeTBk3WyhBOkn+F/4U=
+golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY=
+golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI=
+golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
+golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
+golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
+golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
+golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE=