Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 35 additions & 14 deletions cluster/cloud_discovery.go
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ type cloudEffectLoader func(effects.Tip) (*pb.Effect, error)
func reduceDiscoveredTips(initialTips []effects.Tip, load cloudEffectLoader) (*pb.ReducedEffect, error) {
visited := map[effects.Tip]bool{}

var queue []effects.Tip
var queue, snapshots []effects.Tip

enqueue := func(t effects.Tip) {
if !visited[t] {
Expand Down Expand Up @@ -182,23 +182,44 @@ func reduceDiscoveredTips(initialTips []effects.Tip, load cloudEffectLoader) (*p
lastProgress := walkStarted
progressed := false

for len(queue) > 0 {
tip := queue[0]
queue = queue[1:]
for len(queue) > 0 || len(snapshots) > 0 {
var tip effects.Tip
if len(queue) > 0 {
tip, queue = queue[0], queue[1:]
} else {
tip, snapshots = snapshots[0], snapshots[1:]
}

eff, err := load(tip)
if err != nil {
// A skipped fetch drops the tip's whole dep subtree, which is
// the same partial-ancestry reduce as a truncated walk. Fail
// instead; the caller retries with backoff.
return nil, fmt.Errorf("cloud discover: fetch %v: %w", tip, err)
eff, cached := fetched[tip]
if !cached {
var err error

eff, err = load(tip)
if err != nil {
// A skipped fetch drops the tip's whole dep subtree, which is
// the same partial-ancestry reduce as a truncated walk. Fail
// instead; the caller retries with backoff.
return nil, fmt.Errorf("cloud discover: fetch %v: %w", tip, err)
}

fetched[tip] = eff
}

fetched[tip] = eff
if snap := eff.GetSnapshot(); snap != nil && snap.GetState() != nil && len(queue) == 0 {
lcaTip, lcaFound = tip, true
if snap := eff.GetSnapshot(); snap != nil && snap.GetState() != nil {
// Resolve subscription observations (including unsubscribe deps)
// before deciding whether to expand folded history. A metadata
// effect queued after this snapshot cannot create a data branch.
if len(queue) > 0 {
snapshots = append(snapshots, tip)

break
continue
}

if len(snapshots) == 0 {
lcaTip, lcaFound = tip, true

break
}
}

for _, dep := range eff.GetDeps() {
Expand Down
119 changes: 74 additions & 45 deletions effects/dag.go
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,56 @@ func (d *dag) bfs(tips []Tip, lcaTip *Tip) error {
return err
}

// An empty subscription records no observed data. Referencing it records an
// observation of metadata, not another data branch. Subscription membership is
// maintained separately; keep the effect for causal bookkeeping only.
func emptySubscription(eff *pb.Effect) bool {
return eff.GetSubscription() != nil && len(eff.GetDeps()) == 0
}

// visitDataDeps loads causal dependencies while treating subscription metadata
// as transparent. An unsubscribe can carry observed data deps: follow those,
// without making the subscription itself a data root. A chain containing only
// subscription observations has the same data ancestry as no dependencies.
// Missing effects remain unresolved data dependencies, never assumed metadata.
func visitDataDeps(key string, eff *pb.Effect, visit func(Tip) *pb.Effect) bool {
pending := []*pb.Effect{eff}

var subscriptions map[Tip]bool

hasDataDeps := false

for len(pending) > 0 {
cur := pending[len(pending)-1]
pending = pending[:len(pending)-1]

for ref := range keyDeps(key, cur) {
tip := r(ref)

dep := visit(tip)
if dep.GetSubscription() == nil {
hasDataDeps = true

continue
}

if emptySubscription(dep) || subscriptions[tip] {
continue
}

if subscriptions == nil {
subscriptions = make(map[Tip]bool)
}

subscriptions[tip] = true

pending = append(pending, dep)
}
}

return hasDataDeps
}

// walkKeyHistory is the shared dependency walk for replay and tip normalization.
// Missing nodes remain on the dependency frontier, so an exhausted queue cannot
// disguise an unresolved branch. Non-missing load errors take precedence over
Expand Down Expand Up @@ -310,16 +360,22 @@ func walkKeyHistory(key string, tips []Tip, load func(Tip) (*pb.Effect, error))
snapshots := make([]Tip, 0, len(tips))
frontier := make(map[Tip]struct{}, len(tips))
enqueue := func(t Tip, eff *pb.Effect) {
if eff.GetSubscription() != nil {
delete(frontier, t)

return
}

if snap := eff.GetSnapshot(); snap != nil && snap.GetState() != nil {
snapshots = append(snapshots, t)
} else {
queue = append(queue, t)
}
}

for _, t := range tips {
if _, seen := nodes[t]; seen {
continue
visit := func(t Tip) *pb.Effect {
if eff, seen := nodes[t]; seen {
return eff
}

frontier[t] = struct{}{}
Expand All @@ -330,20 +386,23 @@ func walkKeyHistory(key string, tips []Tip, load func(Tip) (*pb.Effect, error))

recordError(err)

continue
return nil
}

nodes[t] = eff
if eff == nil {
continue
}
// A standalone subscription tip carries no data ancestry. If another
// path later references it, it becomes part of that path's frontier.
if eff.GetSubscription() != nil && len(eff.GetDeps()) == 0 {
delete(frontier, t)
return nil
}

enqueue(t, eff)

return eff
}

for _, t := range tips {
if eff := visit(t); eff.GetSubscription() != nil {
visitDataDeps(key, eff, visit)
}
}

for len(queue) > 0 || len(snapshots) > 0 {
Expand Down Expand Up @@ -371,41 +430,11 @@ func walkKeyHistory(key string, tips []Tip, load func(Tip) (*pb.Effect, error))
return nodes, lcaTip, walkError()
}

// Roots finish queued work without resolving their dependency path.
first := true
for ref := range keyDeps(key, eff) {
if first {
delete(frontier, cur)

first = false
}

dt := r(ref)
if dep, seen := nodes[dt]; seen {
if dep.GetSubscription() != nil && len(dep.GetDeps()) == 0 {
frontier[dt] = struct{}{}
}

continue
}

frontier[dt] = struct{}{}

depEff, err := load(dt)
if err != nil {
nodes[dt] = nil

recordError(err)

continue
}

nodes[dt] = depEff
if depEff == nil {
continue
}

enqueue(dt, depEff)
// A data effect with only subscription observations is a data root,
// exactly as if those metadata references had not been attached.
hasDataDeps := visitDataDeps(key, eff, visit)
if hasDataDeps {
delete(frontier, cur)
}
}

Expand Down
4 changes: 2 additions & 2 deletions effects/dag_frontier_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,8 +123,8 @@ func TestDAGSnapshotDependencyFrontier(t *testing.T) {
}

if subscription != (Tip{}) {
// A subscription first seen as a standalone tip must
// become a frontier entry when A references it.
// Advertising subscription metadata separately must not
// hide A's independent data path around S.
tips = append([]Tip{subscription}, tips...)
}

Expand Down
76 changes: 36 additions & 40 deletions effects/effect.go
Original file line number Diff line number Diff line change
Expand Up @@ -2136,41 +2136,45 @@ func (e *Engine) collectCausalChain(key string, tips []keytrie.EffectRef) []*pb.

var chain []*pb.EffectRef

enqueue := func(tp Tip, standalone bool) {
// Keep subscription refs in the advertised causal history. visitDataDeps
// distinguishes those observations from the data ancestry they carry.
enqueue := func(tp Tip) *pb.Effect {
if tp == zero {
return
return nil
}

eff, ok := e.effectCache.Get(tp)
if visited[tp] {
if !standalone && ok && eff.GetSubscription() != nil && len(eff.GetDeps()) == 0 {
frontier[tp] = struct{}{}
}

return
return eff
}

visited[tp] = true
frontier[tp] = struct{}{}
chain = append(chain, toPbRef(tp))

if !ok {
return
return nil
}

if standalone && eff.GetSubscription() != nil && len(eff.GetDeps()) == 0 {
if eff.GetSubscription() != nil {
delete(frontier, tp)

return eff
}

if snap := eff.GetSnapshot(); snap != nil && snap.GetState() != nil {
snapshots = append(snapshots, tp)
} else {
queue = append(queue, tp)
}

return eff
}

for _, tp := range tips {
enqueue(tp, true)
if eff := enqueue(tp); eff.GetSubscription() != nil {
visitDataDeps(key, eff, enqueue)
}
}

for len(queue) > 0 || len(snapshots) > 0 {
Expand All @@ -2190,15 +2194,9 @@ func (e *Engine) collectCausalChain(key string, tips []keytrie.EffectRef) []*pb.
break
}

first := true
for ref := range keyDeps(key, eff) {
if first {
delete(frontier, cur)

first = false
}

enqueue(r(ref), false)
hasDataDeps := visitDataDeps(key, eff, enqueue)
if hasDataDeps {
delete(frontier, cur)
}
}

Expand Down Expand Up @@ -2230,17 +2228,15 @@ func (e *Engine) ingestNackTips(nack *pb.NackNotify) {
snapshots := make([]Tip, 0, len(nack.GetTipDetails()))
frontier := make(map[Tip]struct{}, len(nack.GetTipDetails()))
nodes := make(map[Tip]*pb.Effect)
enqueue := func(off Tip, standalone bool) {
// Ingest subscription observations normally; only their data ancestry
// participates in the snapshot stopping rule.
enqueue := func(off Tip) *pb.Effect {
if off == zero {
return
return nil
}

if visited[off] {
if eff := nodes[off]; !standalone && eff.GetSubscription() != nil && len(eff.GetDeps()) == 0 {
frontier[off] = struct{}{}
}

return
return nodes[off]
}

visited[off] = true
Expand All @@ -2254,32 +2250,38 @@ func (e *Engine) ingestNackTips(nack *pb.NackNotify) {
slog.Debug("ingestNackTips: handleBackfill failed",
"ref", off, "error", err)

return
return nil
}

if e.effectCache == nil {
return
return nil
}

eff, ok := e.effectCache.Get(off)
if !ok {
return
return nil
}

nodes[off] = eff
if standalone && eff.GetSubscription() != nil && len(eff.GetDeps()) == 0 {
if eff.GetSubscription() != nil {
delete(frontier, off)

return eff
}

if snap := eff.GetSnapshot(); snap != nil && snap.GetState() != nil {
snapshots = append(snapshots, off)
} else {
queue = append(queue, off)
}

return eff
}

for _, detail := range nack.GetTipDetails() {
enqueue(r(detail.GetRef()), true)
if eff := enqueue(r(detail.GetRef())); eff.GetSubscription() != nil {
visitDataDeps(key, eff, enqueue)
}
}

for len(queue) > 0 || len(snapshots) > 0 {
Expand All @@ -2295,15 +2297,9 @@ func (e *Engine) ingestNackTips(nack *pb.NackNotify) {
break
}

first := true
for ref := range keyDeps(string(nack.GetKey()), eff) {
if first {
delete(frontier, cur)

first = false
}

enqueue(r(ref), false)
hasDataDeps := visitDataDeps(key, eff, enqueue)
if hasDataDeps {
delete(frontier, cur)
}
}

Expand Down
Loading