From d78f0aef974a008f1083c362a96194efe43d2757 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:01:06 -0700 Subject: [PATCH] fix(runs): a re-attach goes on with the turn it was in berth run now sends how long its engine has held the card since its last grant or resume, and the new berthd counts the minimum turn from then. Before, every daemon restart reset each holder's min_turn, so a deploy kept waiters out for a full turn again. --- CHANGELOG.md | 3 +++ docs/leasing.md | 2 +- internal/broker/broker.go | 9 ++++++--- internal/broker/minturn_test.go | 18 ++++++++++++++++++ internal/proto/proto.go | 4 ++++ internal/runner/park.go | 20 ++++++++++++++++++++ internal/runner/park_test.go | 29 +++++++++++++++++++++++++++++ internal/runner/runner.go | 6 +++++- internal/server/server.go | 1 + 9 files changed, 87 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ada899c..d83d28c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,9 @@ has a matching GitHub release whose body comes from its section below. ### Fixed +- A berthd restart no longer gives every running engine a fresh minimum turn. `berth run` now tells the new berthd + how long its engine had held the card, and the re-attached lease goes on with the turn it was in. Before, each + deploy reset the TURN column to the full `min_turn` and kept waiters out for that long again. - `berth wire llama-swap` now finds a `berth run` line with a flag written before `--as`, such as `berth run --min-turn 5m --as qwen ...`. Before, it took the model for unwired, and `--write` added a second `berth run` in front of the first. diff --git a/docs/leasing.md b/docs/leasing.md index 79134b2c..84e50d91 100644 --- a/docs/leasing.md +++ b/docs/leasing.md @@ -92,7 +92,7 @@ minimum turn, 3m12s left"), and `berth status` shows the time a lease has left in the TURN column. When the turn ends the normal rules apply: a drain still starts at the slot or job boundary, and a minimum turn never cuts a request off. After a daemon restart, a - re-attach starts a fresh turn. + re-attach goes on with the turn it was in. - **A batch.** A minimum turn is a fixed time; a batch lasts as long as the work. Before an agent or a script queues many jobs on one engine, `berth batch comfyui` (or diff --git a/internal/broker/broker.go b/internal/broker/broker.go index b35af238..4efe10b1 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -248,9 +248,12 @@ type Request struct { // MinTurn is a minimum turn: while the lease is younger than it after // the grant, no waiter drains it, whatever the waiter's priority, // except a hold (a game, a call, the operator's own berth hold). When - // the turn ends the normal rules apply. A re-attach starts a fresh - // turn. 0 = off. + // the turn ends the normal rules apply. A re-attach goes on with the + // turn it was in (Held). 0 = off. MinTurn time.Duration + // Held is how long a re-attach's engine had held the card since its + // last grant or resume, under the daemon that went away. + Held time.Duration // ParkTTL is how long the lease may stay parked before the broker // stops its engine, in place of the node's ParkTTL. 0 = the node's. ParkTTL time.Duration @@ -1842,7 +1845,7 @@ func (b *Broker) reattach(id string, r Request, pid int, start uint64) (_ *Lease l.state, l.reattach = active, true l.pid, l.pidStart = pid, start l.scope = scope - l.granted = now + l.granted = now.Add(-max(r.Held, 0)) l.grantedN = b.order() l.releaseBaseline = b.snap.Baseline() l.releaseGTTBaseline = b.snap.GTTBaseline() diff --git a/internal/broker/minturn_test.go b/internal/broker/minturn_test.go index fe9aa81e..b66c1a2d 100644 --- a/internal/broker/minturn_test.go +++ b/internal/broker/minturn_test.go @@ -221,3 +221,21 @@ func holderDrains(b *Broker, engine string) <-chan string { } return nil } + +// A re-attach after a daemon restart keeps the turn the engine was in: a +// deploy that restarts berthd must not hand every holder a fresh minimum turn. +func TestReattachKeepsTheTurnItWasIn(t *testing.T) { + b, _, _ := setup(t, procs.Fake{Starts: map[int]uint64{100: 7}}) + clk := newMinClock(turnBase) + b.nowFn = clk.now + if _, err := b.Reattach(Request{Engine: "comfyui", Priority: Interactive, MinTurn: 5 * time.Minute, Held: 4 * time.Minute}, 100, 7); err != nil { + t.Fatal(err) + } + clk.set(turnBase.Add(30 * time.Second)) + + w := acquireW(b, Request{Engine: "strata", Priority: Interactive}) + pending(t, w.ch) + if got, want := firstReason(t, w), "next in line: comfyui has its minimum turn, 30s left"; got != want { + t.Errorf("reason = %q, want %q", got, want) + } +} diff --git a/internal/proto/proto.go b/internal/proto/proto.go index e7b97f81..f2bb083d 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -192,6 +192,10 @@ type Msg struct { // younger than it after the grant, no waiter drains it but a hold. // OpAcquire and OpReattach; 0 = off. MinTurn float64 `json:"min_turn,omitempty"` + // Held is how long, in seconds, an OpReattach's engine had held the card + // since its last grant or resume, so the new daemon goes on with that + // turn instead of starting a fresh one. + Held float64 `json:"held,omitempty"` // Max and End are an OpBatch's cap, in seconds, and its end request; // its idle grace travels in Grace. Max float64 `json:"max,omitempty"` diff --git a/internal/runner/park.go b/internal/runner/park.go index a46ac79f..f57cc4d7 100644 --- a/internal/runner/park.go +++ b/internal/runner/park.go @@ -62,6 +62,26 @@ type leaseLine struct { // a drain treats the job as at its boundary (#299). A deadline and not a // flag, so a node that dies mid-yield cannot leave it on. yieldUntil time.Time + // turn is when the lease last got the card, at its grant or a resume. A + // re-attach tells the new daemon how long ago that was. + turn time.Time +} + +// startTurn records that the lease got the card now. +func (l *leaseLine) startTurn() { + l.mu.Lock() + defer l.mu.Unlock() + l.turn = time.Now() +} + +// held is how long the lease has had the card in this turn, 0 before any. +func (l *leaseLine) held() time.Duration { + l.mu.Lock() + defer l.mu.Unlock() + if l.turn.IsZero() { + return 0 + } + return time.Since(l.turn) } // setYield starts a yield that lasts d, or ends it when d is not positive. diff --git a/internal/runner/park_test.go b/internal/runner/park_test.go index a891bb15..8f9b36f9 100644 --- a/internal/runner/park_test.go +++ b/internal/runner/park_test.go @@ -667,3 +667,32 @@ func TestParkedEngineReattachesParked(t *testing.T) { t.Fatalf("the parked engine was drained after the restart; log:\n%s", log.String()) } } + +// A run whose berthd restarts re-attaches inside the turn it was in: the +// new daemon counts the minimum turn from the old grant, so a deploy does +// not give every holder a fresh turn. +func TestReattachGoesOnWithTheTurn(t *testing.T) { + card := &parkCard{total: 24 << 30} + sock := filepath.Join(t.TempDir(), "berth.sock") + b, stop := parkDaemonAt(t, sock, card, 16<<30) + log := &syncBuf{} + exited := make(chan int, 1) + go func() { + exited <- Run(Options{Socket: sock, Engine: "vllm", Priority: "interactive", Needs: []uint64{8 << 30}, + MinTurn: time.Hour, Grace: 200 * time.Millisecond, Argv: []string{"sleep", "30"}, Stderr: log}) + }() + pid := attachAndMeasure(t, b, card) + stopAtEnd(t, pid, exited) + + time.Sleep(2 * time.Second) + stop() + b2, _ := parkDaemonAt(t, sock, card, 16<<30) + waitFor(t, "the re-attach", func() bool { return strings.Contains(log.String(), "re-attached vllm") }) + st := b2.Status().Leases + if len(st) != 1 { + t.Fatalf("leases after the restart = %+v", st) + } + if left := time.Duration(st[0].TurnS * float64(time.Second)); left > time.Hour-2*time.Second { + t.Fatalf("turn left after the restart = %s, want the 2s already held taken off the hour", left) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 637349e6..11bc61e5 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -291,6 +291,9 @@ func Run(o Options) int { // line is the lease's live conn and park state; the drain, the resume // and the container resolver ride it across reattach loops. line := newLine(conn) + if conn != nil { + line.startTurn() + } go forwardSignals(cmd, done, func() bool { return o.Checkpoint && line.isParked() }, line.askStop) // A container engine resolves its containers to their full IDs and names @@ -382,6 +385,7 @@ func Run(o Options) int { case m.Op == proto.OpReturn: go line.returnStep(o, m.LendID, say) case m.Op == proto.OpGranted && m.Resume: + line.startTurn() go line.resume(o, cmd, done, say) case m.Op == proto.OpYield: line.setYield(time.Duration(m.Yield * float64(time.Second))) @@ -525,7 +529,7 @@ func reattach(o Options, cmd *exec.Cmd, done <-chan struct{}, cid *containerRef, func reattachMsg(o Options, pid int, start uint64, cid *containerRef, parked bool, line *leaseLine) proto.Msg { m := proto.Msg{Op: proto.OpReattach, Card: o.Card, Engine: o.Engine, Group: o.Group, Priority: o.Priority, Needs: largest(o.Needs), NeedsEach: each(o.Needs), LearnSize: o.LearnSize, Pinned: o.Pinned, Training: o.Training, Lend: o.Lend != nil, WholeMachine: o.WholeMachine, - MinTurn: o.MinTurn.Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String(), + MinTurn: o.MinTurn.Seconds(), Held: line.held().Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String(), PID: pid, PIDStart: start, Containers: cid.list(), Parked: parked} if t, ok := line.openLend(); ok { m.LendID, m.Lent = t.ID, t.MiB<<20 diff --git a/internal/server/server.go b/internal/server/server.go index e642ef1b..c62bf720 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -288,6 +288,7 @@ func hold(ctx context.Context, conn *proto.Conn, req proto.Msg, n *node.Node, fi LendID: req.LendID, LentBytes: req.Lent, MinTurn: time.Duration(req.MinTurn * float64(time.Second)), + Held: time.Duration(req.Held * float64(time.Second)), ParkTTL: time.Duration(req.ParkTTL * float64(time.Second)), Window: win, Containers: req.ContainerIDs(),