diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ada899..d83d28c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,9 @@ has a matching GitHub release whose body comes from its section below. ### Fixed +- A berthd restart no longer gives every running engine a fresh minimum turn. `berth run` now tells the new berthd + how long its engine had held the card, and the re-attached lease goes on with the turn it was in. Before, each + deploy reset the TURN column to the full `min_turn` and kept waiters out for that long again. - `berth wire llama-swap` now finds a `berth run` line with a flag written before `--as`, such as `berth run --min-turn 5m --as qwen ...`. Before, it took the model for unwired, and `--write` added a second `berth run` in front of the first. diff --git a/docs/leasing.md b/docs/leasing.md index 79134b2..84e50d9 100644 --- a/docs/leasing.md +++ b/docs/leasing.md @@ -92,7 +92,7 @@ minimum turn, 3m12s left"), and `berth status` shows the time a lease has left in the TURN column. When the turn ends the normal rules apply: a drain still starts at the slot or job boundary, and a minimum turn never cuts a request off. After a daemon restart, a - re-attach starts a fresh turn. + re-attach goes on with the turn it was in. - **A batch.** A minimum turn is a fixed time; a batch lasts as long as the work. Before an agent or a script queues many jobs on one engine, `berth batch comfyui` (or diff --git a/internal/broker/broker.go b/internal/broker/broker.go index b35af23..4efe10b 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -248,9 +248,12 @@ type Request struct { // MinTurn is a minimum turn: while the lease is younger than it after // the grant, no waiter drains it, whatever the waiter's priority, // except a hold (a game, a call, the operator's own berth hold). When - // the turn ends the normal rules apply. A re-attach starts a fresh - // turn. 0 = off. + // the turn ends the normal rules apply. A re-attach goes on with the + // turn it was in (Held). 0 = off. MinTurn time.Duration + // Held is how long a re-attach's engine had held the card since its + // last grant or resume, under the daemon that went away. + Held time.Duration // ParkTTL is how long the lease may stay parked before the broker // stops its engine, in place of the node's ParkTTL. 0 = the node's. ParkTTL time.Duration @@ -1842,7 +1845,7 @@ func (b *Broker) reattach(id string, r Request, pid int, start uint64) (_ *Lease l.state, l.reattach = active, true l.pid, l.pidStart = pid, start l.scope = scope - l.granted = now + l.granted = now.Add(-max(r.Held, 0)) l.grantedN = b.order() l.releaseBaseline = b.snap.Baseline() l.releaseGTTBaseline = b.snap.GTTBaseline() diff --git a/internal/broker/minturn_test.go b/internal/broker/minturn_test.go index fe9aa81..b66c1a2 100644 --- a/internal/broker/minturn_test.go +++ b/internal/broker/minturn_test.go @@ -221,3 +221,21 @@ func holderDrains(b *Broker, engine string) <-chan string { } return nil } + +// A re-attach after a daemon restart keeps the turn the engine was in: a +// deploy that restarts berthd must not hand every holder a fresh minimum turn. +func TestReattachKeepsTheTurnItWasIn(t *testing.T) { + b, _, _ := setup(t, procs.Fake{Starts: map[int]uint64{100: 7}}) + clk := newMinClock(turnBase) + b.nowFn = clk.now + if _, err := b.Reattach(Request{Engine: "comfyui", Priority: Interactive, MinTurn: 5 * time.Minute, Held: 4 * time.Minute}, 100, 7); err != nil { + t.Fatal(err) + } + clk.set(turnBase.Add(30 * time.Second)) + + w := acquireW(b, Request{Engine: "strata", Priority: Interactive}) + pending(t, w.ch) + if got, want := firstReason(t, w), "next in line: comfyui has its minimum turn, 30s left"; got != want { + t.Errorf("reason = %q, want %q", got, want) + } +} diff --git a/internal/proto/proto.go b/internal/proto/proto.go index e7b97f8..f2bb083 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -192,6 +192,10 @@ type Msg struct { // younger than it after the grant, no waiter drains it but a hold. // OpAcquire and OpReattach; 0 = off. MinTurn float64 `json:"min_turn,omitempty"` + // Held is how long, in seconds, an OpReattach's engine had held the card + // since its last grant or resume, so the new daemon goes on with that + // turn instead of starting a fresh one. + Held float64 `json:"held,omitempty"` // Max and End are an OpBatch's cap, in seconds, and its end request; // its idle grace travels in Grace. Max float64 `json:"max,omitempty"` diff --git a/internal/runner/park.go b/internal/runner/park.go index a46ac79..f57cc4d 100644 --- a/internal/runner/park.go +++ b/internal/runner/park.go @@ -62,6 +62,26 @@ type leaseLine struct { // a drain treats the job as at its boundary (#299). A deadline and not a // flag, so a node that dies mid-yield cannot leave it on. yieldUntil time.Time + // turn is when the lease last got the card, at its grant or a resume. A + // re-attach tells the new daemon how long ago that was. + turn time.Time +} + +// startTurn records that the lease got the card now. +func (l *leaseLine) startTurn() { + l.mu.Lock() + defer l.mu.Unlock() + l.turn = time.Now() +} + +// held is how long the lease has had the card in this turn, 0 before any. +func (l *leaseLine) held() time.Duration { + l.mu.Lock() + defer l.mu.Unlock() + if l.turn.IsZero() { + return 0 + } + return time.Since(l.turn) } // setYield starts a yield that lasts d, or ends it when d is not positive. diff --git a/internal/runner/park_test.go b/internal/runner/park_test.go index a891bb1..8f9b36f 100644 --- a/internal/runner/park_test.go +++ b/internal/runner/park_test.go @@ -667,3 +667,32 @@ func TestParkedEngineReattachesParked(t *testing.T) { t.Fatalf("the parked engine was drained after the restart; log:\n%s", log.String()) } } + +// A run whose berthd restarts re-attaches inside the turn it was in: the +// new daemon counts the minimum turn from the old grant, so a deploy does +// not give every holder a fresh turn. +func TestReattachGoesOnWithTheTurn(t *testing.T) { + card := &parkCard{total: 24 << 30} + sock := filepath.Join(t.TempDir(), "berth.sock") + b, stop := parkDaemonAt(t, sock, card, 16<<30) + log := &syncBuf{} + exited := make(chan int, 1) + go func() { + exited <- Run(Options{Socket: sock, Engine: "vllm", Priority: "interactive", Needs: []uint64{8 << 30}, + MinTurn: time.Hour, Grace: 200 * time.Millisecond, Argv: []string{"sleep", "30"}, Stderr: log}) + }() + pid := attachAndMeasure(t, b, card) + stopAtEnd(t, pid, exited) + + time.Sleep(2 * time.Second) + stop() + b2, _ := parkDaemonAt(t, sock, card, 16<<30) + waitFor(t, "the re-attach", func() bool { return strings.Contains(log.String(), "re-attached vllm") }) + st := b2.Status().Leases + if len(st) != 1 { + t.Fatalf("leases after the restart = %+v", st) + } + if left := time.Duration(st[0].TurnS * float64(time.Second)); left > time.Hour-2*time.Second { + t.Fatalf("turn left after the restart = %s, want the 2s already held taken off the hour", left) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 637349e..11bc61e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -291,6 +291,9 @@ func Run(o Options) int { // line is the lease's live conn and park state; the drain, the resume // and the container resolver ride it across reattach loops. line := newLine(conn) + if conn != nil { + line.startTurn() + } go forwardSignals(cmd, done, func() bool { return o.Checkpoint && line.isParked() }, line.askStop) // A container engine resolves its containers to their full IDs and names @@ -382,6 +385,7 @@ func Run(o Options) int { case m.Op == proto.OpReturn: go line.returnStep(o, m.LendID, say) case m.Op == proto.OpGranted && m.Resume: + line.startTurn() go line.resume(o, cmd, done, say) case m.Op == proto.OpYield: line.setYield(time.Duration(m.Yield * float64(time.Second))) @@ -525,7 +529,7 @@ func reattach(o Options, cmd *exec.Cmd, done <-chan struct{}, cid *containerRef, func reattachMsg(o Options, pid int, start uint64, cid *containerRef, parked bool, line *leaseLine) proto.Msg { m := proto.Msg{Op: proto.OpReattach, Card: o.Card, Engine: o.Engine, Group: o.Group, Priority: o.Priority, Needs: largest(o.Needs), NeedsEach: each(o.Needs), LearnSize: o.LearnSize, Pinned: o.Pinned, Training: o.Training, Lend: o.Lend != nil, WholeMachine: o.WholeMachine, - MinTurn: o.MinTurn.Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String(), + MinTurn: o.MinTurn.Seconds(), Held: line.held().Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String(), PID: pid, PIDStart: start, Containers: cid.list(), Parked: parked} if t, ok := line.openLend(); ok { m.LendID, m.Lent = t.ID, t.MiB<<20 diff --git a/internal/server/server.go b/internal/server/server.go index e642ef1..c62bf72 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -288,6 +288,7 @@ func hold(ctx context.Context, conn *proto.Conn, req proto.Msg, n *node.Node, fi LendID: req.LendID, LentBytes: req.Lent, MinTurn: time.Duration(req.MinTurn * float64(time.Second)), + Held: time.Duration(req.Held * float64(time.Second)), ParkTTL: time.Duration(req.ParkTTL * float64(time.Second)), Window: win, Containers: req.ContainerIDs(),