Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ has a matching GitHub release whose body comes from its section below.

### Fixed

- A berthd restart no longer gives every running engine a fresh minimum turn. `berth run` now tells the new berthd
how long its engine had held the card, and the re-attached lease goes on with the turn it was in. Before, each
deploy reset the TURN column to the full `min_turn` and kept waiters out for that long again.
- `berth wire llama-swap` now finds a `berth run` line with a flag written before `--as`, such as
`berth run --min-turn 5m --as qwen ...`. Before, it took the model for unwired, and `--write` added a second
`berth run` in front of the first.
Expand Down
2 changes: 1 addition & 1 deletion docs/leasing.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@
minimum turn, 3m12s left"), and `berth status` shows the time a lease has left in the
TURN column. When the turn ends the normal rules apply: a drain still starts at the slot
or job boundary, and a minimum turn never cuts a request off. After a daemon restart, a
re-attach starts a fresh turn.
re-attach goes on with the turn it was in.

- **A batch.** A minimum turn is a fixed time; a batch lasts as long as the work. Before an
agent or a script queues many jobs on one engine, `berth batch comfyui` (or
Expand Down
9 changes: 6 additions & 3 deletions internal/broker/broker.go
Original file line number Diff line number Diff line change
Expand Up @@ -248,9 +248,12 @@ type Request struct {
// MinTurn is a minimum turn: while the lease is younger than it after
// the grant, no waiter drains it, whatever the waiter's priority,
// except a hold (a game, a call, the operator's own berth hold). When
// the turn ends the normal rules apply. A re-attach starts a fresh
// turn. 0 = off.
// the turn ends the normal rules apply. A re-attach goes on with the
// turn it was in (Held). 0 = off.
MinTurn time.Duration
// Held is how long a re-attach's engine had held the card since its
// last grant or resume, under the daemon that went away.
Held time.Duration
// ParkTTL is how long the lease may stay parked before the broker
// stops its engine, in place of the node's ParkTTL. 0 = the node's.
ParkTTL time.Duration
Expand Down Expand Up @@ -1842,7 +1845,7 @@ func (b *Broker) reattach(id string, r Request, pid int, start uint64) (_ *Lease
l.state, l.reattach = active, true
l.pid, l.pidStart = pid, start
l.scope = scope
l.granted = now
l.granted = now.Add(-max(r.Held, 0))
l.grantedN = b.order()
l.releaseBaseline = b.snap.Baseline()
l.releaseGTTBaseline = b.snap.GTTBaseline()
Expand Down
18 changes: 18 additions & 0 deletions internal/broker/minturn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -221,3 +221,21 @@ func holderDrains(b *Broker, engine string) <-chan string {
}
return nil
}

// A re-attach after a daemon restart keeps the turn the engine was in: a
// deploy that restarts berthd must not hand every holder a fresh minimum turn.
func TestReattachKeepsTheTurnItWasIn(t *testing.T) {
b, _, _ := setup(t, procs.Fake{Starts: map[int]uint64{100: 7}})
clk := newMinClock(turnBase)
b.nowFn = clk.now
if _, err := b.Reattach(Request{Engine: "comfyui", Priority: Interactive, MinTurn: 5 * time.Minute, Held: 4 * time.Minute}, 100, 7); err != nil {
t.Fatal(err)
}
clk.set(turnBase.Add(30 * time.Second))

w := acquireW(b, Request{Engine: "strata", Priority: Interactive})
pending(t, w.ch)
if got, want := firstReason(t, w), "next in line: comfyui has its minimum turn, 30s left"; got != want {
t.Errorf("reason = %q, want %q", got, want)
}
}
4 changes: 4 additions & 0 deletions internal/proto/proto.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,10 @@ type Msg struct {
// younger than it after the grant, no waiter drains it but a hold.
// OpAcquire and OpReattach; 0 = off.
MinTurn float64 `json:"min_turn,omitempty"`
// Held is how long, in seconds, an OpReattach's engine had held the card
// since its last grant or resume, so the new daemon goes on with that
// turn instead of starting a fresh one.
Held float64 `json:"held,omitempty"`
// Max and End are an OpBatch's cap, in seconds, and its end request;
// its idle grace travels in Grace.
Max float64 `json:"max,omitempty"`
Expand Down
20 changes: 20 additions & 0 deletions internal/runner/park.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,26 @@ type leaseLine struct {
// a drain treats the job as at its boundary (#299). A deadline and not a
// flag, so a node that dies mid-yield cannot leave it on.
yieldUntil time.Time
// turn is when the lease last got the card, at its grant or a resume. A
// re-attach tells the new daemon how long ago that was.
turn time.Time
}

// startTurn records that the lease got the card now.
func (l *leaseLine) startTurn() {
l.mu.Lock()
defer l.mu.Unlock()
l.turn = time.Now()
}

// held is how long the lease has had the card in this turn, 0 before any.
func (l *leaseLine) held() time.Duration {
l.mu.Lock()
defer l.mu.Unlock()
if l.turn.IsZero() {
return 0
}
return time.Since(l.turn)
}

// setYield starts a yield that lasts d, or ends it when d is not positive.
Expand Down
29 changes: 29 additions & 0 deletions internal/runner/park_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -667,3 +667,32 @@ func TestParkedEngineReattachesParked(t *testing.T) {
t.Fatalf("the parked engine was drained after the restart; log:\n%s", log.String())
}
}

// A run whose berthd restarts re-attaches inside the turn it was in: the
// new daemon counts the minimum turn from the old grant, so a deploy does
// not give every holder a fresh turn.
func TestReattachGoesOnWithTheTurn(t *testing.T) {
card := &parkCard{total: 24 << 30}
sock := filepath.Join(t.TempDir(), "berth.sock")
b, stop := parkDaemonAt(t, sock, card, 16<<30)
log := &syncBuf{}
exited := make(chan int, 1)
go func() {
exited <- Run(Options{Socket: sock, Engine: "vllm", Priority: "interactive", Needs: []uint64{8 << 30},
MinTurn: time.Hour, Grace: 200 * time.Millisecond, Argv: []string{"sleep", "30"}, Stderr: log})
}()
pid := attachAndMeasure(t, b, card)
stopAtEnd(t, pid, exited)

time.Sleep(2 * time.Second)
stop()
b2, _ := parkDaemonAt(t, sock, card, 16<<30)
waitFor(t, "the re-attach", func() bool { return strings.Contains(log.String(), "re-attached vllm") })
st := b2.Status().Leases
if len(st) != 1 {
t.Fatalf("leases after the restart = %+v", st)
}
if left := time.Duration(st[0].TurnS * float64(time.Second)); left > time.Hour-2*time.Second {
t.Fatalf("turn left after the restart = %s, want the 2s already held taken off the hour", left)
}
}
6 changes: 5 additions & 1 deletion internal/runner/runner.go
Original file line number Diff line number Diff line change
Expand Up @@ -291,6 +291,9 @@ func Run(o Options) int {
// line is the lease's live conn and park state; the drain, the resume
// and the container resolver ride it across reattach loops.
line := newLine(conn)
if conn != nil {
line.startTurn()
}
go forwardSignals(cmd, done, func() bool { return o.Checkpoint && line.isParked() }, line.askStop)

// A container engine resolves its containers to their full IDs and names
Expand Down Expand Up @@ -382,6 +385,7 @@ func Run(o Options) int {
case m.Op == proto.OpReturn:
go line.returnStep(o, m.LendID, say)
case m.Op == proto.OpGranted && m.Resume:
line.startTurn()
go line.resume(o, cmd, done, say)
case m.Op == proto.OpYield:
line.setYield(time.Duration(m.Yield * float64(time.Second)))
Expand Down Expand Up @@ -525,7 +529,7 @@ func reattach(o Options, cmd *exec.Cmd, done <-chan struct{}, cid *containerRef,
func reattachMsg(o Options, pid int, start uint64, cid *containerRef, parked bool, line *leaseLine) proto.Msg {
m := proto.Msg{Op: proto.OpReattach, Card: o.Card, Engine: o.Engine, Group: o.Group, Priority: o.Priority,
Needs: largest(o.Needs), NeedsEach: each(o.Needs), LearnSize: o.LearnSize, Pinned: o.Pinned, Training: o.Training, Lend: o.Lend != nil, WholeMachine: o.WholeMachine,
MinTurn: o.MinTurn.Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String(),
MinTurn: o.MinTurn.Seconds(), Held: line.held().Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String(),
PID: pid, PIDStart: start, Containers: cid.list(), Parked: parked}
if t, ok := line.openLend(); ok {
m.LendID, m.Lent = t.ID, t.MiB<<20
Expand Down
1 change: 1 addition & 0 deletions internal/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ func hold(ctx context.Context, conn *proto.Conn, req proto.Msg, n *node.Node, fi
LendID: req.LendID,
LentBytes: req.Lent,
MinTurn: time.Duration(req.MinTurn * float64(time.Second)),
Held: time.Duration(req.Held * float64(time.Second)),
ParkTTL: time.Duration(req.ParkTTL * float64(time.Second)),
Window: win,
Containers: req.ContainerIDs(),
Expand Down
Loading