From bdb9ff2c9fbc4580c9f87b433e8b3c9b89bf5f98 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:47:54 -0700 Subject: [PATCH] feat(broker): a training keeps its lock while berthd is away A berth train run records itself as a training when it loses berthd. With berthd down, berth run refuses to start an engine while a live training record exists, instead of starting it without a lease. A berthd that starts while a training runs treats its record as the lock: it refuses every other request until the training re-attaches, with no time limit, where other engines get the 15 second re-attach window. --- CHANGELOG.md | 4 +++ docs/leasing.md | 6 +++-- internal/broker/broker.go | 38 ++++++++++++++++++++------- internal/broker/training_test.go | 45 ++++++++++++++++++++++++++++++++ internal/cli/train_test.go | 25 ++++++++++++++++++ internal/runner/runner.go | 8 +++++- internal/runs/runs.go | 14 ++++++++++ internal/runs/runs_test.go | 34 ++++++++++++++++++++++++ internal/skill/SKILL.md | 5 ++-- 9 files changed, 165 insertions(+), 14 deletions(-) create mode 100644 internal/runs/runs_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 87c4dae..f813646 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ has a matching GitHub release whose body comes from its section below. instead of waiting out their timeout. A second `berth train` waits in line and starts when the first one ends. Other cards keep serving. `berth train` refuses to start without berthd, and `berth status` shows the lease as `hold,pinned,training`. +- A `berth train` run keeps its lock while berthd is away. With berthd down, `berth run` refuses to start an engine + while a training runs on the node, instead of starting it without a lease. A berthd that starts while a training + runs refuses every other request until the training re-attaches, with no time limit, where other engines get + 15 seconds. - On Windows, `berth notify` is a tray icon in the taskbar. Its notifications are Windows' own toasts, and a click opens the harbor. The tooltip shows the latest event, and keeps it after explorer restarts. The menu opens the diff --git a/docs/leasing.md b/docs/leasing.md index b45ff0c..44a3e7b 100644 --- a/docs/leasing.md +++ b/docs/leasing.md @@ -267,8 +267,10 @@ berth train --as train-mymodel --gpu gpu0,gpu1,gpu2 -- torchrun --nproc_per_node - The workers torchrun or `accelerate launch` starts belong to the lease, so their memory is the run's on every card. - `berth status` shows the lease as `hold,pinned,training`. `berth stop train-mymodel` still stops it, because a person asked. -- A berthd restart or upgrade does not stop the run. It keeps running, and berthd re-attaches it with its pin on - every card. The run re-attaches as a training, so the lock comes back with it. +- A berthd restart or upgrade does not stop the run, and the lock holds while berthd is away. With berthd down, + `berth run` refuses to start an engine while the training runs, instead of starting it without a lease. A new + berthd refuses every other request until the training re-attaches, with no time limit, and the training comes + back as a training. `berth run --pinned --priority foreground` still works for a run that must not be drained but may share its card. A pinned run does not lock the card: other requests wait for it until their own deadline. diff --git a/internal/broker/broker.go b/internal/broker/broker.go index 00a6eb0..b1b6a95 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -1069,7 +1069,12 @@ func (b *Broker) runFor(pid int) *runs.Record { // found none at its start, while this daemon is young enough that its berth // run is still dialling, and it has not re-attached yet. func (b *Broker) dueToReattach(r runs.Record, now time.Time) bool { - if (r.Reason != runs.ReasonLost && r.Reason != runs.ReasonStart) || now.Sub(b.started) >= b.cfg.ReattachWindow { + if r.Reason != runs.ReasonLost && r.Reason != runs.ReasonStart { + return false + } + // A training waits as long as its process lives: the card is its own + // until it comes back, however long the restart took. + if !r.Training && now.Sub(b.started) >= b.cfg.ReattachWindow { return false } for _, l := range b.leases { @@ -1962,20 +1967,35 @@ func (b *Broker) training() *lease { return nil } -// lockedFor is the refusal a request that is not a training gets while t -// locks the card. -func (b *Broker) lockedFor(t *lease) string { +// lockedBy names the training that locks this card: a training lease, or a +// training run of the last daemon that has not re-attached yet. "" when +// none does. b.mu must be held. +func (b *Broker) lockedBy(now time.Time) string { + if t := b.training(); t != nil { + return t.engine + } + for _, r := range b.runs { + if r.Training && b.dueToReattach(r, now) { + return r.Engine + } + } + return "" +} + +// lockedFor is the refusal a request that is not a training gets while +// the training engine locks the card. +func (b *Broker) lockedFor(engine string) string { card := b.cfg.Card if card == "" { card = "the card" } - return fmt.Sprintf("%s is locked for training by %s; try again after it finishes", card, t.engine) + return fmt.Sprintf("%s is locked for training by %s; try again after it finishes", card, engine) } // lockOut refuses every waiter on the card that is not a training, as a // training arrives. b.mu must be held. func (b *Broker) lockOut(t *lease) { - why := b.lockedFor(t) + why := b.lockedFor(t.engine) for _, q := range b.queue { if !q.training { select { @@ -1990,10 +2010,10 @@ func (b *Broker) lockOut(t *lease) { // b.mu must be held. func (b *Broker) refuse(r Request) error { why := "" - t := b.training() + lock := b.lockedBy(b.now()) switch limit := sat(b.snap.Total, b.cfg.Reserve); { - case t != nil && !r.Training: - why = b.lockedFor(t) + case lock != "" && !r.Training: + why = b.lockedFor(lock) case b.snap.APU && r.Needs > 0: why = "APU requires an exclusive lease; omit needs-mib (VRAM and GTT are separate pools)" case r.Needs > 0 && b.snap.Total > 0 && r.Needs > limit: diff --git a/internal/broker/training_test.go b/internal/broker/training_test.go index 05df84a..a7288dc 100644 --- a/internal/broker/training_test.go +++ b/internal/broker/training_test.go @@ -4,12 +4,14 @@ package broker import ( + "context" "errors" "strings" "testing" "time" "github.com/T-Crypt/berth/internal/procs" + "github.com/T-Crypt/berth/internal/runs" ) // refusedNow is the refusal a request gets at once, well inside its own wait. @@ -152,3 +154,46 @@ func TestTrainingLocksOutAQueuedGang(t *testing.T) { t.Fatal("the training was not granted once the game ended") } } + +// trainingLost is a node whose training (pid 200) lost the last daemon: its +// record is on disk, and this daemon's re-attach window is long past. +func trainingLost(t *testing.T) *Broker { + t.Helper() + b, _, _ := setup(t, procs.Fake{Starts: map[int]uint64{200: 9}}) + b.cfg.RunsDir = t.TempDir() + b.cfg.ReattachWindow = time.Nanosecond + if _, err := runs.Write(b.cfg.RunsDir, runs.Record{ + Engine: "train-a", PID: 200, Start: 9, Started: time.Now(), Reason: runs.ReasonLost, Training: true, + }); err != nil { + t.Fatal(err) + } + time.Sleep(time.Millisecond) + b.Tick(context.Background()) + return b +} + +// A training that has not re-attached yet still locks the card after a +// daemon restart, with no time limit: other requests are refused at once. +func TestTrainingLocksTheCardUntilItReattaches(t *testing.T) { + b := trainingLost(t) + why := refusedNow(t, "a request before the training re-attaches", acquire(b, Request{Engine: "chat", Priority: Foreground, Wait: time.Hour})) + if !strings.Contains(why, "locked for training by train-a") { + t.Errorf("refusal %q does not name the training", why) + } + + l, err := b.Reattach(Request{Engine: "train-a", Training: true}, 200, 9) + if err != nil { + t.Fatalf("the training's re-attach: %v", err) + } + refusedNow(t, "a request after the training re-attached", acquire(b, Request{Engine: "chat", Priority: Foreground, Wait: time.Hour})) + l.Release() +} + +// The lock ends with the training: once its process is gone, the record +// goes and the card serves again. +func TestTrainingRecordLocksOnlyWhileItsProcessLives(t *testing.T) { + b := trainingLost(t) + b.tree = procs.Fake{Starts: map[int]uint64{}} + b.Tick(context.Background()) + must(t, acquire(b, Request{Engine: "chat", Priority: Foreground})) +} diff --git a/internal/cli/train_test.go b/internal/cli/train_test.go index a9bb9bb..2698280 100644 --- a/internal/cli/train_test.go +++ b/internal/cli/train_test.go @@ -4,11 +4,15 @@ package cli import ( + "os" "path/filepath" "strings" "testing" + "time" + "github.com/T-Crypt/berth/internal/procs" "github.com/T-Crypt/berth/internal/runner" + "github.com/T-Crypt/berth/internal/runs" ) // berth train sets the lease's shape itself, so a flag that would change it @@ -36,3 +40,24 @@ func TestTrainRefusesToStartWithoutADaemon(t *testing.T) { t.Errorf("stdout = %q, want empty", out) } } + +// With berthd down, a run refuses to start while a training that lost the +// daemon still runs on the node, instead of starting unprotected beside it. +func TestRunRefusesWithoutADaemonWhileATrainingRuns(t *testing.T) { + isolate(t) + sock := filepath.Join(t.TempDir(), "berth.sock") + self := os.Getpid() + if _, err := runs.Write(runs.Dir(sock), runs.Record{ + Engine: "train-a", PID: self, Start: procs.System{}.StartTime(self), Started: time.Now(), + Reason: runs.ReasonLost, Training: true, + }); err != nil { + t.Fatal(err) + } + code, out, errOut := runCLI(t, "run", "--as", "chat", "--socket", sock, "--", "echo", "started") + if code != runner.ExitRefused || strings.Contains(out, "started") { + t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal and no start", code, out, errOut) + } + if !strings.Contains(errOut, "train-a is training") { + t.Errorf("stderr %q does not name the training", errOut) + } +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 4db7f61..d85ee94 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -228,6 +228,12 @@ func Run(o Options) int { say("berthd unreachable (%v); not starting %s", err, o.Engine) return ExitRefused } + // A training that lost berthd still owns the card: nothing starts + // beside it unprotected. + if t, ok := runs.Training(runs.Dir(o.Socket), procs.System{}.StartTime); ok { + say("berthd unreachable (%v) and %s is training on this node; not starting %s", err, t.Engine, o.Engine) + return ExitRefused + } // The engine starts unleased and the line below dials on, so a // daemon that comes up later (berthd starting beside this service // at boot) re-attaches it the way it would a run it lost. @@ -753,7 +759,7 @@ func record(o Options, cmd *exec.Cmd, reason string) func() { pid, self := cmd.Process.Pid, os.Getpid() remove, err := runs.Write(runs.Dir(o.Socket), runs.Record{ Engine: o.Engine, PID: pid, Start: procs.System{}.StartTime(pid), Started: time.Now(), Reason: reason, - Wrapper: self, WrapperStart: procs.System{}.StartTime(self), + Wrapper: self, WrapperStart: procs.System{}.StartTime(self), Training: o.Training, }) if err != nil { fmt.Fprintf(o.Stderr, "berth: could not record the unprotected run: %v\n", err) diff --git a/internal/runs/runs.go b/internal/runs/runs.go index 644c343..56127ad 100644 --- a/internal/runs/runs.go +++ b/internal/runs/runs.go @@ -30,6 +30,10 @@ type Record struct { // can re-attach the engine. 0 in a record from before v0.5.1. Wrapper int `json:"wrapper_pid,omitempty"` WrapperStart uint64 `json:"wrapper_start,omitempty"` + // Training marks a berth train run. While it lives without a lease, it + // still locks the node: berth run will not start an engine without + // berthd, and a new berthd refuses other requests until it re-attaches. + Training bool `json:"training,omitempty"` } // WrapperState is "alive" while the berth run that wrote r still runs, @@ -83,6 +87,16 @@ func Write(dir string, r Record) (func(), error) { return func() { os.Remove(p) }, nil } +// Training is a live training record in dir, and whether there is one. +func Training(dir string, alive func(pid int) uint64) (Record, bool) { + for _, r := range List(dir, alive) { + if r.Training { + return r, true + } + } + return Record{}, false +} + // List returns the records whose process is still the same process // (alive reports its current start time, 0 if gone), and deletes the rest. func List(dir string, alive func(pid int) uint64) []Record { diff --git a/internal/runs/runs_test.go b/internal/runs/runs_test.go new file mode 100644 index 0000000..f54b83d --- /dev/null +++ b/internal/runs/runs_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package runs + +import ( + "testing" + "time" +) + +// Training finds a live training record and passes over the rest: a plain +// run, and a training whose process is gone. +func TestTrainingIsALiveTrainingRecord(t *testing.T) { + dir := t.TempDir() + for _, r := range []Record{ + {Engine: "chat", PID: 10, Start: 1, Started: time.Now(), Reason: ReasonLost}, + {Engine: "train-old", PID: 11, Start: 1, Started: time.Now(), Reason: ReasonLost, Training: true}, + {Engine: "train-a", PID: 12, Start: 5, Started: time.Now(), Reason: ReasonLost, Training: true}, + } { + if _, err := Write(dir, r); err != nil { + t.Fatal(err) + } + } + alive := func(pid int) uint64 { + return map[int]uint64{10: 1, 12: 5}[pid] + } + r, ok := Training(dir, alive) + if !ok || r.Engine != "train-a" { + t.Fatalf("Training = %+v, %v; want train-a", r, ok) + } + if _, ok := Training(t.TempDir(), alive); ok { + t.Error("an empty directory has a training") + } +} diff --git a/internal/skill/SKILL.md b/internal/skill/SKILL.md index 1d63f51..9c7e93a 100644 --- a/internal/skill/SKILL.md +++ b/internal/skill/SKILL.md @@ -32,8 +32,9 @@ do not assume any engine is protected. gateway), or it can load onto a card that is already full and crash both. - A **pinned** lease (a hold, a claim, `berth run --pinned`) is never drained. If a request waits "held by ... pinned", the user's game or app has the card. Tell them; do not work around it. -- A refusal that says a card "is locked for training" means a `berth train` run has it. Do not retry in a - loop, start the engine another way, or stop the training. Tell the user, or use another card or node. +- A refusal that says a card "is locked for training", or that a run "is training on this node", means a + `berth train` run has it. Do not retry in a loop, start the engine another way, restart berthd, or stop the + training. Tell the user, or use another card or node. - Do not edit berth.toml, systemd units or an engine's config without showing the change and its undo first. Run `berth config check` after any berth.toml edit.