From 383c09d6c97a671685112119b7ce697f45c93e13 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:55:56 -0700 Subject: [PATCH] feat(runner): berth train --exclusive locks the card in the driver berth train --exclusive puts the granted cards in exclusive-process compute mode after the grant and before the command starts, so a program that opens CUDA without Berth cannot open them, and puts them back to the default mode when the run ends. It refuses to start when another compute process is on the card, when no sudo rule lets nvidia-smi -c run without a password, and on a card with no NVIDIA UUID. Linux only. --- CHANGELOG.md | 4 ++ docs/leasing.md | 28 +++++++++ internal/cli/commands.go | 6 +- internal/cli/run.go | 6 +- internal/cli/train_unix_test.go | 24 +++++++ internal/runner/exclusive.go | 50 +++++++++++++++ internal/runner/exclusive_linux_test.go | 84 +++++++++++++++++++++++++ internal/runner/options.go | 3 +- internal/runner/runner.go | 33 +++++++--- 9 files changed, 225 insertions(+), 13 deletions(-) create mode 100644 internal/runner/exclusive.go create mode 100644 internal/runner/exclusive_linux_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f813646..c7f2094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ has a matching GitHub release whose body comes from its section below. instead of waiting out their timeout. A second `berth train` waits in line and starts when the first one ends. Other cards keep serving. `berth train` refuses to start without berthd, and `berth status` shows the lease as `hold,pinned,training`. +- `berth train --exclusive` also locks the card in the driver on Linux with an NVIDIA card. It puts the granted cards + in exclusive-process compute mode for the run's life, so a program that opens CUDA without Berth cannot open + them, and puts them back to the default mode when the run ends. It needs a sudo rule for `nvidia-smi -c`, and + refuses to start without one, or when another compute process is already on the card. - A `berth train` run keeps its lock while berthd is away. With berthd down, `berth run` refuses to start an engine while a training runs on the node, instead of starting it without a lease. A berthd that starts while a training runs refuses every other request until the training re-attaches, with no time limit, where other engines get diff --git a/docs/leasing.md b/docs/leasing.md index 44a3e7b..a9e000e 100644 --- a/docs/leasing.md +++ b/docs/leasing.md @@ -273,6 +273,34 @@ berth train --as train-mymodel --gpu gpu0,gpu1,gpu2 -- torchrun --nproc_per_node back as a training. `berth run --pinned --priority foreground` still works for a run that must not be drained but may share its card. + +### Lock the card in the driver too + +The training lock covers everything that asks Berth for the card. A program that opens CUDA without Berth only +shows up as a stowaway. `berth train --exclusive` closes that gap on Linux with an NVIDIA card. After the grant, it +puts the cards in the driver's exclusive-process compute mode, so the training's process is the only one that can +open a CUDA context there. When the run ends, it puts them back to the default mode. + +```sh +berth train --exclusive --as train-mymodel -- python train.py +``` + +- Setting the mode needs root. Give your user a sudo rule for `nvidia-smi -c` alone, with `sudo visudo -f + /etc/sudoers.d/berth-exclusive`: + + ``` + YOUR_USER ALL=(root) NOPASSWD: /usr/bin/nvidia-smi -i * -c EXCLUSIVE_PROCESS, /usr/bin/nvidia-smi -i * -c DEFAULT + ``` + + Without the rule, `berth train --exclusive` refuses to start instead of running without the guard. +- The run refuses to start when another compute process is already on the card, because that process would own + the card's one context and the training could not open it. +- Exclusive mode allows one process per card. A training that opens CUDA from several processes on one card + (DataLoader workers that use CUDA, two ranks on one card) fails to start in it. One rank per card, as torchrun + gives each card, works. +- Graphics are not compute: the desktop and games keep drawing on the card. +- If the `berth train` process is killed with SIGKILL, the mode stays set. `sudo nvidia-smi -c DEFAULT` puts it + back. A pinned run does not lock the card: other requests wait for it until their own deadline. A pin does not stop memory taken outside Berth. A game started on a leased card waits behind the run in Berth but diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 6c3da9f..a4f5def 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -447,7 +447,11 @@ waits in line and starts when the first one ends. Other cards keep serving. A game or a hold that already has the card is not drained: the training waits for it. berth train refuses to start when berthd is not running, because the -lock lives in berthd.`, +lock lives in berthd. + +--exclusive (Linux, NVIDIA) also puts the cards in the driver's exclusive-process +compute mode for the run's life, so a program that opens CUDA without Berth +cannot open them. It needs a sudo rule for nvidia-smi -c (see docs/leasing.md).`, Run: runTrain, }, { diff --git a/internal/cli/run.go b/internal/cli/run.go index 309ac85..a1d29b8 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -62,6 +62,7 @@ func runEngine(c *Context, training bool) int { sliceName := fs.String("slice", "", "NAME.slice to start the engine's scope under (systemd user manager, Linux): the slice unit's CPU, memory and swap policy applies to the engine and everything it forks; over the entry's slice") grace := fs.Duration("grace", 30*time.Second, "time each drain step gets before the next") failClosed := fs.Bool("fail-closed", false, "do not start CMD without berthd or stop it without a safe boundary") + exclusiveMode := fs.Bool("exclusive", false, "berth train only: put the cards in exclusive-process compute mode for the run's life, so no program outside Berth can open them (Linux, NVIDIA, needs a sudo rule for nvidia-smi -c)") cfgPath := fs.String("config", config.DefaultPath(), "berth.toml: the engine entry matching --as (or --group) supplies the settings a flag does not set; a set flag wins and says so; missing = no defaults ($BERTH_CONFIG)") if code, done := c.Parse(fs); done { return code @@ -71,6 +72,9 @@ func runEngine(c *Context, training bool) int { if *engine == "" { return c.fail(required) } + if *exclusiveMode && !training { + return c.fail(Systemf("--exclusive is for berth train: exclusive mode allows one process on the card, which only a training should claim")) + } if training { for _, name := range trainSets { if c.Changed(name) { @@ -183,7 +187,7 @@ func runEngine(c *Context, training bool) int { // Nothing here goes to stdout: the engine owns it. berth's own notices are // diagnostics on stderr, and --quiet never hides the unprotected warning. return runner.Run(runner.OptionsFor(&e, runner.Over{ - Socket: *sock, Front: *frontAddr, Group: *group, Pinned: *pinned, Training: training, Switched: *switched, Wait: *wait, Grace: *grace, + Socket: *sock, Front: *frontAddr, Group: *group, Pinned: *pinned, Training: training, Exclusive: *exclusiveMode, Switched: *switched, Wait: *wait, Grace: *grace, FailClosed: *failClosed, Argv: argv, Stderr: c.Err, Log: c.Log(), })) } diff --git a/internal/cli/train_unix_test.go b/internal/cli/train_unix_test.go index 6b3e61d..7b8175e 100644 --- a/internal/cli/train_unix_test.go +++ b/internal/cli/train_unix_test.go @@ -49,3 +49,27 @@ func TestTrainLocksTheCardAgainstBerthRun(t *testing.T) { t.Errorf("berth train exit = %d, want 0", code) } } + +// --exclusive is part of the run: on a card the driver names no UUID for, +// berth train --exclusive is refused after the grant and its command never +// starts, so it never runs without the guard it asked for. +func TestTrainExclusiveRefusesWithoutAnNVIDIACard(t *testing.T) { + isolate(t) + sock := startDaemon(t) + code, out, errOut := runCLI(t, "train", "--exclusive", "--as", "train-a", "--socket", sock, "--", "echo", "started") + if code == ExitOK || strings.Contains(out, "started") { + t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal and no start", code, out, errOut) + } + if !strings.Contains(errOut, "--exclusive") { + t.Errorf("stderr %q does not name --exclusive", errOut) + } +} + +// --exclusive belongs to berth train alone. +func TestRunRefusesExclusive(t *testing.T) { + isolate(t) + code, _, errOut := runCLI(t, "run", "--exclusive", "--as", "x", "--", "true") + if code != ExitSystem || !strings.Contains(errOut, "--exclusive is for berth train") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } +} diff --git a/internal/runner/exclusive.go b/internal/runner/exclusive.go new file mode 100644 index 0000000..85b8f99 --- /dev/null +++ b/internal/runner/exclusive.go @@ -0,0 +1,50 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package runner + +import ( + "errors" + "fmt" + "runtime" + "strings" + "time" +) + +// exclusiveTimeout bounds each nvidia-smi call the exclusive mode makes. +const exclusiveTimeout = 30 * time.Second + +// exclusive puts the granted cards (their UUIDs, comma separated) in the +// driver's exclusive-process compute mode, so the run's process is the only +// one that can open a CUDA context there, and returns the func that puts +// them back to the default mode. It refuses when another compute process is +// already on the cards: in exclusive mode that process would lock the run +// itself out. Setting the mode needs root, through `sudo -n`, so a missing +// sudo rule refuses the run instead of starting it without the guard. +func exclusive(o Options, uuids string) (func(func(string, ...any)), error) { + if runtime.GOOS != "linux" { + return nil, errors.New("exclusive compute mode is Linux only") + } + if uuids == "" { + return nil, errors.New("the granted card has no NVIDIA UUID") + } + out, err := o.CLI([]string{"nvidia-smi", "-i", uuids, "--query-compute-apps=pid,process_name", "--format=csv,noheader"}, exclusiveTimeout) + if err != nil { + return nil, fmt.Errorf("nvidia-smi could not list the compute processes: %v", err) + } + if apps := strings.TrimSpace(out); apps != "" { + return nil, fmt.Errorf("another compute process is on the card (%s); exclusive mode would lock the training out", strings.ReplaceAll(apps, "\n", "; ")) + } + set := func(mode string) error { + _, err := o.CLI([]string{"sudo", "-n", "nvidia-smi", "-i", uuids, "-c", mode}, exclusiveTimeout) + return err + } + if err := set("EXCLUSIVE_PROCESS"); err != nil { + return nil, fmt.Errorf("could not set exclusive-process mode (it needs a sudo rule that runs nvidia-smi -c without a password): %v", err) + } + return func(say func(string, ...any)) { + if err := set("DEFAULT"); err != nil { + say("could not put %s back to the default compute mode: %v; run: sudo nvidia-smi -i %s -c DEFAULT", uuids, err, uuids) + } + }, nil +} diff --git a/internal/runner/exclusive_linux_test.go b/internal/runner/exclusive_linux_test.go new file mode 100644 index 0000000..818420a --- /dev/null +++ b/internal/runner/exclusive_linux_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package runner + +import ( + "errors" + "slices" + "strings" + "testing" + "time" +) + +// fakeSMI answers the exclusive mode's nvidia-smi calls: apps is what the +// compute-app query prints, sudoErr fails the mode change. It records every +// argv. +type fakeSMI struct { + apps string + sudoErr error + calls [][]string +} + +func (f *fakeSMI) cli(argv []string, _ time.Duration) (string, error) { + f.calls = append(f.calls, argv) + if argv[0] == "sudo" { + return "", f.sudoErr + } + return f.apps, nil +} + +// On an empty card the mode goes to exclusive-process before the run, and +// back to the default when it ends, on the granted cards only. +func TestExclusiveSetsAndRestoresTheMode(t *testing.T) { + f := &fakeSMI{} + restore, err := exclusive(Options{CLI: f.cli}, "GPU-a,GPU-b") + if err != nil { + t.Fatal(err) + } + set := []string{"sudo", "-n", "nvidia-smi", "-i", "GPU-a,GPU-b", "-c", "EXCLUSIVE_PROCESS"} + if len(f.calls) != 2 || !slices.Equal(f.calls[1], set) { + t.Fatalf("calls = %q, want the query then %q", f.calls, set) + } + restore(func(string, ...any) {}) + back := []string{"sudo", "-n", "nvidia-smi", "-i", "GPU-a,GPU-b", "-c", "DEFAULT"} + if len(f.calls) != 3 || !slices.Equal(f.calls[2], back) { + t.Fatalf("calls = %q, want %q last", f.calls, back) + } +} + +// A compute process already on the card would own its one context and lock +// the training out, so the run is refused and the mode is left alone. +func TestExclusiveRefusesWhenAProcessIsOnTheCard(t *testing.T) { + f := &fakeSMI{apps: "4242, python3\n"} + _, err := exclusive(Options{CLI: f.cli}, "GPU-a") + if err == nil || !strings.Contains(err.Error(), "4242, python3") { + t.Fatalf("err = %v, want a refusal naming the process", err) + } + for _, c := range f.calls { + if c[0] == "sudo" { + t.Fatalf("the mode was changed beside a running process: %q", c) + } + } +} + +// Without a sudo rule the run is refused instead of starting without the +// guard the operator asked for. +func TestExclusiveRefusesWithoutSudo(t *testing.T) { + f := &fakeSMI{sudoErr: errors.New("a password is required")} + _, err := exclusive(Options{CLI: f.cli}, "GPU-a") + if err == nil || !strings.Contains(err.Error(), "sudo rule") { + t.Fatalf("err = %v, want it to name the sudo rule", err) + } +} + +// A card with no NVIDIA UUID (an AMD card) has no exclusive mode to set. +func TestExclusiveNeedsAnNVIDIACard(t *testing.T) { + f := &fakeSMI{} + if _, err := exclusive(Options{CLI: f.cli}, ""); err == nil { + t.Fatal("exclusive mode set on a card with no UUID") + } + if len(f.calls) != 0 { + t.Errorf("calls = %q, want none", f.calls) + } +} diff --git a/internal/runner/options.go b/internal/runner/options.go index dae3bbb..23c5257 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,6 +25,7 @@ type Over struct { Group string Pinned bool Training bool + Exclusive bool Switched bool Wait time.Duration Grace time.Duration @@ -62,7 +63,7 @@ func OptionsFor(e *config.Engine, o Over) Options { } opts := Options{ Socket: o.Socket, Engine: e.Name, Card: e.GPU, Group: o.Group, Priority: prio, - Needs: e.NeedsMiB.Bytes(), Pinned: o.Pinned, Training: o.Training, Switched: o.Switched, Wait: o.Wait, + Needs: e.NeedsMiB.Bytes(), Pinned: o.Pinned, Training: o.Training, Exclusive: o.Exclusive, Switched: o.Switched, Wait: o.Wait, DrainPost: e.DrainPost, DrainSlots: e.DrainSlots, DrainComfyUI: e.DrainComfyUI, DrainMax: e.DrainMax.Duration, MinTurn: e.MinTurn.Duration, ParkTTL: e.ParkTTL.Duration, Window: e.Window, Container: e.Container, Compose: e.Compose, APIKeyEnv: e.APIKeyEnv, APIKeyFile: e.APIKeyFile, diff --git a/internal/runner/runner.go b/internal/runner/runner.go index d85ee94..637349e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -56,6 +56,10 @@ type Options struct { LearnSize bool Pinned bool // never drained: an app that must not be stopped (Blender, a capture) Training bool // berth train: a pinned hold that locks its cards; berthd refuses every other request there + // Exclusive: berth train --exclusive puts the granted cards in the + // driver's exclusive-process compute mode for the run's life, so no + // program outside Berth can open them either. + Exclusive bool Switched bool // a person's switch (the wake page's Launch, /v1/berth/switch): wins a juggle // MinTurn is a minimum turn: while the lease is younger than it after // the grant, no request of equal or lower priority drains it, and @@ -241,13 +245,25 @@ func Run(o Options) int { conn, raw = nil, nil } + if o.CLI == nil { + o.CLI = execCLI + } var cardEnv string if conn != nil { defer raw.Close() - var ok bool - if cardEnv, ok = acquire(o, conn, say); !ok { + granted, ok := acquire(o, conn, say) + if !ok { return ExitRefused } + cardEnv = visibleDevices(o.Card, granted, os.LookupEnv, say) + if o.Exclusive { + restore, err := exclusive(o, granted.CardUUID) + if err != nil { + say("--exclusive: %v; not starting %s", err, o.Engine) + return ExitRefused + } + defer restore(say) + } } cmd := command(o, o.Output) @@ -269,9 +285,6 @@ func Run(o Options) int { done := make(chan struct{}) - if o.CLI == nil { - o.CLI = execCLI - } var cid containerRef o.container = &cid @@ -424,30 +437,30 @@ func Run(o Options) int { // acquire asks berthd for the card and waits for the grant. It returns the // device entry the engine starts with, and false when there is no grant. -func acquire(o Options, conn *proto.Conn, say func(string, ...any)) (string, bool) { +func acquire(o Options, conn *proto.Conn, say func(string, ...any)) (proto.Msg, bool) { if err := conn.Send(proto.Msg{Op: proto.OpAcquire, Card: o.Card, Engine: o.Engine, Group: o.Group, Priority: o.Priority, Needs: largest(o.Needs), NeedsEach: each(o.Needs), LearnSize: o.LearnSize, Wait: o.Wait.Seconds(), Pinned: o.Pinned, Training: o.Training, Switched: o.Switched, Lend: o.Lend != nil, WholeMachine: o.WholeMachine, MinTurn: o.MinTurn.Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String()}); err != nil { say("acquire: %v", err) - return "", false + return proto.Msg{}, false } for { m, err := conn.Recv() if err != nil { say("berthd went away while %s was waiting: %v", o.Engine, err) - return "", false + return proto.Msg{}, false } switch m.Op { case proto.OpWaiting: say("waiting: %s", m.Reason) case proto.OpRefused: say("refused: %s", m.Reason) - return "", false + return proto.Msg{}, false case proto.OpGranted: if m.Reason != "" { say("granted %s, WARNING: %s", m.Lease, m.Reason) } else { say("granted %s", m.Lease) } - return visibleDevices(o.Card, m, os.LookupEnv, say), true + return m, true } } }