diff --git a/CHANGELOG.md b/CHANGELOG.md index f813646..c7f2094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ has a matching GitHub release whose body comes from its section below. instead of waiting out their timeout. A second `berth train` waits in line and starts when the first one ends. Other cards keep serving. `berth train` refuses to start without berthd, and `berth status` shows the lease as `hold,pinned,training`. +- `berth train --exclusive` also locks the card in the driver on Linux with an NVIDIA card. It puts the granted cards + in exclusive-process compute mode for the run's life, so a program that opens CUDA without Berth cannot open + them, and puts them back to the default mode when the run ends. It needs a sudo rule for `nvidia-smi -c`, and + refuses to start without one, or when another compute process is already on the card. - A `berth train` run keeps its lock while berthd is away. With berthd down, `berth run` refuses to start an engine while a training runs on the node, instead of starting it without a lease. A berthd that starts while a training runs refuses every other request until the training re-attaches, with no time limit, where other engines get diff --git a/docs/leasing.md b/docs/leasing.md index 44a3e7b..a9e000e 100644 --- a/docs/leasing.md +++ b/docs/leasing.md @@ -273,6 +273,34 @@ berth train --as train-mymodel --gpu gpu0,gpu1,gpu2 -- torchrun --nproc_per_node back as a training. `berth run --pinned --priority foreground` still works for a run that must not be drained but may share its card. + +### Lock the card in the driver too + +The training lock covers everything that asks Berth for the card. A program that opens CUDA without Berth only +shows up as a stowaway. `berth train --exclusive` closes that gap on Linux with an NVIDIA card. After the grant, it +puts the cards in the driver's exclusive-process compute mode, so the training's process is the only one that can +open a CUDA context there. When the run ends, it puts them back to the default mode. + +```sh +berth train --exclusive --as train-mymodel -- python train.py +``` + +- Setting the mode needs root. Give your user a sudo rule for `nvidia-smi -c` alone, with `sudo visudo -f + /etc/sudoers.d/berth-exclusive`: + + ``` + YOUR_USER ALL=(root) NOPASSWD: /usr/bin/nvidia-smi -i * -c EXCLUSIVE_PROCESS, /usr/bin/nvidia-smi -i * -c DEFAULT + ``` + + Without the rule, `berth train --exclusive` refuses to start instead of running without the guard. +- The run refuses to start when another compute process is already on the card, because that process would own + the card's one context and the training could not open it. +- Exclusive mode allows one process per card. A training that opens CUDA from several processes on one card + (DataLoader workers that use CUDA, two ranks on one card) fails to start in it. One rank per card, as torchrun + gives each card, works. +- Graphics are not compute: the desktop and games keep drawing on the card. +- If the `berth train` process is killed with SIGKILL, the mode stays set. `sudo nvidia-smi -c DEFAULT` puts it + back. A pinned run does not lock the card: other requests wait for it until their own deadline. A pin does not stop memory taken outside Berth. A game started on a leased card waits behind the run in Berth but diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 6c3da9f..a4f5def 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -447,7 +447,11 @@ waits in line and starts when the first one ends. Other cards keep serving. A game or a hold that already has the card is not drained: the training waits for it. berth train refuses to start when berthd is not running, because the -lock lives in berthd.`, +lock lives in berthd. + +--exclusive (Linux, NVIDIA) also puts the cards in the driver's exclusive-process +compute mode for the run's life, so a program that opens CUDA without Berth +cannot open them. It needs a sudo rule for nvidia-smi -c (see docs/leasing.md).`, Run: runTrain, }, { diff --git a/internal/cli/run.go b/internal/cli/run.go index 309ac85..a1d29b8 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -62,6 +62,7 @@ func runEngine(c *Context, training bool) int { sliceName := fs.String("slice", "", "NAME.slice to start the engine's scope under (systemd user manager, Linux): the slice unit's CPU, memory and swap policy applies to the engine and everything it forks; over the entry's slice") grace := fs.Duration("grace", 30*time.Second, "time each drain step gets before the next") failClosed := fs.Bool("fail-closed", false, "do not start CMD without berthd or stop it without a safe boundary") + exclusiveMode := fs.Bool("exclusive", false, "berth train only: put the cards in exclusive-process compute mode for the run's life, so no program outside Berth can open them (Linux, NVIDIA, needs a sudo rule for nvidia-smi -c)") cfgPath := fs.String("config", config.DefaultPath(), "berth.toml: the engine entry matching --as (or --group) supplies the settings a flag does not set; a set flag wins and says so; missing = no defaults ($BERTH_CONFIG)") if code, done := c.Parse(fs); done { return code @@ -71,6 +72,9 @@ func runEngine(c *Context, training bool) int { if *engine == "" { return c.fail(required) } + if *exclusiveMode && !training { + return c.fail(Systemf("--exclusive is for berth train: exclusive mode allows one process on the card, which only a training should claim")) + } if training { for _, name := range trainSets { if c.Changed(name) { @@ -183,7 +187,7 @@ func runEngine(c *Context, training bool) int { // Nothing here goes to stdout: the engine owns it. berth's own notices are // diagnostics on stderr, and --quiet never hides the unprotected warning. return runner.Run(runner.OptionsFor(&e, runner.Over{ - Socket: *sock, Front: *frontAddr, Group: *group, Pinned: *pinned, Training: training, Switched: *switched, Wait: *wait, Grace: *grace, + Socket: *sock, Front: *frontAddr, Group: *group, Pinned: *pinned, Training: training, Exclusive: *exclusiveMode, Switched: *switched, Wait: *wait, Grace: *grace, FailClosed: *failClosed, Argv: argv, Stderr: c.Err, Log: c.Log(), })) } diff --git a/internal/cli/train_unix_test.go b/internal/cli/train_unix_test.go index 6b3e61d..7b8175e 100644 --- a/internal/cli/train_unix_test.go +++ b/internal/cli/train_unix_test.go @@ -49,3 +49,27 @@ func TestTrainLocksTheCardAgainstBerthRun(t *testing.T) { t.Errorf("berth train exit = %d, want 0", code) } } + +// --exclusive is part of the run: on a card the driver names no UUID for, +// berth train --exclusive is refused after the grant and its command never +// starts, so it never runs without the guard it asked for. +func TestTrainExclusiveRefusesWithoutAnNVIDIACard(t *testing.T) { + isolate(t) + sock := startDaemon(t) + code, out, errOut := runCLI(t, "train", "--exclusive", "--as", "train-a", "--socket", sock, "--", "echo", "started") + if code == ExitOK || strings.Contains(out, "started") { + t.Fatalf("exit %d, stdout %q, stderr %q; want a refusal and no start", code, out, errOut) + } + if !strings.Contains(errOut, "--exclusive") { + t.Errorf("stderr %q does not name --exclusive", errOut) + } +} + +// --exclusive belongs to berth train alone. +func TestRunRefusesExclusive(t *testing.T) { + isolate(t) + code, _, errOut := runCLI(t, "run", "--exclusive", "--as", "x", "--", "true") + if code != ExitSystem || !strings.Contains(errOut, "--exclusive is for berth train") { + t.Fatalf("exit %d, stderr %q", code, errOut) + } +} diff --git a/internal/runner/exclusive.go b/internal/runner/exclusive.go new file mode 100644 index 0000000..85b8f99 --- /dev/null +++ b/internal/runner/exclusive.go @@ -0,0 +1,50 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package runner + +import ( + "errors" + "fmt" + "runtime" + "strings" + "time" +) + +// exclusiveTimeout bounds each nvidia-smi call the exclusive mode makes. +const exclusiveTimeout = 30 * time.Second + +// exclusive puts the granted cards (their UUIDs, comma separated) in the +// driver's exclusive-process compute mode, so the run's process is the only +// one that can open a CUDA context there, and returns the func that puts +// them back to the default mode. It refuses when another compute process is +// already on the cards: in exclusive mode that process would lock the run +// itself out. Setting the mode needs root, through `sudo -n`, so a missing +// sudo rule refuses the run instead of starting it without the guard. +func exclusive(o Options, uuids string) (func(func(string, ...any)), error) { + if runtime.GOOS != "linux" { + return nil, errors.New("exclusive compute mode is Linux only") + } + if uuids == "" { + return nil, errors.New("the granted card has no NVIDIA UUID") + } + out, err := o.CLI([]string{"nvidia-smi", "-i", uuids, "--query-compute-apps=pid,process_name", "--format=csv,noheader"}, exclusiveTimeout) + if err != nil { + return nil, fmt.Errorf("nvidia-smi could not list the compute processes: %v", err) + } + if apps := strings.TrimSpace(out); apps != "" { + return nil, fmt.Errorf("another compute process is on the card (%s); exclusive mode would lock the training out", strings.ReplaceAll(apps, "\n", "; ")) + } + set := func(mode string) error { + _, err := o.CLI([]string{"sudo", "-n", "nvidia-smi", "-i", uuids, "-c", mode}, exclusiveTimeout) + return err + } + if err := set("EXCLUSIVE_PROCESS"); err != nil { + return nil, fmt.Errorf("could not set exclusive-process mode (it needs a sudo rule that runs nvidia-smi -c without a password): %v", err) + } + return func(say func(string, ...any)) { + if err := set("DEFAULT"); err != nil { + say("could not put %s back to the default compute mode: %v; run: sudo nvidia-smi -i %s -c DEFAULT", uuids, err, uuids) + } + }, nil +} diff --git a/internal/runner/exclusive_linux_test.go b/internal/runner/exclusive_linux_test.go new file mode 100644 index 0000000..818420a --- /dev/null +++ b/internal/runner/exclusive_linux_test.go @@ -0,0 +1,84 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package runner + +import ( + "errors" + "slices" + "strings" + "testing" + "time" +) + +// fakeSMI answers the exclusive mode's nvidia-smi calls: apps is what the +// compute-app query prints, sudoErr fails the mode change. It records every +// argv. +type fakeSMI struct { + apps string + sudoErr error + calls [][]string +} + +func (f *fakeSMI) cli(argv []string, _ time.Duration) (string, error) { + f.calls = append(f.calls, argv) + if argv[0] == "sudo" { + return "", f.sudoErr + } + return f.apps, nil +} + +// On an empty card the mode goes to exclusive-process before the run, and +// back to the default when it ends, on the granted cards only. +func TestExclusiveSetsAndRestoresTheMode(t *testing.T) { + f := &fakeSMI{} + restore, err := exclusive(Options{CLI: f.cli}, "GPU-a,GPU-b") + if err != nil { + t.Fatal(err) + } + set := []string{"sudo", "-n", "nvidia-smi", "-i", "GPU-a,GPU-b", "-c", "EXCLUSIVE_PROCESS"} + if len(f.calls) != 2 || !slices.Equal(f.calls[1], set) { + t.Fatalf("calls = %q, want the query then %q", f.calls, set) + } + restore(func(string, ...any) {}) + back := []string{"sudo", "-n", "nvidia-smi", "-i", "GPU-a,GPU-b", "-c", "DEFAULT"} + if len(f.calls) != 3 || !slices.Equal(f.calls[2], back) { + t.Fatalf("calls = %q, want %q last", f.calls, back) + } +} + +// A compute process already on the card would own its one context and lock +// the training out, so the run is refused and the mode is left alone. +func TestExclusiveRefusesWhenAProcessIsOnTheCard(t *testing.T) { + f := &fakeSMI{apps: "4242, python3\n"} + _, err := exclusive(Options{CLI: f.cli}, "GPU-a") + if err == nil || !strings.Contains(err.Error(), "4242, python3") { + t.Fatalf("err = %v, want a refusal naming the process", err) + } + for _, c := range f.calls { + if c[0] == "sudo" { + t.Fatalf("the mode was changed beside a running process: %q", c) + } + } +} + +// Without a sudo rule the run is refused instead of starting without the +// guard the operator asked for. +func TestExclusiveRefusesWithoutSudo(t *testing.T) { + f := &fakeSMI{sudoErr: errors.New("a password is required")} + _, err := exclusive(Options{CLI: f.cli}, "GPU-a") + if err == nil || !strings.Contains(err.Error(), "sudo rule") { + t.Fatalf("err = %v, want it to name the sudo rule", err) + } +} + +// A card with no NVIDIA UUID (an AMD card) has no exclusive mode to set. +func TestExclusiveNeedsAnNVIDIACard(t *testing.T) { + f := &fakeSMI{} + if _, err := exclusive(Options{CLI: f.cli}, ""); err == nil { + t.Fatal("exclusive mode set on a card with no UUID") + } + if len(f.calls) != 0 { + t.Errorf("calls = %q, want none", f.calls) + } +} diff --git a/internal/runner/options.go b/internal/runner/options.go index dae3bbb..23c5257 100644 --- a/internal/runner/options.go +++ b/internal/runner/options.go @@ -25,6 +25,7 @@ type Over struct { Group string Pinned bool Training bool + Exclusive bool Switched bool Wait time.Duration Grace time.Duration @@ -62,7 +63,7 @@ func OptionsFor(e *config.Engine, o Over) Options { } opts := Options{ Socket: o.Socket, Engine: e.Name, Card: e.GPU, Group: o.Group, Priority: prio, - Needs: e.NeedsMiB.Bytes(), Pinned: o.Pinned, Training: o.Training, Switched: o.Switched, Wait: o.Wait, + Needs: e.NeedsMiB.Bytes(), Pinned: o.Pinned, Training: o.Training, Exclusive: o.Exclusive, Switched: o.Switched, Wait: o.Wait, DrainPost: e.DrainPost, DrainSlots: e.DrainSlots, DrainComfyUI: e.DrainComfyUI, DrainMax: e.DrainMax.Duration, MinTurn: e.MinTurn.Duration, ParkTTL: e.ParkTTL.Duration, Window: e.Window, Container: e.Container, Compose: e.Compose, APIKeyEnv: e.APIKeyEnv, APIKeyFile: e.APIKeyFile, diff --git a/internal/runner/runner.go b/internal/runner/runner.go index d85ee94..637349e 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -56,6 +56,10 @@ type Options struct { LearnSize bool Pinned bool // never drained: an app that must not be stopped (Blender, a capture) Training bool // berth train: a pinned hold that locks its cards; berthd refuses every other request there + // Exclusive: berth train --exclusive puts the granted cards in the + // driver's exclusive-process compute mode for the run's life, so no + // program outside Berth can open them either. + Exclusive bool Switched bool // a person's switch (the wake page's Launch, /v1/berth/switch): wins a juggle // MinTurn is a minimum turn: while the lease is younger than it after // the grant, no request of equal or lower priority drains it, and @@ -241,13 +245,25 @@ func Run(o Options) int { conn, raw = nil, nil } + if o.CLI == nil { + o.CLI = execCLI + } var cardEnv string if conn != nil { defer raw.Close() - var ok bool - if cardEnv, ok = acquire(o, conn, say); !ok { + granted, ok := acquire(o, conn, say) + if !ok { return ExitRefused } + cardEnv = visibleDevices(o.Card, granted, os.LookupEnv, say) + if o.Exclusive { + restore, err := exclusive(o, granted.CardUUID) + if err != nil { + say("--exclusive: %v; not starting %s", err, o.Engine) + return ExitRefused + } + defer restore(say) + } } cmd := command(o, o.Output) @@ -269,9 +285,6 @@ func Run(o Options) int { done := make(chan struct{}) - if o.CLI == nil { - o.CLI = execCLI - } var cid containerRef o.container = &cid @@ -424,30 +437,30 @@ func Run(o Options) int { // acquire asks berthd for the card and waits for the grant. It returns the // device entry the engine starts with, and false when there is no grant. -func acquire(o Options, conn *proto.Conn, say func(string, ...any)) (string, bool) { +func acquire(o Options, conn *proto.Conn, say func(string, ...any)) (proto.Msg, bool) { if err := conn.Send(proto.Msg{Op: proto.OpAcquire, Card: o.Card, Engine: o.Engine, Group: o.Group, Priority: o.Priority, Needs: largest(o.Needs), NeedsEach: each(o.Needs), LearnSize: o.LearnSize, Wait: o.Wait.Seconds(), Pinned: o.Pinned, Training: o.Training, Switched: o.Switched, Lend: o.Lend != nil, WholeMachine: o.WholeMachine, MinTurn: o.MinTurn.Seconds(), ParkTTL: o.ParkTTL.Seconds(), Window: o.Window.String()}); err != nil { say("acquire: %v", err) - return "", false + return proto.Msg{}, false } for { m, err := conn.Recv() if err != nil { say("berthd went away while %s was waiting: %v", o.Engine, err) - return "", false + return proto.Msg{}, false } switch m.Op { case proto.OpWaiting: say("waiting: %s", m.Reason) case proto.OpRefused: say("refused: %s", m.Reason) - return "", false + return proto.Msg{}, false case proto.OpGranted: if m.Reason != "" { say("granted %s, WARNING: %s", m.Lease, m.Reason) } else { say("granted %s", m.Lease) } - return visibleDevices(o.Card, m, os.LookupEnv, say), true + return m, true } } }