Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ has a matching GitHub release whose body comes from its section below.
without anyone typing `needs_mib` or keeping it up to date. An engine not yet measured takes the whole card, and
the grant line in `berth events` names the size it used
([#311](https://github.com/T-Crypt/berth/issues/311)).
- `berth status` has a SCHEDULE table and the harbor a SCHEDULES panel. Each lists every engine that has a
`window`, with its hours, whether they are open, and how long until they close or open. An engine shows before its
window opens, and a `berth run --window` lease shows its own hours. The status JSON carries the same rows as
`schedules`, and each lease carries its `window`.

### Fixed

Expand Down
5 changes: 3 additions & 2 deletions docs/leasing.md
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,9 @@ maintenance or before a VM takes it. Name the card by its name or its PCI addres
[Run an engine whenever the card is free](gateway.md#run-an-engine-whenever-the-card-is-free).
- **A window puts either on the clock.** `window = "23:00-07:00"` on the entry, or `berth run --window 23:00-07:00`,
holds the lease to those hours on the node's wall clock. It may cross midnight. A lease that asks outside them
waits for them with no deadline (unless `--wait` sets one) and holds up nobody behind it. `berth status` says when
the window opens, and a background engine is not started at all. When the window closes its holder drains at its usual boundary, so a slot or a job boundary comes first and a
waits for them with no deadline (unless `--wait` sets one) and holds up nobody behind it. A background engine is
not started at all outside its hours. `berth status` (SCHEDULE) and the harbor (SCHEDULES) list every engine that
has a window, whether or not it runs, with its hours and when it next opens or closes. When the window closes its holder drains at its usual boundary, so a slot or a job boundary comes first and a
request in flight finishes.

```toml
Expand Down
1 change: 1 addition & 0 deletions internal/broker/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ func (b *Broker) Status() proto.Status {
GPUPIDs: pids, Used: l.used(),
GTTUsed: l.gttUsed(), GTTContested: l.contestedGTT,
Contested: l.contested, Age: now.Sub(since).Seconds(), Pinned: l.pinned,
Window: l.window.String(),
Training: l.training,
}
if l.state == parked {
Expand Down
16 changes: 16 additions & 0 deletions internal/broker/window_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,3 +197,19 @@ func TestWindowWaiterOnSeveralCardsOutlastsTheDefaultWait(t *testing.T) {
t.Fatal("nightly is not on both cards")
}
}

// The status names the hours a lease is held to, waiting or granted, and
// leaves a lease with no window without one.
func TestStatusCarriesTheLeaseWindow(t *testing.T) {
b, clk := startTurnBroker(t, procs.Map{})
clk.set(at2330)
must(t, acquire(b, Request{Engine: "nightly", Priority: Batch, Needs: 8 * GiB, Window: night}))
must(t, acquire(b, Request{Engine: "chat", Priority: Interactive, Needs: 4 * GiB}))
got := map[string]string{}
for _, l := range b.Status().Leases {
got[l.Engine] = l.Window
}
if got["nightly"] != "23:00-07:00" || got["chat"] != "" {
t.Errorf("lease windows = %v, want nightly 23:00-07:00 and chat none", got)
}
}
6 changes: 5 additions & 1 deletion internal/cli/daemon.go
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,11 @@ func runDaemon(c *Context) int {
go samplerLoop(ctx, rec, hr, n, watcher, ev, &lastHost, &lastAlerts)
x := server.Extras{Engines: watcher.Snapshot, Host: lastHost.Load, History: rec.Get,
Insights: func() *proto.Insights { return watcher.Insights(time.Now()) },
Requests: watcher.Requests, Alerts: lastAlerts.Load, Finished: watcher.Finished}
Requests: watcher.Requests, Alerts: lastAlerts.Load, Finished: watcher.Finished,
Windows: map[string]proto.Window{}}
for _, e := range cfg.Engine {
x.Windows[e.Name] = e.Window
}
if *devicesFile != "" {
x.Pair = &pair.Store{Path: *devicesFile}
}
Expand Down
10 changes: 10 additions & 0 deletions internal/cli/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,16 @@ func (v statusView) Human(c *Context, ok bool) error {
}
lease("LEASE", st.Leases, false)
lease("WAITING", st.Waiting, true)
if len(st.Schedules) > 0 {
fmt.Fprint(tw, "\nSCHEDULE\tHOURS\tNOW\n")
for _, s := range st.Schedules {
now := c.Style.Warn("closed") + ", opens in " + age(s.ChangeS)
if s.Open {
now = c.Style.OK("open") + ", closes in " + age(s.ChangeS)
}
fmt.Fprintf(tw, "%s\t%s\t%s\n", s.Engine, s.Window, esc(now))
}
}
if len(st.Squatters) > 0 {
fmt.Fprint(tw, "\n"+cardCol("SQUATTER", "CARD")+"\tPID\tHOLDS\tSEEN\tUNPROTECTED RUN\n")
for _, s := range st.Squatters {
Expand Down
23 changes: 23 additions & 0 deletions internal/cli/status_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -442,3 +442,26 @@ func TestClaimsTablesSayHoldsAndNameCards(t *testing.T) {
t.Errorf("one card claims table:\n%s", s)
}
}

// berth status lists every engine held to hours and where the node's clock
// sits in them, so a window shows before its engine runs.
func TestStatusShowsTheSchedules(t *testing.T) {
var out, errOut bytes.Buffer
c := &Context{Out: &out, Err: &errOut, Format: FormatHuman, Style: newStyle(&out), errStyle: newStyle(&errOut)}
st := &proto.Status{
API: proto.APIVersion, Version: "test",
Schedules: []proto.Schedule{
{Engine: "nightly", Window: "23:00-07:00", Open: false, ChangeS: 3 * 3600},
{Engine: "render", Window: "18:00-22:00", Open: true, ChangeS: 2*3600 + 15*60},
},
}
if err := (statusView{st: st}).Human(c, true); err != nil {
t.Fatal(err)
}
s := out.String()
for _, want := range []string{"SCHEDULE", "HOURS", "nightly", "23:00-07:00", "closed, opens in 3h", "render", "open, closes in 2h15m"} {
if !strings.Contains(s, want) {
t.Errorf("stdout lacks %q:\n%s", want, s)
}
}
}
4 changes: 4 additions & 0 deletions internal/harbor/web/demo.json
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,10 @@
"age_seconds": 40
}
],
"schedules": [
{"engine": "nightly-train", "window": "23:00-07:00", "open": false, "change_s": 10800},
{"engine": "render", "window": "18:00-22:00", "open": true, "change_s": 8100}
],
"waiting": [
{
"id": "LIT-0145",
Expand Down
4 changes: 4 additions & 0 deletions internal/harbor/web/harbor.css
Original file line number Diff line number Diff line change
Expand Up @@ -1036,6 +1036,10 @@ body.ready #panel-engines > :nth-child(n+2) { animation-delay: 60ms; }
.sq.sw-up { border-left-color: var(--green); }
.sq.sw-waiting, .sq.sw-starting { border-left-color: var(--amber); }
.sq.sw-failed { border-left-color: var(--magenta); }
/* A schedule row: green while its hours are open, dim while shut. */
.sq.sched { border-left-color: var(--dim-2); }
.sq.sched .name { color: var(--ink-strong); }
.sq.sched-open { border-left-color: var(--green); }
.sw-state { display: inline-flex; align-items: center; gap: var(--s1); white-space: nowrap; }
.sw-chip { border: 1px solid var(--hair-2); border-radius: var(--r-pill); padding: 0 var(--s2); font-size: var(--fs-micro); color: var(--dim); }
.sw-cordon, .sw-pinned { color: var(--amber); border-color: var(--amber-dim); }
Expand Down
23 changes: 21 additions & 2 deletions internal/harbor/web/harbor.js
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ let shortcutsDlg, railMenuDlg;
let lastLogSig = null; // filter+query+pause signature: repaint only on change, append otherwise
let clearLogFilter = null; // set by buildLogSearch: the page's Esc reaches it (R-07)
let lastTick = 0; // the relative-time ticker's own 1 s cadence
let berthsRoot, claimsRoot, claimsMeta, stowRoot, notickRoot, machineRoot, machineCharts, machineMeta, socketLine;
let berthsRoot, schedulesRoot, schedulesMeta, claimsRoot, claimsMeta, stowRoot, notickRoot, machineRoot, machineCharts, machineMeta, socketLine;
let kpiStripEl, tideEl, tideLegend, tideMeta;
let boardMeta, berthsMeta, stowMeta, noticketMeta, enginesMeta;
let throughputMeta, throughputDecode, throughputRpm;
Expand Down Expand Up @@ -168,6 +168,8 @@ function main() {
shortcutsDlg = $("shortcuts");
railMenuDlg = $("rail-menu");
berthsRoot = $("berths");
schedulesRoot = $("schedules");
schedulesMeta = $("schedules-meta");
claimsRoot = $("claims");
claimsMeta = $("claims-meta");
stowRoot = $("stowaways");
Expand Down Expand Up @@ -508,7 +510,7 @@ let pendingSection = null;
const HARBOR_SECTIONS = [
["sec-counter", "Counter"], ["sec-tide", "Tide"], ["sec-timeline", "Timeline"],
["sec-board", "Board"], ["sec-log", "Log"], ["sec-berths", "Berths"],
["sec-machine", "Machine"], ["sec-claims", "Claims"], ["sec-stow", "Stowaways"], ["sec-noticket", "No ticket"],
["sec-machine", "Machine"], ["sec-schedules", "Schedules"], ["sec-claims", "Claims"], ["sec-stow", "Stowaways"], ["sec-noticket", "No ticket"],
];
const ENGINES_SECTIONS = [
["sec-engines", "Engines"], ["sec-library", "Library"], ["priority-panel", "Ranking"], ["sec-pool", "KV pool"],
Expand Down Expand Up @@ -2097,6 +2099,22 @@ function claimLine(cl) {
function memoryWord(cl) {
return fmtGiB(cl.used || 0) + " GiB" + (cl.gtt_used ? " + " + fmtGiB(cl.gtt_used) + " GiB GTT" : "");
}
// renderSchedules lists the engines held to hours, by the node's clock, as
// berth status does: an engine shows here before its window opens.
function renderSchedules(s) {
const list = el("div", "squatter-list");
const all = s.schedules || [];
const open = all.filter((sc) => sc.open);
schedulesMeta.textContent = all.length ? open.length + " open of " + all.length : "none";
if (all.length === 0) list.append(el("div", "empty-note", "no engine is held to hours: set window = \"23:00-07:00\" on an [[engine]]"));
for (const sc of all) {
const row = el("div", "sq sched" + (sc.open ? " sched-open" : ""));
const now = sc.open ? "open, closes in " + fmtAge(sc.change_s) : "closed, opens in " + fmtAge(sc.change_s);
row.append(el("div", "top", el("span", "name", sc.engine), el("span", null, sc.window)), el("div", "meta", now));
list.append(row);
}
schedulesRoot.replaceChildren(list);
}
function renderClaims(s) {
const list = el("div", "squatter-list");
const all = s.claims || [];
Expand Down Expand Up @@ -2469,6 +2487,7 @@ function renderLight(s) {
paintTabCounts(entries.length, (s.engines || []).length);
renderBerths(s);
paintAlerts(s);
renderSchedules(s);
renderClaims(s);
renderStowaways(s);
renderNoTicket(s);
Expand Down
7 changes: 7 additions & 0 deletions internal/harbor/web/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,13 @@ <h2 class="panel-title">THE MACHINE</h2>
<div id="machine-charts" class="machine-charts adv-only"></div>
<div class="socket-line" id="socket-line" role="status"></div>
</section>
<section class="panel schedules-section" id="sec-schedules" aria-label="schedules">
<div class="panel-head">
<h2 class="panel-title">SCHEDULES</h2>
<span class="panel-meta" id="schedules-meta">engine hours</span>
</div>
<div id="schedules"></div>
</section>
<section class="panel claims-section" id="sec-claims" aria-label="claims">
<div class="panel-head">
<h2 class="panel-title">CLAIMS</h2>
Expand Down
7 changes: 7 additions & 0 deletions internal/proto/proto.go
Original file line number Diff line number Diff line change
Expand Up @@ -377,6 +377,10 @@ type Status struct {
Priority *PriorityStatus `json:"priority,omitempty"`
Claims []Claim `json:"claims,omitempty"`
Footprints []EngineFootprint `json:"footprints,omitempty"`
// Schedules are the engines held to hours, by engine name: each
// configured window, and a lease's own (berth run --window) in place of
// its engine's, whether or not anything runs now.
Schedules []Schedule `json:"schedules,omitempty"`
// ParkedBytes is the RAM the parked engines hold: each one's Berth
// scope's anon plus shmem, or its VRAM footprint where Berth reads no
// scope. A reading, never a reservation.
Expand Down Expand Up @@ -595,6 +599,9 @@ type Lease struct {
// expects the waiting engine to start, from the measured starts in
// its queue chain only; 0 = unknown (no book, or nothing measured).
EstimateS float64 `json:"estimate_start_s,omitempty"`
// Window is the hours the broker holds the lease to, "23:00-07:00";
// "" = none.
Window string `json:"window,omitempty"`
}

type Squatter struct {
Expand Down
31 changes: 31 additions & 0 deletions internal/proto/window.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,37 @@ func (w Window) Opens(t time.Time) time.Duration {
return time.Duration(left)*time.Minute - time.Duration(t.Second())*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond
}

// Closes is how long a lease inside w keeps its hours: 0 outside, and with
// no window, which never closes.
func (w Window) Closes(t time.Time) time.Duration {
if !w.Set() || !w.Contains(t) {
return 0
}
left := (w.End - minuteOf(t) + minutesPerDay) % minutesPerDay
return time.Duration(left)*time.Minute - time.Duration(t.Second())*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond
}

// Schedule is one engine's hours as the node's clock reads them now.
type Schedule struct {
Engine string `json:"engine"`
Window string `json:"window"`
Open bool `json:"open"`
// ChangeS is the seconds until the window closes when Open, else until
// it opens.
ChangeS float64 `json:"change_s"`
}

// ScheduleAt reads w for engine at t.
func ScheduleAt(engine string, w Window, t time.Time) Schedule {
s := Schedule{Engine: engine, Window: w.String(), Open: w.Contains(t)}
if s.Open {
s.ChangeS = w.Closes(t).Seconds()
} else {
s.ChangeS = w.Opens(t).Seconds()
}
return s
}

func minuteOf(t time.Time) int { return t.Hour()*60 + t.Minute() }

// String is the window as a config file writes it, "23:00-07:00". The zero
Expand Down
36 changes: 36 additions & 0 deletions internal/proto/window_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,39 @@ func TestWindowContainsAndOpens(t *testing.T) {
}
}
}

func TestWindowCloses(t *testing.T) {
at := func(h, m int) time.Time { return time.Date(2026, 11, 2, h, m, 0, 0, zone) }
night := Window{Start: 23 * 60, End: 7 * 60}
work := Window{Start: 9*60 + 30, End: 17 * 60}
for _, tc := range []struct {
name string
w Window
at time.Time
closes time.Duration
}{
{"night at its start", night, at(23, 0), 8 * time.Hour},
{"night past midnight", night, at(0, 30), 6*time.Hour + 30*time.Minute},
{"night half a minute before its end", night, at(6, 59).Add(30 * time.Second), 30 * time.Second},
{"night at its end", night, at(7, 0), 0},
{"work at noon", work, at(12, 0), 5 * time.Hour},
{"work the evening after", work, at(18, 0), 0},
{"no window", Window{}, at(12, 0), 0},
} {
if got := tc.w.Closes(tc.at); got != tc.closes {
t.Errorf("%s: Closes = %v, want %v", tc.name, got, tc.closes)
}
}
}

func TestScheduleAt(t *testing.T) {
night := Window{Start: 23 * 60, End: 7 * 60}
open := ScheduleAt("agent", night, time.Date(2026, 11, 2, 1, 0, 0, 0, zone))
if want := (Schedule{Engine: "agent", Window: "23:00-07:00", Open: true, ChangeS: 6 * 3600}); open != want {
t.Errorf("inside: %+v, want %+v", open, want)
}
shut := ScheduleAt("agent", night, time.Date(2026, 11, 2, 20, 0, 0, 0, zone))
if want := (Schedule{Engine: "agent", Window: "23:00-07:00", Open: false, ChangeS: 3 * 3600}); shut != want {
t.Errorf("outside: %+v, want %+v", shut, want)
}
}
33 changes: 33 additions & 0 deletions internal/server/e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -516,3 +516,36 @@ func TestComfyUIDrainWaitsForJobBoundary(t *testing.T) {
t.Fatalf("no handoff line:\n%s", out1.String())
}
}

// The socket's status carries the configured hours of an engine with no
// lease, so a client sees a schedule before its engine runs.
func TestSocketStatusCarriesTheSchedules(t *testing.T) {
sock := filepath.Join(t.TempDir(), "berth.sock")
log := slog.New(slog.NewTextHandler(io.Discard, nil))
b := broker.New(broker.Config{Poll: 20 * time.Millisecond, DefaultWait: 5 * time.Second, JuggleWindow: -1, Log: log}, emptyCard{}, procs.System{})
ln, err := server.Listen(sock)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
go b.Run(ctx)
windows := map[string]proto.Window{"nightly": {Start: 23 * 60, End: 7 * 60}, "chat": {}}
go server.Serve(ctx, ln, node.One(b), server.Extras{Windows: windows}, log)

conn, raw, err := server.Dial(sock)
if err != nil {
t.Fatal(err)
}
defer raw.Close()
if err := conn.Send(proto.Msg{Op: proto.OpStatus}); err != nil {
t.Fatal(err)
}
m, err := conn.Recv()
if err != nil || m.Status == nil {
t.Fatalf("status: %v %+v", err, m)
}
if s := m.Status.Schedules; len(s) != 1 || s[0].Engine != "nightly" || s[0].Window != "23:00-07:00" || s[0].ChangeS <= 0 {
t.Fatalf("schedules = %+v, want nightly 23:00-07:00 alone", s)
}
}
35 changes: 35 additions & 0 deletions internal/server/schedules.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Copyright 2026 Trevin Tindall (T-Crypt)
// SPDX-License-Identifier: Apache-2.0

package server

import (
"slices"
"strings"
"time"

"github.com/T-Crypt/berth/internal/proto"
)

// schedules is every engine held to hours at now, one row per engine by
// name: the configured windows, with a lease's own window in place of its
// engine's, since that is the one the broker enforces.
func schedules(configured map[string]proto.Window, st *proto.Status, now time.Time) []proto.Schedule {
hours := make(map[string]proto.Window, len(configured))
for name, w := range configured {
if w.Set() {
hours[name] = w
}
}
for _, l := range slices.Concat(st.Leases, st.Waiting) {
if w, err := proto.ParseWindow(l.Window); err == nil && w.Set() {
hours[l.Engine] = w
}
}
out := make([]proto.Schedule, 0, len(hours))
for name, w := range hours {
out = append(out, proto.ScheduleAt(name, w, now))
}
slices.SortFunc(out, func(a, b proto.Schedule) int { return strings.Compare(a.Engine, b.Engine) })
return out
}
Loading
Loading