Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,16 @@ has a matching GitHub release whose body comes from its section below.

## [Unreleased]

### Fixed

- A minimum turn (`--min-turn`) now protects only a lease whose engine has served a request. An engine started by
a health probe or a warm-up, read up and idle with nothing served, no longer keeps the card for its full turn
while a real request waits.
- An engine that reports no in-flight figure (no processing gauge, no `/slots`, no queue) now reads as unknown
instead of idle. Before, an NInfer build without `/slots`, or Ollama, read idle while it served, and a juggle
handed its card over as if no request were running. `berth status --format json` shows the new
`in_flight_reported` field per engine.

## [v0.6.0] - 2026-11-14

v0.6.0 gives each card a schedule and makes Berth explain its choices. `berth schedule` books an engine's hours,
Expand Down
33 changes: 29 additions & 4 deletions internal/broker/broker.go
Original file line number Diff line number Diff line change
Expand Up @@ -291,9 +291,14 @@ type lease struct {
lockCh chan string
switched bool // a person's switch (Request.Switched)
minTurn time.Duration
parkTTL time.Duration // 0 = the node's
window proto.Window // zero = no window
prio Priority
// unused: since the grant the engine was read up and idle, and never
// busy. A probe started it, nobody asked it anything, so it has no
// minimum turn to keep. served: it was read busy once.
unused bool
served bool
parkTTL time.Duration // 0 = the node's
window proto.Window // zero = no window
prio Priority
// basePrio is the class the lease asked for, or the operator set; prio
// differs from it while a booking holds (booking.go).
basePrio Priority
Expand Down Expand Up @@ -425,7 +430,26 @@ func (l *lease) awaitingGrant() bool {
// than the turn, by the broker's clock (a duration from the grant, not
// an order among leases).
func (l *lease) inTurn(now time.Time) bool {
return l.minTurn > 0 && now.Sub(l.granted) < l.minTurn
return l.minTurn > 0 && !l.unused && now.Sub(l.granted) < l.minTurn
}

// noteServed reads whether each active lease's engine serves, for the
// minimum turn: a lease seen busy once has served; one seen only idle is
// unused. With no reading (no watcher, an engine still loading) neither
// is set, and the turn holds as it always did.
func (b *Broker) noteServed() {
if b.cfg.Busy == nil {
return
}
for _, l := range b.leases {
if l.state != active || l.minTurn == 0 || l.served {
continue
}
if busy, known := b.cfg.Busy(l.key()); known {
l.served = busy
l.unused = !busy
}
}
}

// turnWhy is the line a waiter blocked by l's minimum turn is told.
Expand Down Expand Up @@ -981,6 +1005,7 @@ func (b *Broker) Tick(ctx context.Context) {
b.measureRAM(l)
}
b.claimed = claimed
b.noteServed()
b.closeWindows(now)
b.closeForDowntime(now)
b.evictParked(now)
Expand Down
52 changes: 52 additions & 0 deletions internal/broker/minturn_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,3 +239,55 @@ func TestReattachKeepsTheTurnItWasIn(t *testing.T) {
t.Errorf("reason = %q, want %q", got, want)
}
}

// turnBusy is the engine reading a minimum-turn test sets: the watcher's
// answer for the llama-swap group, known and idle until the test says busy.
type turnBusy struct{ busy atomic.Bool }

func (r *turnBusy) read(key string) (bool, bool) { return r.busy.Load(), key == "llama-swap" }

func startBusyTurnBroker(t *testing.T, r *turnBusy) (*Broker, *minClock) {
t.Helper()
b, _ := jugglers(t, Config{JuggleWindow: -1, Busy: r.read})
clk := newMinClock(turnBase)
b.nowFn = clk.now
return b, clk
}

// A lease whose engine was started by a probe and never served a request
// (seen up and idle since the grant) has no minimum turn to keep: a waiter
// takes the card at once instead of waiting out five minutes nobody uses.
func TestMinTurnNeedsARequestServed(t *testing.T) {
r := &turnBusy{}
b, clk := startBusyTurnBroker(t, r)
holder := must(t, acquire(b, Request{Engine: "NInfer-Qwen3.8-27B", Group: "llama-swap", Priority: Foreground, MinTurn: 5 * time.Minute}))
b.Tick(context.Background())
clk.set(turnBase.Add(30 * time.Second))

w := acquireW(b, Request{Engine: "strata", Priority: Foreground})
pending(t, w.ch)
if why := drained(t, holder); !strings.Contains(why, "strata") {
t.Fatalf("drain reason = %q, want the waiter named", why)
}
}

// Once the engine has served a request its turn holds, through the idle
// gaps between requests.
func TestMinTurnHoldsAfterARequestServed(t *testing.T) {
r := &turnBusy{}
b, clk := startBusyTurnBroker(t, r)
holder := must(t, acquire(b, Request{Engine: "NInfer-Qwen3.8-27B", Group: "llama-swap", Priority: Foreground, MinTurn: 5 * time.Minute}))
b.Tick(context.Background())
r.busy.Store(true)
b.Tick(context.Background())
r.busy.Store(false)
b.Tick(context.Background())
clk.set(turnBase.Add(30 * time.Second))

w := acquireW(b, Request{Engine: "strata", Priority: Foreground})
pending(t, w.ch)
notDrained(t, holder)
if got, want := firstReason(t, w), "next in line: NInfer-Qwen3.8-27B has its minimum turn, 4m30s left"; got != want {
t.Errorf("reason = %q, want %q", got, want)
}
}
2 changes: 1 addition & 1 deletion internal/broker/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func (b *Broker) Status() proto.Status {
if l.state == parked {
pl.ParkedSince = parkedAge(l, now)
}
if l.minTurn > 0 {
if l.minTurn > 0 && !l.unused {
if rem := l.minTurn - now.Sub(l.granted); rem > 0 {
pl.TurnS = rem.Seconds()
}
Expand Down
2 changes: 1 addition & 1 deletion internal/engines/busy.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func (w *Watcher) Busy(name string) (busy, known bool) {
return true, true // starting or stopping
}
}
return false, true
return false, e.Metrics.InFlightReported
}
return false, false
}
49 changes: 48 additions & 1 deletion internal/engines/busy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
package engines

import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"

Expand All @@ -18,7 +22,7 @@ func TestBusy(t *testing.T) {
"llama-swap": {Name: "llama-swap", Up: true, At: now,
Models: []proto.Model{{Name: "NInfer-Qwen3.8-27B", State: "ready"}},
Slots: []proto.Slot{{Model: "NInfer-Qwen3.8-27B", Processing: true}}},
"comfyui": {Name: "comfyui", Up: true, At: now},
"comfyui": {Name: "comfyui", Up: true, At: now, Metrics: proto.EngineMetrics{InFlightReported: true}},
"strata": {Name: "strata", Up: false, At: now},
"old": {Name: "old", Up: true, At: now.Add(-time.Hour)},
}
Expand Down Expand Up @@ -50,3 +54,46 @@ func TestBusy(t *testing.T) {
t.Error("a nil watcher claimed to know")
}
}

// An engine whose reading carries no request-in-flight figure cannot be
// called idle: the NInfer build on aphotic answers /metrics with only its
// own ninfer_* counters and serves no /slots, so a request it is serving
// reads exactly like no request. Busy says unknown for it, and idle again
// once a reading names the in-flight count.
func TestBusyIsUnknownWithoutAnInFlightFigure(t *testing.T) {
ninfer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/metrics":
fmt.Fprintln(w, `ninfer_requests_total{outcome="ok"} 12`)
default:
http.NotFound(w, r)
}
}))
defer ninfer.Close()
gauged := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/metrics":
fmt.Fprintln(w, "llamacpp:requests_processing 0")
default:
http.NotFound(w, r)
}
}))
defer gauged.Close()

w := &Watcher{Engines: []config.Engine{
{Name: "old-ninfer", Type: "ninfer", URL: ninfer.URL},
{Name: "gauged", Type: "ninfer", URL: gauged.URL},
}}
w.PollOnce(context.Background())
for _, c := range []struct {
name string
busy, known bool
}{
{"old-ninfer", false, false},
{"gauged", false, true},
} {
if b, k := w.Busy(c.name); b != c.busy || k != c.known {
t.Errorf("Busy(%q) = %v,%v want %v,%v", c.name, b, k, c.busy, c.known)
}
}
}
25 changes: 25 additions & 0 deletions internal/engines/engines.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ func (w *Watcher) read(ctx context.Context, e config.Engine) proto.Engine {
}
if ss, serr := w.slots(ctx, base+"/slots", "", ""); serr == nil {
out.Slots = ss
out.Metrics.InFlightReported = true
}
}
case "custom":
Expand Down Expand Up @@ -503,6 +504,7 @@ func (w *Watcher) custom(ctx context.Context, base string, e config.Engine, out
PromptTokens: f("prompt_tokens"), OutputTokens: f("output_tokens"),
PromptSeconds: f("prompt_seconds"), OutputSeconds: f("output_seconds"),
Processing: f("processing"), Waiting: f("waiting"),
InFlightReported: inFlight(vals, e.Map),
}
out.MetricsReported = true
}
Expand Down Expand Up @@ -602,6 +604,7 @@ func (w *Watcher) metrics(ctx context.Context, url string, names map[string]stri
m.PromptSeconds, m.OutputSeconds = f("prompt_seconds"), f("output_seconds")
m.Processing, m.Waiting = f("processing"), f("waiting")
m.DraftTokens, m.DraftAccepted = vals["ninfer:draft_tokens_total"], vals["ninfer:draft_accepted_tokens_total"]
m.InFlightReported = inFlight(vals, names)
return m, nil
}

Expand All @@ -623,9 +626,13 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine)
if err := json.Unmarshal(b, &r); err != nil {
return fmt.Errorf("/running: %w", err)
}
// The engine reports in-flight work only when every model on it does:
// one model that cannot say hides its requests from the sum.
reported := true
for _, m := range r.Running {
out.Models = append(out.Models, proto.Model{Name: m.Model, State: m.State, File: models.ModelArg(m.Cmd)})
if m.State != "ready" || m.Proxy == "" {
reported = reported && m.State != "ready"
continue
}
px := strings.TrimRight(m.Proxy, "/")
Expand All @@ -635,6 +642,7 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine)
out.Metrics = out.Metrics.Add(mm)
out.MetricsReported = true
}
modelReported := err == nil && mm.InFlightReported
// Best-effort per model: the proxy's context window and its live KV
// slots, tagged with the model they sit on. A 404/501 on either
// leaves the engine up with what it has.
Expand All @@ -648,8 +656,11 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine)
}
if ss, serr := w.slots(ctx, px+"/slots", m.Model, ""); serr == nil {
out.Slots = append(out.Slots, ss...)
modelReported = true
}
reported = reported && modelReported
}
out.Metrics.InFlightReported = reported
for _, md := range out.Models {
if md.State == "ready" && md.Window > out.Window {
out.Window = md.Window
Expand Down Expand Up @@ -814,13 +825,27 @@ func (w *Watcher) strata(ctx context.Context, base, keyEnv string, out *proto.En
out.Metrics.Processing = 1
}
out.Metrics.Waiting = float64(m.Live.Queued)
out.Metrics.InFlightReported = m.Live.Running != nil || m.Live.State != ""
out.Metrics.PromptTokens = m.Totals.PromptTokens
out.Metrics.OutputTokens = m.Totals.OutputTokens
out.Metrics.PromptSeconds = m.Totals.PromptMS / 1000
out.Metrics.OutputSeconds = m.Totals.DecodeMS / 1000
return nil
}

// inFlight is whether a Prometheus reading carries the engine's in-flight
// gauges, so that a zero in them means idle rather than not reported.
func inFlight(vals map[string]float64, names map[string]string) bool {
for _, field := range []string{"processing", "waiting"} {
if name, ok := names[field]; ok {
if _, ok := vals[name]; ok {
return true
}
}
}
return false
}

// ParseProm reads Prometheus text exposition into name -> value, summing
// series that differ only by labels.
func ParseProm(text string) map[string]float64 {
Expand Down
9 changes: 7 additions & 2 deletions internal/engines/more.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ func (w *Watcher) promMetrics(ctx context.Context, url, keyEnv, prefix string, n
PromptTokens: f("prompt_tokens"), OutputTokens: f("output_tokens"),
PromptSeconds: f("prompt_seconds"), OutputSeconds: f("output_seconds"),
Processing: f("processing"), Waiting: f("waiting"),
InFlightReported: inFlight(vals, names),
}, true
}

Expand Down Expand Up @@ -257,8 +258,11 @@ func (w *Watcher) freetoken(ctx context.Context, base, keyEnv string, out *proto
out.Metrics.PromptTokens = pick("tokens_prompt", "prompt_tokens", "input_tokens")
out.Metrics.Waiting = pick("queue", "requests_waiting", "pending_requests")
for _, k := range []string{"generating", "busy", "active"} {
if v, ok := body[k].(bool); ok && v {
out.Metrics.Processing = 1
if v, ok := body[k].(bool); ok {
out.Metrics.InFlightReported = true
if v {
out.Metrics.Processing = 1
}
}
}
return nil
Expand All @@ -273,6 +277,7 @@ func (w *Watcher) comfyui(ctx context.Context, base, keyEnv string, out *proto.E
return err
}
out.Metrics.Processing, out.Metrics.Waiting = float64(running), float64(pending)
out.Metrics.InFlightReported = true
if b, err := w.getKey(ctx, base+"/system_stats", keyEnv); err == nil {
var s struct {
System struct {
Expand Down
4 changes: 4 additions & 0 deletions internal/proto/proto.go
Original file line number Diff line number Diff line change
Expand Up @@ -756,6 +756,10 @@ type EngineMetrics struct {
Waiting float64 `json:"waiting"`
DraftTokens float64 `json:"draft_tokens,omitempty"`
DraftAccepted float64 `json:"draft_accepted,omitempty"`
// InFlightReported is true when the reading carries a request-in-flight
// figure (a processing or waiting gauge, a job queue). Without it a zero
// Processing is unknown, not idle.
InFlightReported bool `json:"in_flight_reported,omitempty"`
}

func (m EngineMetrics) Add(o EngineMetrics) EngineMetrics {
Expand Down
Loading