From dad18a4dc2cfed4207258dc17d13232058b52f24 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:07:15 -0700 Subject: [PATCH 1/3] fix(engines): read an engine with no in-flight figure as unknown, not idle Busy said "known idle" for any engine that answered, even when its reading had no way to show a running request. The NInfer build on aphotic answers /metrics with only ninfer_* counters and serves no /slots, so it read idle while it served, and so did Ollama. A juggle hands an idle holder's card over at once, and the minimum-turn change that follows needs a truthful idle. Each reader now sets EngineMetrics.InFlightReported when its reading carries a processing or waiting gauge, a /slots list or a job queue. llama-swap reports only when every ready model on it does. --- internal/engines/busy.go | 2 +- internal/engines/busy_test.go | 49 ++++++++++++++++++++++++++++++++++- internal/engines/engines.go | 25 ++++++++++++++++++ internal/engines/more.go | 9 +++++-- internal/proto/proto.go | 4 +++ 5 files changed, 85 insertions(+), 4 deletions(-) diff --git a/internal/engines/busy.go b/internal/engines/busy.go index 506e709d..bd6a15b0 100644 --- a/internal/engines/busy.go +++ b/internal/engines/busy.go @@ -52,7 +52,7 @@ func (w *Watcher) Busy(name string) (busy, known bool) { return true, true // starting or stopping } } - return false, true + return false, e.Metrics.InFlightReported } return false, false } diff --git a/internal/engines/busy_test.go b/internal/engines/busy_test.go index cfb2d3fb..24bbf723 100644 --- a/internal/engines/busy_test.go +++ b/internal/engines/busy_test.go @@ -4,6 +4,10 @@ package engines import ( + "context" + "fmt" + "net/http" + "net/http/httptest" "testing" "time" @@ -18,7 +22,7 @@ func TestBusy(t *testing.T) { "llama-swap": {Name: "llama-swap", Up: true, At: now, Models: []proto.Model{{Name: "NInfer-Qwen3.8-27B", State: "ready"}}, Slots: []proto.Slot{{Model: "NInfer-Qwen3.8-27B", Processing: true}}}, - "comfyui": {Name: "comfyui", Up: true, At: now}, + "comfyui": {Name: "comfyui", Up: true, At: now, Metrics: proto.EngineMetrics{InFlightReported: true}}, "strata": {Name: "strata", Up: false, At: now}, "old": {Name: "old", Up: true, At: now.Add(-time.Hour)}, } @@ -50,3 +54,46 @@ func TestBusy(t *testing.T) { t.Error("a nil watcher claimed to know") } } + +// An engine whose reading carries no request-in-flight figure cannot be +// called idle: the NInfer build on aphotic answers /metrics with only its +// own ninfer_* counters and serves no /slots, so a request it is serving +// reads exactly like no request. Busy says unknown for it, and idle again +// once a reading names the in-flight count. +func TestBusyIsUnknownWithoutAnInFlightFigure(t *testing.T) { + ninfer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/metrics": + fmt.Fprintln(w, `ninfer_requests_total{outcome="ok"} 12`) + default: + http.NotFound(w, r) + } + })) + defer ninfer.Close() + gauged := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/metrics": + fmt.Fprintln(w, "llamacpp:requests_processing 0") + default: + http.NotFound(w, r) + } + })) + defer gauged.Close() + + w := &Watcher{Engines: []config.Engine{ + {Name: "old-ninfer", Type: "ninfer", URL: ninfer.URL}, + {Name: "gauged", Type: "ninfer", URL: gauged.URL}, + }} + w.PollOnce(context.Background()) + for _, c := range []struct { + name string + busy, known bool + }{ + {"old-ninfer", false, false}, + {"gauged", false, true}, + } { + if b, k := w.Busy(c.name); b != c.busy || k != c.known { + t.Errorf("Busy(%q) = %v,%v want %v,%v", c.name, b, k, c.busy, c.known) + } + } +} diff --git a/internal/engines/engines.go b/internal/engines/engines.go index faff156b..1e90c165 100644 --- a/internal/engines/engines.go +++ b/internal/engines/engines.go @@ -343,6 +343,7 @@ func (w *Watcher) read(ctx context.Context, e config.Engine) proto.Engine { } if ss, serr := w.slots(ctx, base+"/slots", "", ""); serr == nil { out.Slots = ss + out.Metrics.InFlightReported = true } } case "custom": @@ -503,6 +504,7 @@ func (w *Watcher) custom(ctx context.Context, base string, e config.Engine, out PromptTokens: f("prompt_tokens"), OutputTokens: f("output_tokens"), PromptSeconds: f("prompt_seconds"), OutputSeconds: f("output_seconds"), Processing: f("processing"), Waiting: f("waiting"), + InFlightReported: inFlight(vals, e.Map), } out.MetricsReported = true } @@ -602,6 +604,7 @@ func (w *Watcher) metrics(ctx context.Context, url string, names map[string]stri m.PromptSeconds, m.OutputSeconds = f("prompt_seconds"), f("output_seconds") m.Processing, m.Waiting = f("processing"), f("waiting") m.DraftTokens, m.DraftAccepted = vals["ninfer:draft_tokens_total"], vals["ninfer:draft_accepted_tokens_total"] + m.InFlightReported = inFlight(vals, names) return m, nil } @@ -623,9 +626,13 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine) if err := json.Unmarshal(b, &r); err != nil { return fmt.Errorf("/running: %w", err) } + // The engine reports in-flight work only when every model on it does: + // one model that cannot say hides its requests from the sum. + reported := true for _, m := range r.Running { out.Models = append(out.Models, proto.Model{Name: m.Model, State: m.State, File: models.ModelArg(m.Cmd)}) if m.State != "ready" || m.Proxy == "" { + reported = reported && m.State != "ready" continue } px := strings.TrimRight(m.Proxy, "/") @@ -635,6 +642,7 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine) out.Metrics = out.Metrics.Add(mm) out.MetricsReported = true } + modelReported := err == nil && mm.InFlightReported // Best-effort per model: the proxy's context window and its live KV // slots, tagged with the model they sit on. A 404/501 on either // leaves the engine up with what it has. @@ -648,8 +656,11 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine) } if ss, serr := w.slots(ctx, px+"/slots", m.Model, ""); serr == nil { out.Slots = append(out.Slots, ss...) + modelReported = true } + reported = reported && modelReported } + out.Metrics.InFlightReported = reported for _, md := range out.Models { if md.State == "ready" && md.Window > out.Window { out.Window = md.Window @@ -814,6 +825,7 @@ func (w *Watcher) strata(ctx context.Context, base, keyEnv string, out *proto.En out.Metrics.Processing = 1 } out.Metrics.Waiting = float64(m.Live.Queued) + out.Metrics.InFlightReported = m.Live.Running != nil || m.Live.State != "" out.Metrics.PromptTokens = m.Totals.PromptTokens out.Metrics.OutputTokens = m.Totals.OutputTokens out.Metrics.PromptSeconds = m.Totals.PromptMS / 1000 @@ -821,6 +833,19 @@ func (w *Watcher) strata(ctx context.Context, base, keyEnv string, out *proto.En return nil } +// inFlight is whether a Prometheus reading carries the engine's in-flight +// gauges, so that a zero in them means idle rather than not reported. +func inFlight(vals map[string]float64, names map[string]string) bool { + for _, field := range []string{"processing", "waiting"} { + if name, ok := names[field]; ok { + if _, ok := vals[name]; ok { + return true + } + } + } + return false +} + // ParseProm reads Prometheus text exposition into name -> value, summing // series that differ only by labels. func ParseProm(text string) map[string]float64 { diff --git a/internal/engines/more.go b/internal/engines/more.go index 1f16c9fc..9d43adc2 100644 --- a/internal/engines/more.go +++ b/internal/engines/more.go @@ -80,6 +80,7 @@ func (w *Watcher) promMetrics(ctx context.Context, url, keyEnv, prefix string, n PromptTokens: f("prompt_tokens"), OutputTokens: f("output_tokens"), PromptSeconds: f("prompt_seconds"), OutputSeconds: f("output_seconds"), Processing: f("processing"), Waiting: f("waiting"), + InFlightReported: inFlight(vals, names), }, true } @@ -257,8 +258,11 @@ func (w *Watcher) freetoken(ctx context.Context, base, keyEnv string, out *proto out.Metrics.PromptTokens = pick("tokens_prompt", "prompt_tokens", "input_tokens") out.Metrics.Waiting = pick("queue", "requests_waiting", "pending_requests") for _, k := range []string{"generating", "busy", "active"} { - if v, ok := body[k].(bool); ok && v { - out.Metrics.Processing = 1 + if v, ok := body[k].(bool); ok { + out.Metrics.InFlightReported = true + if v { + out.Metrics.Processing = 1 + } } } return nil @@ -273,6 +277,7 @@ func (w *Watcher) comfyui(ctx context.Context, base, keyEnv string, out *proto.E return err } out.Metrics.Processing, out.Metrics.Waiting = float64(running), float64(pending) + out.Metrics.InFlightReported = true if b, err := w.getKey(ctx, base+"/system_stats", keyEnv); err == nil { var s struct { System struct { diff --git a/internal/proto/proto.go b/internal/proto/proto.go index ef9cc04f..5d1473e2 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -756,6 +756,10 @@ type EngineMetrics struct { Waiting float64 `json:"waiting"` DraftTokens float64 `json:"draft_tokens,omitempty"` DraftAccepted float64 `json:"draft_accepted,omitempty"` + // InFlightReported is true when the reading carries a request-in-flight + // figure (a processing or waiting gauge, a job queue). Without it a zero + // Processing is unknown, not idle. + InFlightReported bool `json:"in_flight_reported,omitempty"` } func (m EngineMetrics) Add(o EngineMetrics) EngineMetrics { From 10d5a22ffba7f5182d621ebc996fbc4e976eb337 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:07:15 -0700 Subject: [PATCH 2/3] test(broker): a minimum turn needs a request served --- internal/broker/minturn_test.go | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/internal/broker/minturn_test.go b/internal/broker/minturn_test.go index b66c1a2d..35333c65 100644 --- a/internal/broker/minturn_test.go +++ b/internal/broker/minturn_test.go @@ -239,3 +239,55 @@ func TestReattachKeepsTheTurnItWasIn(t *testing.T) { t.Errorf("reason = %q, want %q", got, want) } } + +// turnBusy is the engine reading a minimum-turn test sets: the watcher's +// answer for the llama-swap group, known and idle until the test says busy. +type turnBusy struct{ busy atomic.Bool } + +func (r *turnBusy) read(key string) (bool, bool) { return r.busy.Load(), key == "llama-swap" } + +func startBusyTurnBroker(t *testing.T, r *turnBusy) (*Broker, *minClock) { + t.Helper() + b, _ := jugglers(t, Config{JuggleWindow: -1, Busy: r.read}) + clk := newMinClock(turnBase) + b.nowFn = clk.now + return b, clk +} + +// A lease whose engine was started by a probe and never served a request +// (seen up and idle since the grant) has no minimum turn to keep: a waiter +// takes the card at once instead of waiting out five minutes nobody uses. +func TestMinTurnNeedsARequestServed(t *testing.T) { + r := &turnBusy{} + b, clk := startBusyTurnBroker(t, r) + holder := must(t, acquire(b, Request{Engine: "NInfer-Qwen3.8-27B", Group: "llama-swap", Priority: Foreground, MinTurn: 5 * time.Minute})) + b.Tick(context.Background()) + clk.set(turnBase.Add(30 * time.Second)) + + w := acquireW(b, Request{Engine: "strata", Priority: Foreground}) + pending(t, w.ch) + if why := drained(t, holder); !strings.Contains(why, "strata") { + t.Fatalf("drain reason = %q, want the waiter named", why) + } +} + +// Once the engine has served a request its turn holds, through the idle +// gaps between requests. +func TestMinTurnHoldsAfterARequestServed(t *testing.T) { + r := &turnBusy{} + b, clk := startBusyTurnBroker(t, r) + holder := must(t, acquire(b, Request{Engine: "NInfer-Qwen3.8-27B", Group: "llama-swap", Priority: Foreground, MinTurn: 5 * time.Minute})) + b.Tick(context.Background()) + r.busy.Store(true) + b.Tick(context.Background()) + r.busy.Store(false) + b.Tick(context.Background()) + clk.set(turnBase.Add(30 * time.Second)) + + w := acquireW(b, Request{Engine: "strata", Priority: Foreground}) + pending(t, w.ch) + notDrained(t, holder) + if got, want := firstReason(t, w), "next in line: NInfer-Qwen3.8-27B has its minimum turn, 4m30s left"; got != want { + t.Errorf("reason = %q, want %q", got, want) + } +} From 8736715510d3b2ca3132c40d3cd1d47dfb69577d Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:07:17 -0700 Subject: [PATCH 3/3] fix(broker): a minimum turn protects only a lease that served a request A lease's minimum turn ran from the grant. When a health probe or a warm-up started an engine, the lease kept the card for its whole turn although nobody sent it a request. On aphotic a HADAL warm-up started NInfer, drained Strata mid-session, and Strata's next request waited out NInfer's five minutes. Each tick the broker reads whether an active lease's engine is busy. A lease read idle with no busy reading since the grant has no turn to keep, and the status shows none. With no reading (no watcher, an engine still loading, an engine with no in-flight figure) the turn holds as before. --- CHANGELOG.md | 10 ++++++++++ internal/broker/broker.go | 33 +++++++++++++++++++++++++++++---- internal/broker/status.go | 2 +- 3 files changed, 40 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4fb98a2..fd52e184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ has a matching GitHub release whose body comes from its section below. ## [Unreleased] +### Fixed + +- A minimum turn (`--min-turn`) now protects only a lease whose engine has served a request. An engine started by + a health probe or a warm-up, read up and idle with nothing served, no longer keeps the card for its full turn + while a real request waits. +- An engine that reports no in-flight figure (no processing gauge, no `/slots`, no queue) now reads as unknown + instead of idle. Before, an NInfer build without `/slots`, or Ollama, read idle while it served, and a juggle + handed its card over as if no request were running. `berth status --format json` shows the new + `in_flight_reported` field per engine. + ## [v0.6.0] - 2026-11-14 v0.6.0 gives each card a schedule and makes Berth explain its choices. `berth schedule` books an engine's hours, diff --git a/internal/broker/broker.go b/internal/broker/broker.go index 4efe10b1..e5754190 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -291,9 +291,14 @@ type lease struct { lockCh chan string switched bool // a person's switch (Request.Switched) minTurn time.Duration - parkTTL time.Duration // 0 = the node's - window proto.Window // zero = no window - prio Priority + // unused: since the grant the engine was read up and idle, and never + // busy. A probe started it, nobody asked it anything, so it has no + // minimum turn to keep. served: it was read busy once. + unused bool + served bool + parkTTL time.Duration // 0 = the node's + window proto.Window // zero = no window + prio Priority // basePrio is the class the lease asked for, or the operator set; prio // differs from it while a booking holds (booking.go). basePrio Priority @@ -425,7 +430,26 @@ func (l *lease) awaitingGrant() bool { // than the turn, by the broker's clock (a duration from the grant, not // an order among leases). func (l *lease) inTurn(now time.Time) bool { - return l.minTurn > 0 && now.Sub(l.granted) < l.minTurn + return l.minTurn > 0 && !l.unused && now.Sub(l.granted) < l.minTurn +} + +// noteServed reads whether each active lease's engine serves, for the +// minimum turn: a lease seen busy once has served; one seen only idle is +// unused. With no reading (no watcher, an engine still loading) neither +// is set, and the turn holds as it always did. +func (b *Broker) noteServed() { + if b.cfg.Busy == nil { + return + } + for _, l := range b.leases { + if l.state != active || l.minTurn == 0 || l.served { + continue + } + if busy, known := b.cfg.Busy(l.key()); known { + l.served = busy + l.unused = !busy + } + } } // turnWhy is the line a waiter blocked by l's minimum turn is told. @@ -981,6 +1005,7 @@ func (b *Broker) Tick(ctx context.Context) { b.measureRAM(l) } b.claimed = claimed + b.noteServed() b.closeWindows(now) b.closeForDowntime(now) b.evictParked(now) diff --git a/internal/broker/status.go b/internal/broker/status.go index a5acc929..0fa0a8db 100644 --- a/internal/broker/status.go +++ b/internal/broker/status.go @@ -93,7 +93,7 @@ func (b *Broker) Status() proto.Status { if l.state == parked { pl.ParkedSince = parkedAge(l, now) } - if l.minTurn > 0 { + if l.minTurn > 0 && !l.unused { if rem := l.minTurn - now.Sub(l.granted); rem > 0 { pl.TurnS = rem.Seconds() }