diff --git a/CHANGELOG.md b/CHANGELOG.md index c4fb98a..fd52e18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ has a matching GitHub release whose body comes from its section below. ## [Unreleased] +### Fixed + +- A minimum turn (`--min-turn`) now protects only a lease whose engine has served a request. An engine started by + a health probe or a warm-up, read up and idle with nothing served, no longer keeps the card for its full turn + while a real request waits. +- An engine that reports no in-flight figure (no processing gauge, no `/slots`, no queue) now reads as unknown + instead of idle. Before, an NInfer build without `/slots`, or Ollama, read idle while it served, and a juggle + handed its card over as if no request were running. `berth status --format json` shows the new + `in_flight_reported` field per engine. + ## [v0.6.0] - 2026-11-14 v0.6.0 gives each card a schedule and makes Berth explain its choices. `berth schedule` books an engine's hours, diff --git a/internal/broker/broker.go b/internal/broker/broker.go index 4efe10b..e575419 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -291,9 +291,14 @@ type lease struct { lockCh chan string switched bool // a person's switch (Request.Switched) minTurn time.Duration - parkTTL time.Duration // 0 = the node's - window proto.Window // zero = no window - prio Priority + // unused: since the grant the engine was read up and idle, and never + // busy. A probe started it, nobody asked it anything, so it has no + // minimum turn to keep. served: it was read busy once. + unused bool + served bool + parkTTL time.Duration // 0 = the node's + window proto.Window // zero = no window + prio Priority // basePrio is the class the lease asked for, or the operator set; prio // differs from it while a booking holds (booking.go). basePrio Priority @@ -425,7 +430,26 @@ func (l *lease) awaitingGrant() bool { // than the turn, by the broker's clock (a duration from the grant, not // an order among leases). func (l *lease) inTurn(now time.Time) bool { - return l.minTurn > 0 && now.Sub(l.granted) < l.minTurn + return l.minTurn > 0 && !l.unused && now.Sub(l.granted) < l.minTurn +} + +// noteServed reads whether each active lease's engine serves, for the +// minimum turn: a lease seen busy once has served; one seen only idle is +// unused. With no reading (no watcher, an engine still loading) neither +// is set, and the turn holds as it always did. +func (b *Broker) noteServed() { + if b.cfg.Busy == nil { + return + } + for _, l := range b.leases { + if l.state != active || l.minTurn == 0 || l.served { + continue + } + if busy, known := b.cfg.Busy(l.key()); known { + l.served = busy + l.unused = !busy + } + } } // turnWhy is the line a waiter blocked by l's minimum turn is told. @@ -981,6 +1005,7 @@ func (b *Broker) Tick(ctx context.Context) { b.measureRAM(l) } b.claimed = claimed + b.noteServed() b.closeWindows(now) b.closeForDowntime(now) b.evictParked(now) diff --git a/internal/broker/minturn_test.go b/internal/broker/minturn_test.go index b66c1a2..35333c6 100644 --- a/internal/broker/minturn_test.go +++ b/internal/broker/minturn_test.go @@ -239,3 +239,55 @@ func TestReattachKeepsTheTurnItWasIn(t *testing.T) { t.Errorf("reason = %q, want %q", got, want) } } + +// turnBusy is the engine reading a minimum-turn test sets: the watcher's +// answer for the llama-swap group, known and idle until the test says busy. +type turnBusy struct{ busy atomic.Bool } + +func (r *turnBusy) read(key string) (bool, bool) { return r.busy.Load(), key == "llama-swap" } + +func startBusyTurnBroker(t *testing.T, r *turnBusy) (*Broker, *minClock) { + t.Helper() + b, _ := jugglers(t, Config{JuggleWindow: -1, Busy: r.read}) + clk := newMinClock(turnBase) + b.nowFn = clk.now + return b, clk +} + +// A lease whose engine was started by a probe and never served a request +// (seen up and idle since the grant) has no minimum turn to keep: a waiter +// takes the card at once instead of waiting out five minutes nobody uses. +func TestMinTurnNeedsARequestServed(t *testing.T) { + r := &turnBusy{} + b, clk := startBusyTurnBroker(t, r) + holder := must(t, acquire(b, Request{Engine: "NInfer-Qwen3.8-27B", Group: "llama-swap", Priority: Foreground, MinTurn: 5 * time.Minute})) + b.Tick(context.Background()) + clk.set(turnBase.Add(30 * time.Second)) + + w := acquireW(b, Request{Engine: "strata", Priority: Foreground}) + pending(t, w.ch) + if why := drained(t, holder); !strings.Contains(why, "strata") { + t.Fatalf("drain reason = %q, want the waiter named", why) + } +} + +// Once the engine has served a request its turn holds, through the idle +// gaps between requests. +func TestMinTurnHoldsAfterARequestServed(t *testing.T) { + r := &turnBusy{} + b, clk := startBusyTurnBroker(t, r) + holder := must(t, acquire(b, Request{Engine: "NInfer-Qwen3.8-27B", Group: "llama-swap", Priority: Foreground, MinTurn: 5 * time.Minute})) + b.Tick(context.Background()) + r.busy.Store(true) + b.Tick(context.Background()) + r.busy.Store(false) + b.Tick(context.Background()) + clk.set(turnBase.Add(30 * time.Second)) + + w := acquireW(b, Request{Engine: "strata", Priority: Foreground}) + pending(t, w.ch) + notDrained(t, holder) + if got, want := firstReason(t, w), "next in line: NInfer-Qwen3.8-27B has its minimum turn, 4m30s left"; got != want { + t.Errorf("reason = %q, want %q", got, want) + } +} diff --git a/internal/broker/status.go b/internal/broker/status.go index a5acc92..0fa0a8d 100644 --- a/internal/broker/status.go +++ b/internal/broker/status.go @@ -93,7 +93,7 @@ func (b *Broker) Status() proto.Status { if l.state == parked { pl.ParkedSince = parkedAge(l, now) } - if l.minTurn > 0 { + if l.minTurn > 0 && !l.unused { if rem := l.minTurn - now.Sub(l.granted); rem > 0 { pl.TurnS = rem.Seconds() } diff --git a/internal/engines/busy.go b/internal/engines/busy.go index 506e709..bd6a15b 100644 --- a/internal/engines/busy.go +++ b/internal/engines/busy.go @@ -52,7 +52,7 @@ func (w *Watcher) Busy(name string) (busy, known bool) { return true, true // starting or stopping } } - return false, true + return false, e.Metrics.InFlightReported } return false, false } diff --git a/internal/engines/busy_test.go b/internal/engines/busy_test.go index cfb2d3f..24bbf72 100644 --- a/internal/engines/busy_test.go +++ b/internal/engines/busy_test.go @@ -4,6 +4,10 @@ package engines import ( + "context" + "fmt" + "net/http" + "net/http/httptest" "testing" "time" @@ -18,7 +22,7 @@ func TestBusy(t *testing.T) { "llama-swap": {Name: "llama-swap", Up: true, At: now, Models: []proto.Model{{Name: "NInfer-Qwen3.8-27B", State: "ready"}}, Slots: []proto.Slot{{Model: "NInfer-Qwen3.8-27B", Processing: true}}}, - "comfyui": {Name: "comfyui", Up: true, At: now}, + "comfyui": {Name: "comfyui", Up: true, At: now, Metrics: proto.EngineMetrics{InFlightReported: true}}, "strata": {Name: "strata", Up: false, At: now}, "old": {Name: "old", Up: true, At: now.Add(-time.Hour)}, } @@ -50,3 +54,46 @@ func TestBusy(t *testing.T) { t.Error("a nil watcher claimed to know") } } + +// An engine whose reading carries no request-in-flight figure cannot be +// called idle: the NInfer build on aphotic answers /metrics with only its +// own ninfer_* counters and serves no /slots, so a request it is serving +// reads exactly like no request. Busy says unknown for it, and idle again +// once a reading names the in-flight count. +func TestBusyIsUnknownWithoutAnInFlightFigure(t *testing.T) { + ninfer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/metrics": + fmt.Fprintln(w, `ninfer_requests_total{outcome="ok"} 12`) + default: + http.NotFound(w, r) + } + })) + defer ninfer.Close() + gauged := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/metrics": + fmt.Fprintln(w, "llamacpp:requests_processing 0") + default: + http.NotFound(w, r) + } + })) + defer gauged.Close() + + w := &Watcher{Engines: []config.Engine{ + {Name: "old-ninfer", Type: "ninfer", URL: ninfer.URL}, + {Name: "gauged", Type: "ninfer", URL: gauged.URL}, + }} + w.PollOnce(context.Background()) + for _, c := range []struct { + name string + busy, known bool + }{ + {"old-ninfer", false, false}, + {"gauged", false, true}, + } { + if b, k := w.Busy(c.name); b != c.busy || k != c.known { + t.Errorf("Busy(%q) = %v,%v want %v,%v", c.name, b, k, c.busy, c.known) + } + } +} diff --git a/internal/engines/engines.go b/internal/engines/engines.go index faff156..1e90c16 100644 --- a/internal/engines/engines.go +++ b/internal/engines/engines.go @@ -343,6 +343,7 @@ func (w *Watcher) read(ctx context.Context, e config.Engine) proto.Engine { } if ss, serr := w.slots(ctx, base+"/slots", "", ""); serr == nil { out.Slots = ss + out.Metrics.InFlightReported = true } } case "custom": @@ -503,6 +504,7 @@ func (w *Watcher) custom(ctx context.Context, base string, e config.Engine, out PromptTokens: f("prompt_tokens"), OutputTokens: f("output_tokens"), PromptSeconds: f("prompt_seconds"), OutputSeconds: f("output_seconds"), Processing: f("processing"), Waiting: f("waiting"), + InFlightReported: inFlight(vals, e.Map), } out.MetricsReported = true } @@ -602,6 +604,7 @@ func (w *Watcher) metrics(ctx context.Context, url string, names map[string]stri m.PromptSeconds, m.OutputSeconds = f("prompt_seconds"), f("output_seconds") m.Processing, m.Waiting = f("processing"), f("waiting") m.DraftTokens, m.DraftAccepted = vals["ninfer:draft_tokens_total"], vals["ninfer:draft_accepted_tokens_total"] + m.InFlightReported = inFlight(vals, names) return m, nil } @@ -623,9 +626,13 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine) if err := json.Unmarshal(b, &r); err != nil { return fmt.Errorf("/running: %w", err) } + // The engine reports in-flight work only when every model on it does: + // one model that cannot say hides its requests from the sum. + reported := true for _, m := range r.Running { out.Models = append(out.Models, proto.Model{Name: m.Model, State: m.State, File: models.ModelArg(m.Cmd)}) if m.State != "ready" || m.Proxy == "" { + reported = reported && m.State != "ready" continue } px := strings.TrimRight(m.Proxy, "/") @@ -635,6 +642,7 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine) out.Metrics = out.Metrics.Add(mm) out.MetricsReported = true } + modelReported := err == nil && mm.InFlightReported // Best-effort per model: the proxy's context window and its live KV // slots, tagged with the model they sit on. A 404/501 on either // leaves the engine up with what it has. @@ -648,8 +656,11 @@ func (w *Watcher) llamaSwap(ctx context.Context, base string, out *proto.Engine) } if ss, serr := w.slots(ctx, px+"/slots", m.Model, ""); serr == nil { out.Slots = append(out.Slots, ss...) + modelReported = true } + reported = reported && modelReported } + out.Metrics.InFlightReported = reported for _, md := range out.Models { if md.State == "ready" && md.Window > out.Window { out.Window = md.Window @@ -814,6 +825,7 @@ func (w *Watcher) strata(ctx context.Context, base, keyEnv string, out *proto.En out.Metrics.Processing = 1 } out.Metrics.Waiting = float64(m.Live.Queued) + out.Metrics.InFlightReported = m.Live.Running != nil || m.Live.State != "" out.Metrics.PromptTokens = m.Totals.PromptTokens out.Metrics.OutputTokens = m.Totals.OutputTokens out.Metrics.PromptSeconds = m.Totals.PromptMS / 1000 @@ -821,6 +833,19 @@ func (w *Watcher) strata(ctx context.Context, base, keyEnv string, out *proto.En return nil } +// inFlight is whether a Prometheus reading carries the engine's in-flight +// gauges, so that a zero in them means idle rather than not reported. +func inFlight(vals map[string]float64, names map[string]string) bool { + for _, field := range []string{"processing", "waiting"} { + if name, ok := names[field]; ok { + if _, ok := vals[name]; ok { + return true + } + } + } + return false +} + // ParseProm reads Prometheus text exposition into name -> value, summing // series that differ only by labels. func ParseProm(text string) map[string]float64 { diff --git a/internal/engines/more.go b/internal/engines/more.go index 1f16c9f..9d43adc 100644 --- a/internal/engines/more.go +++ b/internal/engines/more.go @@ -80,6 +80,7 @@ func (w *Watcher) promMetrics(ctx context.Context, url, keyEnv, prefix string, n PromptTokens: f("prompt_tokens"), OutputTokens: f("output_tokens"), PromptSeconds: f("prompt_seconds"), OutputSeconds: f("output_seconds"), Processing: f("processing"), Waiting: f("waiting"), + InFlightReported: inFlight(vals, names), }, true } @@ -257,8 +258,11 @@ func (w *Watcher) freetoken(ctx context.Context, base, keyEnv string, out *proto out.Metrics.PromptTokens = pick("tokens_prompt", "prompt_tokens", "input_tokens") out.Metrics.Waiting = pick("queue", "requests_waiting", "pending_requests") for _, k := range []string{"generating", "busy", "active"} { - if v, ok := body[k].(bool); ok && v { - out.Metrics.Processing = 1 + if v, ok := body[k].(bool); ok { + out.Metrics.InFlightReported = true + if v { + out.Metrics.Processing = 1 + } } } return nil @@ -273,6 +277,7 @@ func (w *Watcher) comfyui(ctx context.Context, base, keyEnv string, out *proto.E return err } out.Metrics.Processing, out.Metrics.Waiting = float64(running), float64(pending) + out.Metrics.InFlightReported = true if b, err := w.getKey(ctx, base+"/system_stats", keyEnv); err == nil { var s struct { System struct { diff --git a/internal/proto/proto.go b/internal/proto/proto.go index ef9cc04..5d1473e 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -756,6 +756,10 @@ type EngineMetrics struct { Waiting float64 `json:"waiting"` DraftTokens float64 `json:"draft_tokens,omitempty"` DraftAccepted float64 `json:"draft_accepted,omitempty"` + // InFlightReported is true when the reading carries a request-in-flight + // figure (a processing or waiting gauge, a job queue). Without it a zero + // Processing is unknown, not idle. + InFlightReported bool `json:"in_flight_reported,omitempty"` } func (m EngineMetrics) Add(o EngineMetrics) EngineMetrics {