From c6a7ac2bac920612bd54d2fe044ed83c0d888953 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:53:09 -0700 Subject: [PATCH] feat(gateway): a not_ready 503 says the lease's place in line A non-streaming request whose start timed out got a bare 503 with Retry-After 30, even while its lease stood in line for the card. The gateway now reads the lease's place in line from berthd when the start times out, and the 503 carries X-Berth-Queue-Position and, when the broker has measured the starts ahead, X-Berth-Start-Estimate: N; lower-bound. Retry-After is the larger of the estimate and 30 s. The broker's estimate counts only the measured starts ahead in the queue and not the holder's time left, so the header marks it as a lower bound instead of passing it off as a promise. --- CHANGELOG.md | 4 +++ docs/gateway.md | 7 ++++ internal/gateway/gateway.go | 34 ++++++++++++++++++- internal/gateway/gateway_test.go | 56 ++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a20904..7ac6b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,10 @@ has a matching GitHub release whose body comes from its section below. - The harbor's SWITCH tab has a QUEUE panel. It lists the waiters in grant order, and a paired device gets FIRST, a priority picker and DROP on each one. These buttons go from the harbor straight to berthd, never through the gateway, because agents hold the gateway's API key and a queue change is the operator's to make. +- A gateway request whose start times out (`not_ready`) while its lease is still in line for the card says where + it stands. The 503 carries `X-Berth-Queue-Position` and, when berthd has measured the starts ahead, + `X-Berth-Start-Estimate: N; lower-bound`. `Retry-After` is the larger of the estimate and 30 s. The estimate + leaves out the holder's time left, so it is a minimum, not a promise. - `berth wire llama-swap` names each llama-swap group where llama-swap swaps two or more wired models itself. There llama-swap unloads the old model before Berth sees the request, so `min_turn`, the engine ranking and the juggle guard never apply between them. The JSON carries the groups as `swapping`. The llama-swap section of diff --git a/docs/gateway.md b/docs/gateway.md index 2d1b3b9..05a6d27 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -18,6 +18,13 @@ every retry: for 15 s, doubling to 5 min, the gateway answers at once with the same cause, and `Retry-After` says when the next start is allowed. A start that answers clears it. A model listed under `serves` by two engines fails `berth config check`. +- **Queue position.** A `not_ready` start whose lease is still in line for the card says where it + stands. `X-Berth-Queue-Position` is its place in line, 1 for next. `X-Berth-Start-Estimate: + 45; lower-bound` is the seconds berthd expects the start to take once the card frees. It counts + only the measured starts ahead in the queue and not the holder's time left, so treat it as a + minimum. It is missing when nothing was measured. `Retry-After` is then the larger of that + estimate and 30 s. A streaming request gets the same error as an SSE event, without these + headers. - **API key.** Bearer token on every route; `berth key` prints it (created on first use, mode 600). Loopback only; browser origins and non-loopback Host names are refused. diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 28a937d..d724424 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -580,6 +580,11 @@ type startError struct { code string // no_start, outranked, drained_while_starting, no_berth, start_failed, engine_oom, not_ready, no_headroom msg string retry time.Duration // when set, the client is told to try again after this (Retry-After) + // queue is the engine's lease's place in line when the start timed out + // waiting for the card, 0 when it was not waiting; estimate is the + // broker's lower-bound start estimate for it, 0 for unknown. + queue int + estimate time.Duration } func (e *startError) Error() string { return e.msg } @@ -791,6 +796,12 @@ func (g *Gateway) startFailed(w http.ResponseWriter, engine, what string, err er if se != nil && se.retry > 0 { w.Header().Set("Retry-After", strconv.Itoa(int((se.retry+time.Second-1)/time.Second))) } + if se != nil && se.queue > 0 { + w.Header().Set("X-Berth-Queue-Position", strconv.Itoa(se.queue)) + if se.estimate > 0 { + w.Header().Set("X-Berth-Start-Estimate", fmt.Sprintf("%d; lower-bound", int((se.estimate+time.Second-1)/time.Second))) + } + } apiErrorCode(w, http.StatusServiceUnavailable, code, err.Error()) } @@ -1011,11 +1022,32 @@ func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) e case <-time.After(250 * time.Millisecond): } if time.Now().After(deadline) { - return &startError{code: "not_ready", msg: fmt.Sprintf("%s did not answer within %s", e.Name, ready)} + return g.queued(e, &startError{code: "not_ready", msg: fmt.Sprintf("%s did not answer within %s", e.Name, ready)}) } } } +// queued adds where the engine's lease waits to a start that timed out: its +// place in line and the broker's start estimate. The estimate counts only the +// measured starts ahead of it, never the holder's time left, so it is a lower +// bound, and the client is told to retry no sooner than it or 30 s. +func (g *Gateway) queued(e *config.Engine, se *startError) *startError { + v := node.View(g.nodeStatus(), e.GPU) + if v == nil { + return se + } + for _, l := range v.Waiting { + if l.Engine == e.Name && l.Position > 0 && (se.queue == 0 || l.Position < se.queue) { + se.queue, se.estimate = l.Position, time.Duration(l.EstimateS*float64(time.Second)) + } + } + if se.queue > 0 { + se.retry = max(30*time.Second, se.estimate) + se.msg += fmt.Sprintf("; its lease is number %d in line for the card", se.queue) + } + return se +} + func (g *Gateway) ready() time.Duration { if g.Ready > 0 { return g.Ready diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 00f4a06..2779394 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -31,6 +31,7 @@ import ( "github.com/T-Crypt/berth/internal/gpu" "github.com/T-Crypt/berth/internal/node" "github.com/T-Crypt/berth/internal/procs" + "github.com/T-Crypt/berth/internal/proto" "github.com/T-Crypt/berth/internal/runner" "github.com/T-Crypt/berth/internal/server" ) @@ -1008,3 +1009,58 @@ func TestComfyPromptResolves(t *testing.T) { t.Fatalf("queue with comfyui down = %d %s, want the 503", resp.StatusCode, body) } } + +// A start that times out while its lease still waits for the card tells the +// client where it stands: its place in line, and a Retry-After of at least +// 30 s. Here a pinned hold keeps the card, so the gateway's run waits. +func TestNotReadyCarriesTheQueue(t *testing.T) { + sock, b := daemon(t) + hold, err := b.Acquire(context.Background(), broker.Request{Engine: "game", Priority: broker.Hold, Pinned: true, Wait: time.Second}, nil) + if err != nil { + t.Fatal(err) + } + defer hold.Release() + g := &Gateway{Socket: sock, Key: testKey, Ready: 500 * time.Millisecond, Log: slogDiscard(), + Engines: []config.Engine{{Name: "e", URL: "http://" + freeAddr(t), Start: []string{os.Args[0]}, Serves: []string{"m"}}}} + gw := httptest.NewServer(g.Handler()) + defer gw.Close() + + resp := call(t, "POST", gw.URL+"/v1/chat/completions", `{"model":"m"}`) + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("X-Berth-Queue-Position") != "1" || + resp.Header.Get("Retry-After") != "30" || !strings.Contains(string(body), "number 1 in line") { + t.Fatalf("status %d, queue %q, Retry-After %q: %s", resp.StatusCode, + resp.Header.Get("X-Berth-Queue-Position"), resp.Header.Get("Retry-After"), body) + } +} + +// The broker's start estimate for the engine's lease reaches the client +// rounded up and marked as a lower bound, and Retry-After waits for the +// larger of it and 30 s. Another engine's place in line is not this one's. +func TestQueuedStartHeaders(t *testing.T) { + for _, tc := range []struct { + estimate float64 + retry, queue, start string + }{ + {44.2, "45", "2", "45; lower-bound"}, + {5, "30", "2", "5; lower-bound"}, + {0, "30", "2", ""}, + } { + sock := statusDaemon(t, proto.Status{ + GPUs: []proto.GPU{{Card: "gpu0", Scheduled: true}}, + Waiting: []proto.Lease{ + {Card: "gpu0", Engine: "other", Position: 1, EstimateS: 900}, + {Card: "gpu0", Engine: "e", Position: 2, EstimateS: tc.estimate}, + }, + }) + g := &Gateway{Socket: sock, Log: slogDiscard()} + w := httptest.NewRecorder() + g.startFailed(w, "e", "m", g.queued(&config.Engine{Name: "e"}, &startError{code: "not_ready", msg: "e did not answer"})) + h := w.Result().Header + got := [...]string{h.Get("Retry-After"), h.Get("X-Berth-Queue-Position"), h.Get("X-Berth-Start-Estimate")} + if got != [...]string{tc.retry, tc.queue, tc.start} { + t.Errorf("estimate %v s: Retry-After, queue, start = %q", tc.estimate, got) + } + } +}