diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a20904..7ac6b26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,10 @@ has a matching GitHub release whose body comes from its section below. - The harbor's SWITCH tab has a QUEUE panel. It lists the waiters in grant order, and a paired device gets FIRST, a priority picker and DROP on each one. These buttons go from the harbor straight to berthd, never through the gateway, because agents hold the gateway's API key and a queue change is the operator's to make. +- A gateway request whose start times out (`not_ready`) while its lease is still in line for the card says where + it stands. The 503 carries `X-Berth-Queue-Position` and, when berthd has measured the starts ahead, + `X-Berth-Start-Estimate: N; lower-bound`. `Retry-After` is the larger of the estimate and 30 s. The estimate + leaves out the holder's time left, so it is a minimum, not a promise. - `berth wire llama-swap` names each llama-swap group where llama-swap swaps two or more wired models itself. There llama-swap unloads the old model before Berth sees the request, so `min_turn`, the engine ranking and the juggle guard never apply between them. The JSON carries the groups as `swapping`. The llama-swap section of diff --git a/docs/gateway.md b/docs/gateway.md index 2d1b3b9..05a6d27 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -18,6 +18,13 @@ every retry: for 15 s, doubling to 5 min, the gateway answers at once with the same cause, and `Retry-After` says when the next start is allowed. A start that answers clears it. A model listed under `serves` by two engines fails `berth config check`. +- **Queue position.** A `not_ready` start whose lease is still in line for the card says where it + stands. `X-Berth-Queue-Position` is its place in line, 1 for next. `X-Berth-Start-Estimate: + 45; lower-bound` is the seconds berthd expects the start to take once the card frees. It counts + only the measured starts ahead in the queue and not the holder's time left, so treat it as a + minimum. It is missing when nothing was measured. `Retry-After` is then the larger of that + estimate and 30 s. A streaming request gets the same error as an SSE event, without these + headers. - **API key.** Bearer token on every route; `berth key` prints it (created on first use, mode 600). Loopback only; browser origins and non-loopback Host names are refused. diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index 28a937d..d724424 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -580,6 +580,11 @@ type startError struct { code string // no_start, outranked, drained_while_starting, no_berth, start_failed, engine_oom, not_ready, no_headroom msg string retry time.Duration // when set, the client is told to try again after this (Retry-After) + // queue is the engine's lease's place in line when the start timed out + // waiting for the card, 0 when it was not waiting; estimate is the + // broker's lower-bound start estimate for it, 0 for unknown. + queue int + estimate time.Duration } func (e *startError) Error() string { return e.msg } @@ -791,6 +796,12 @@ func (g *Gateway) startFailed(w http.ResponseWriter, engine, what string, err er if se != nil && se.retry > 0 { w.Header().Set("Retry-After", strconv.Itoa(int((se.retry+time.Second-1)/time.Second))) } + if se != nil && se.queue > 0 { + w.Header().Set("X-Berth-Queue-Position", strconv.Itoa(se.queue)) + if se.estimate > 0 { + w.Header().Set("X-Berth-Start-Estimate", fmt.Sprintf("%d; lower-bound", int((se.estimate+time.Second-1)/time.Second))) + } + } apiErrorCode(w, http.StatusServiceUnavailable, code, err.Error()) } @@ -1011,11 +1022,32 @@ func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) e case <-time.After(250 * time.Millisecond): } if time.Now().After(deadline) { - return &startError{code: "not_ready", msg: fmt.Sprintf("%s did not answer within %s", e.Name, ready)} + return g.queued(e, &startError{code: "not_ready", msg: fmt.Sprintf("%s did not answer within %s", e.Name, ready)}) } } } +// queued adds where the engine's lease waits to a start that timed out: its +// place in line and the broker's start estimate. The estimate counts only the +// measured starts ahead of it, never the holder's time left, so it is a lower +// bound, and the client is told to retry no sooner than it or 30 s. +func (g *Gateway) queued(e *config.Engine, se *startError) *startError { + v := node.View(g.nodeStatus(), e.GPU) + if v == nil { + return se + } + for _, l := range v.Waiting { + if l.Engine == e.Name && l.Position > 0 && (se.queue == 0 || l.Position < se.queue) { + se.queue, se.estimate = l.Position, time.Duration(l.EstimateS*float64(time.Second)) + } + } + if se.queue > 0 { + se.retry = max(30*time.Second, se.estimate) + se.msg += fmt.Sprintf("; its lease is number %d in line for the card", se.queue) + } + return se +} + func (g *Gateway) ready() time.Duration { if g.Ready > 0 { return g.Ready diff --git a/internal/gateway/gateway_test.go b/internal/gateway/gateway_test.go index 00f4a06..2779394 100644 --- a/internal/gateway/gateway_test.go +++ b/internal/gateway/gateway_test.go @@ -31,6 +31,7 @@ import ( "github.com/T-Crypt/berth/internal/gpu" "github.com/T-Crypt/berth/internal/node" "github.com/T-Crypt/berth/internal/procs" + "github.com/T-Crypt/berth/internal/proto" "github.com/T-Crypt/berth/internal/runner" "github.com/T-Crypt/berth/internal/server" ) @@ -1008,3 +1009,58 @@ func TestComfyPromptResolves(t *testing.T) { t.Fatalf("queue with comfyui down = %d %s, want the 503", resp.StatusCode, body) } } + +// A start that times out while its lease still waits for the card tells the +// client where it stands: its place in line, and a Retry-After of at least +// 30 s. Here a pinned hold keeps the card, so the gateway's run waits. +func TestNotReadyCarriesTheQueue(t *testing.T) { + sock, b := daemon(t) + hold, err := b.Acquire(context.Background(), broker.Request{Engine: "game", Priority: broker.Hold, Pinned: true, Wait: time.Second}, nil) + if err != nil { + t.Fatal(err) + } + defer hold.Release() + g := &Gateway{Socket: sock, Key: testKey, Ready: 500 * time.Millisecond, Log: slogDiscard(), + Engines: []config.Engine{{Name: "e", URL: "http://" + freeAddr(t), Start: []string{os.Args[0]}, Serves: []string{"m"}}}} + gw := httptest.NewServer(g.Handler()) + defer gw.Close() + + resp := call(t, "POST", gw.URL+"/v1/chat/completions", `{"model":"m"}`) + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("X-Berth-Queue-Position") != "1" || + resp.Header.Get("Retry-After") != "30" || !strings.Contains(string(body), "number 1 in line") { + t.Fatalf("status %d, queue %q, Retry-After %q: %s", resp.StatusCode, + resp.Header.Get("X-Berth-Queue-Position"), resp.Header.Get("Retry-After"), body) + } +} + +// The broker's start estimate for the engine's lease reaches the client +// rounded up and marked as a lower bound, and Retry-After waits for the +// larger of it and 30 s. Another engine's place in line is not this one's. +func TestQueuedStartHeaders(t *testing.T) { + for _, tc := range []struct { + estimate float64 + retry, queue, start string + }{ + {44.2, "45", "2", "45; lower-bound"}, + {5, "30", "2", "5; lower-bound"}, + {0, "30", "2", ""}, + } { + sock := statusDaemon(t, proto.Status{ + GPUs: []proto.GPU{{Card: "gpu0", Scheduled: true}}, + Waiting: []proto.Lease{ + {Card: "gpu0", Engine: "other", Position: 1, EstimateS: 900}, + {Card: "gpu0", Engine: "e", Position: 2, EstimateS: tc.estimate}, + }, + }) + g := &Gateway{Socket: sock, Log: slogDiscard()} + w := httptest.NewRecorder() + g.startFailed(w, "e", "m", g.queued(&config.Engine{Name: "e"}, &startError{code: "not_ready", msg: "e did not answer"})) + h := w.Result().Header + got := [...]string{h.Get("Retry-After"), h.Get("X-Berth-Queue-Position"), h.Get("X-Berth-Start-Estimate")} + if got != [...]string{tc.retry, tc.queue, tc.start} { + t.Errorf("estimate %v s: Retry-After, queue, start = %q", tc.estimate, got) + } + } +}