Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ has a matching GitHub release whose body comes from its section below.
- The harbor's SWITCH tab has a QUEUE panel. It lists the waiters in grant order, and a paired device gets FIRST, a
priority picker and DROP on each one. These buttons go from the harbor straight to berthd, never through the
gateway, because agents hold the gateway's API key and a queue change is the operator's to make.
- A gateway request whose start times out (`not_ready`) while its lease is still in line for the card says where
it stands. The 503 carries `X-Berth-Queue-Position` and, when berthd has measured the starts ahead,
`X-Berth-Start-Estimate: N; lower-bound`. `Retry-After` is the larger of the estimate and 30 s. The estimate
leaves out the holder's time left, so it is a minimum, not a promise.
- `berth wire llama-swap` names each llama-swap group where llama-swap swaps two or more wired models itself.
There llama-swap unloads the old model before Berth sees the request, so `min_turn`, the engine ranking and the
juggle guard never apply between them. The JSON carries the groups as `swapping`. The llama-swap section of
Expand Down
7 changes: 7 additions & 0 deletions docs/gateway.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,13 @@
every retry: for 15 s, doubling to 5 min, the gateway answers at once with the same cause,
and `Retry-After` says when the next start is allowed. A start that answers clears it. A
model listed under `serves` by two engines fails `berth config check`.
- **Queue position.** A `not_ready` start whose lease is still in line for the card says where it
stands. `X-Berth-Queue-Position` is its place in line, 1 for next. `X-Berth-Start-Estimate:
45; lower-bound` is the seconds berthd expects the start to take once the card frees. It counts
only the measured starts ahead in the queue and not the holder's time left, so treat it as a
minimum. It is missing when nothing was measured. `Retry-After` is then the larger of that
estimate and 30 s. A streaming request gets the same error as an SSE event, without these
headers.
- **API key.** Bearer token on every route; `berth key` prints it (created on first use, mode
600). Loopback only; browser origins and non-loopback Host names are refused.

Expand Down
34 changes: 33 additions & 1 deletion internal/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -580,6 +580,11 @@ type startError struct {
code string // no_start, outranked, drained_while_starting, no_berth, start_failed, engine_oom, not_ready, no_headroom
msg string
retry time.Duration // when set, the client is told to try again after this (Retry-After)
// queue is the engine's lease's place in line when the start timed out
// waiting for the card, 0 when it was not waiting; estimate is the
// broker's lower-bound start estimate for it, 0 for unknown.
queue int
estimate time.Duration
}

func (e *startError) Error() string { return e.msg }
Expand Down Expand Up @@ -791,6 +796,12 @@ func (g *Gateway) startFailed(w http.ResponseWriter, engine, what string, err er
if se != nil && se.retry > 0 {
w.Header().Set("Retry-After", strconv.Itoa(int((se.retry+time.Second-1)/time.Second)))
}
if se != nil && se.queue > 0 {
w.Header().Set("X-Berth-Queue-Position", strconv.Itoa(se.queue))
if se.estimate > 0 {
w.Header().Set("X-Berth-Start-Estimate", fmt.Sprintf("%d; lower-bound", int((se.estimate+time.Second-1)/time.Second)))
}
}
apiErrorCode(w, http.StatusServiceUnavailable, code, err.Error())
}

Expand Down Expand Up @@ -1011,11 +1022,32 @@ func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) e
case <-time.After(250 * time.Millisecond):
}
if time.Now().After(deadline) {
return &startError{code: "not_ready", msg: fmt.Sprintf("%s did not answer within %s", e.Name, ready)}
return g.queued(e, &startError{code: "not_ready", msg: fmt.Sprintf("%s did not answer within %s", e.Name, ready)})
}
}
}

// queued adds where the engine's lease waits to a start that timed out: its
// place in line and the broker's start estimate. The estimate counts only the
// measured starts ahead of it, never the holder's time left, so it is a lower
// bound, and the client is told to retry no sooner than it or 30 s.
func (g *Gateway) queued(e *config.Engine, se *startError) *startError {
v := node.View(g.nodeStatus(), e.GPU)
if v == nil {
return se
}
for _, l := range v.Waiting {
if l.Engine == e.Name && l.Position > 0 && (se.queue == 0 || l.Position < se.queue) {
se.queue, se.estimate = l.Position, time.Duration(l.EstimateS*float64(time.Second))
}
}
if se.queue > 0 {
se.retry = max(30*time.Second, se.estimate)
se.msg += fmt.Sprintf("; its lease is number %d in line for the card", se.queue)
}
return se
}

func (g *Gateway) ready() time.Duration {
if g.Ready > 0 {
return g.Ready
Expand Down
56 changes: 56 additions & 0 deletions internal/gateway/gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/T-Crypt/berth/internal/gpu"
"github.com/T-Crypt/berth/internal/node"
"github.com/T-Crypt/berth/internal/procs"
"github.com/T-Crypt/berth/internal/proto"
"github.com/T-Crypt/berth/internal/runner"
"github.com/T-Crypt/berth/internal/server"
)
Expand Down Expand Up @@ -1008,3 +1009,58 @@ func TestComfyPromptResolves(t *testing.T) {
t.Fatalf("queue with comfyui down = %d %s, want the 503", resp.StatusCode, body)
}
}

// A start that times out while its lease still waits for the card tells the
// client where it stands: its place in line, and a Retry-After of at least
// 30 s. Here a pinned hold keeps the card, so the gateway's run waits.
func TestNotReadyCarriesTheQueue(t *testing.T) {
sock, b := daemon(t)
hold, err := b.Acquire(context.Background(), broker.Request{Engine: "game", Priority: broker.Hold, Pinned: true, Wait: time.Second}, nil)
if err != nil {
t.Fatal(err)
}
defer hold.Release()
g := &Gateway{Socket: sock, Key: testKey, Ready: 500 * time.Millisecond, Log: slogDiscard(),
Engines: []config.Engine{{Name: "e", URL: "http://" + freeAddr(t), Start: []string{os.Args[0]}, Serves: []string{"m"}}}}
gw := httptest.NewServer(g.Handler())
defer gw.Close()

resp := call(t, "POST", gw.URL+"/v1/chat/completions", `{"model":"m"}`)
body, _ := io.ReadAll(resp.Body)
resp.Body.Close()
if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("X-Berth-Queue-Position") != "1" ||
resp.Header.Get("Retry-After") != "30" || !strings.Contains(string(body), "number 1 in line") {
t.Fatalf("status %d, queue %q, Retry-After %q: %s", resp.StatusCode,
resp.Header.Get("X-Berth-Queue-Position"), resp.Header.Get("Retry-After"), body)
}
}

// The broker's start estimate for the engine's lease reaches the client
// rounded up and marked as a lower bound, and Retry-After waits for the
// larger of it and 30 s. Another engine's place in line is not this one's.
func TestQueuedStartHeaders(t *testing.T) {
for _, tc := range []struct {
estimate float64
retry, queue, start string
}{
{44.2, "45", "2", "45; lower-bound"},
{5, "30", "2", "5; lower-bound"},
{0, "30", "2", ""},
} {
sock := statusDaemon(t, proto.Status{
GPUs: []proto.GPU{{Card: "gpu0", Scheduled: true}},
Waiting: []proto.Lease{
{Card: "gpu0", Engine: "other", Position: 1, EstimateS: 900},
{Card: "gpu0", Engine: "e", Position: 2, EstimateS: tc.estimate},
},
})
g := &Gateway{Socket: sock, Log: slogDiscard()}
w := httptest.NewRecorder()
g.startFailed(w, "e", "m", g.queued(&config.Engine{Name: "e"}, &startError{code: "not_ready", msg: "e did not answer"}))
h := w.Result().Header
got := [...]string{h.Get("Retry-After"), h.Get("X-Berth-Queue-Position"), h.Get("X-Berth-Start-Estimate")}
if got != [...]string{tc.retry, tc.queue, tc.start} {
t.Errorf("estimate %v s: Retry-After, queue, start = %q", tc.estimate, got)
}
}
}
Loading