Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ has a matching GitHub release whose body comes from its section below.

### Fixed

- The gateway no longer sends a request to an engine whose lease is draining, parking or releasing. It holds the
request until the lease is active again, or gone and the engine is started anew. Before, Strata took a request
in the seconds it spent stopping and ended it with 0 tokens.
- A minimum turn (`--min-turn`) now protects only a lease whose engine has served a request. An engine started by
a health probe or a warm-up, read up and idle with nothing served, no longer keeps the card for its full turn
while a real request waits.
Expand Down
15 changes: 15 additions & 0 deletions internal/fit/fit.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,21 @@ func Parked(st *proto.Status, name string) bool {
return false
}

// Leaving is whether name's lease is giving the card back: draining, parking
// or releasing. The engine may still answer, but a request sent to it now
// lands on an engine that is about to stop.
func Leaving(st *proto.Status, name string) bool {
if st == nil {
return false
}
for _, l := range st.Leases {
if l.Engine == name && (l.State == "draining" || l.State == "parking" || l.State == "releasing") {
return true
}
}
return false
}

// CardFree is whether nothing holds, uses or waits for the card and the card
// serves this machine. A parked lease holds it: a batch job must not evict an
// engine someone parked in RAM to come back to. A stowaway or a tracked app
Expand Down
111 changes: 111 additions & 0 deletions internal/gateway/draining_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
// Copyright 2026 Trevin Tindall (T-Crypt)
// SPDX-License-Identifier: Apache-2.0

//go:build unix

package gateway

import (
"fmt"
"io"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"

"github.com/T-Crypt/berth/internal/config"
"github.com/T-Crypt/berth/internal/proto"
"github.com/T-Crypt/berth/internal/server"
)

// liveStatusDaemon is a berthd whose status the test changes while the
// gateway reads it.
func liveStatusDaemon(t *testing.T, st *atomic.Pointer[proto.Status]) string {
t.Helper()
sock := filepath.Join(t.TempDir(), "berth.sock")
ln, err := server.Listen(sock)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { ln.Close() })
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
conn := proto.NewConn(c)
if m, err := conn.Recv(); err == nil && m.Op == proto.OpStatus {
_ = conn.Send(proto.Msg{Op: proto.OpStatus, Status: st.Load()})
}
c.Close()
}
}()
return sock
}

// An engine whose lease is draining still answers, and on aphotic Strata took
// a request in the seconds it spent stopping and ended it with 0 tokens. The
// gateway holds a request for a draining engine and sends it on only once
// the lease is active again (or gone, and the engine is started anew).
func TestGatewayHoldsARequestWhileItsEngineDrains(t *testing.T) {
const model = "qwen3.8-flash"
var served atomic.Int32
engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/v1/models":
fmt.Fprintf(w, `{"object":"list","data":[{"id":%q}]}`, model)
case "/v1/chat/completions":
served.Add(1)
fmt.Fprint(w, `{"choices":[{"message":{"content":"from the engine"}}]}`)
default:
http.NotFound(w, r)
}
}))
defer engine.Close()

var st atomic.Pointer[proto.Status]
up := []proto.Engine{{Name: "strata", Type: "strata", URL: engine.URL, Up: true, State: "up"}}
st.Store(&proto.Status{Engines: up, Leases: []proto.Lease{{ID: "strata-58", Engine: "strata", Group: "strata", State: "draining"}}})
g := &Gateway{Socket: liveStatusDaemon(t, &st), Ready: 5 * time.Second, Key: testKey,
Engines: []config.Engine{{Name: "strata", Type: "strata", URL: engine.URL, Serves: []string{model}}}}
gw := httptest.NewServer(g.Handler())
defer gw.Close()

type reply struct {
code int
body string
}
done := make(chan reply, 1)
go func() {
resp := call(t, "POST", gw.URL+"/v1/chat/completions", fmt.Sprintf(`{"model":%q}`, model))
b, _ := io.ReadAll(resp.Body)
resp.Body.Close()
done <- reply{resp.StatusCode, string(b)}
}()

select {
case r := <-done:
t.Fatalf("answered while the engine drained: %d %s", r.code, r.body)
case <-time.After(700 * time.Millisecond):
}
if n := served.Load(); n != 0 {
t.Fatalf("the draining engine got %d request(s)", n)
}

st.Store(&proto.Status{Engines: up, Leases: []proto.Lease{{ID: "strata-58", Engine: "strata", Group: "strata", State: "active"}}})
select {
case r := <-done:
if r.code != http.StatusOK || !strings.Contains(r.body, "from the engine") {
t.Fatalf("after the drain: %d %s", r.code, r.body)
}
case <-time.After(5 * time.Second):
t.Fatal("the held request never went on after the lease was active again")
}
if n := served.Load(); n != 1 {
t.Fatalf("the engine served %d request(s), want 1", n)
}
}
15 changes: 14 additions & 1 deletion internal/gateway/gateway.go
Original file line number Diff line number Diff line change
Expand Up @@ -463,7 +463,7 @@ func (g *Gateway) resolve(ctx context.Context, model, card string) (target, bool
for _, e := range g.live() {
// A parked engine answers its list with the weights in RAM; the
// resume, not the list, says when it may be proxied.
if e.Up && g.engineOn(st, e.Name, card) && !fit.Parked(st, e.Name) && slices.Contains(g.listed(ctx, e.URL), model) {
if e.Up && g.engineOn(st, e.Name, card) && !fit.Parked(st, e.Name) && !fit.Leaving(st, e.Name) && slices.Contains(g.listed(ctx, e.URL), model) {
return target{url: e.URL, live: true}, true
}
}
Expand Down Expand Up @@ -930,13 +930,26 @@ func (g *Gateway) comfyProxyTo(t target, w http.ResponseWriter, r *http.Request)
// ensure starts e under a lease unless this gateway already has, then waits
// until it answers. A start the broker refuses comes back as the reason.
// switched marks a person's switch, which wins a juggle.
// leavingEvery is how often a request held for a draining engine reads the
// lease again.
var leavingEvery = 250 * time.Millisecond

func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) error {
// ComfyUI answers no /v1/models; its queue endpoint is the liveness
// probe instead, so a cold start waits for the real server.
probe := g.answers
if e.Type == "comfyui" {
probe = g.comfyAnswers
}
// A draining engine answers until it stops: the request waits for the
// lease to settle, active again or gone and started anew.
for fit.Leaving(g.nodeStatus(), e.Name) {
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(leavingEvery):
}
}
// A parked engine answers while the weights sit in RAM: the resume
// goes before any probe, or the request is proxied to RAM.
if parks(e) {
Expand Down
Loading