diff --git a/CHANGELOG.md b/CHANGELOG.md index fd52e18..b702c5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ has a matching GitHub release whose body comes from its section below. ### Fixed +- The gateway no longer sends a request to an engine whose lease is draining, parking or releasing. It holds the + request until the lease is active again, or gone and the engine is started anew. Before, Strata took a request + in the seconds it spent stopping and ended it with 0 tokens. - A minimum turn (`--min-turn`) now protects only a lease whose engine has served a request. An engine started by a health probe or a warm-up, read up and idle with nothing served, no longer keeps the card for its full turn while a real request waits. diff --git a/internal/fit/fit.go b/internal/fit/fit.go index dc31a12..aebac9e 100644 --- a/internal/fit/fit.go +++ b/internal/fit/fit.go @@ -134,6 +134,21 @@ func Parked(st *proto.Status, name string) bool { return false } +// Leaving is whether name's lease is giving the card back: draining, parking +// or releasing. The engine may still answer, but a request sent to it now +// lands on an engine that is about to stop. +func Leaving(st *proto.Status, name string) bool { + if st == nil { + return false + } + for _, l := range st.Leases { + if l.Engine == name && (l.State == "draining" || l.State == "parking" || l.State == "releasing") { + return true + } + } + return false +} + // CardFree is whether nothing holds, uses or waits for the card and the card // serves this machine. A parked lease holds it: a batch job must not evict an // engine someone parked in RAM to come back to. A stowaway or a tracked app diff --git a/internal/gateway/draining_test.go b/internal/gateway/draining_test.go new file mode 100644 index 0000000..fe7e2a0 --- /dev/null +++ b/internal/gateway/draining_test.go @@ -0,0 +1,111 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package gateway + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/config" + "github.com/T-Crypt/berth/internal/proto" + "github.com/T-Crypt/berth/internal/server" +) + +// liveStatusDaemon is a berthd whose status the test changes while the +// gateway reads it. +func liveStatusDaemon(t *testing.T, st *atomic.Pointer[proto.Status]) string { + t.Helper() + sock := filepath.Join(t.TempDir(), "berth.sock") + ln, err := server.Listen(sock) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + conn := proto.NewConn(c) + if m, err := conn.Recv(); err == nil && m.Op == proto.OpStatus { + _ = conn.Send(proto.Msg{Op: proto.OpStatus, Status: st.Load()}) + } + c.Close() + } + }() + return sock +} + +// An engine whose lease is draining still answers, and on aphotic Strata took +// a request in the seconds it spent stopping and ended it with 0 tokens. The +// gateway holds a request for a draining engine and sends it on only once +// the lease is active again (or gone, and the engine is started anew). +func TestGatewayHoldsARequestWhileItsEngineDrains(t *testing.T) { + const model = "qwen3.8-flash" + var served atomic.Int32 + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/models": + fmt.Fprintf(w, `{"object":"list","data":[{"id":%q}]}`, model) + case "/v1/chat/completions": + served.Add(1) + fmt.Fprint(w, `{"choices":[{"message":{"content":"from the engine"}}]}`) + default: + http.NotFound(w, r) + } + })) + defer engine.Close() + + var st atomic.Pointer[proto.Status] + up := []proto.Engine{{Name: "strata", Type: "strata", URL: engine.URL, Up: true, State: "up"}} + st.Store(&proto.Status{Engines: up, Leases: []proto.Lease{{ID: "strata-58", Engine: "strata", Group: "strata", State: "draining"}}}) + g := &Gateway{Socket: liveStatusDaemon(t, &st), Ready: 5 * time.Second, Key: testKey, + Engines: []config.Engine{{Name: "strata", Type: "strata", URL: engine.URL, Serves: []string{model}}}} + gw := httptest.NewServer(g.Handler()) + defer gw.Close() + + type reply struct { + code int + body string + } + done := make(chan reply, 1) + go func() { + resp := call(t, "POST", gw.URL+"/v1/chat/completions", fmt.Sprintf(`{"model":%q}`, model)) + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + done <- reply{resp.StatusCode, string(b)} + }() + + select { + case r := <-done: + t.Fatalf("answered while the engine drained: %d %s", r.code, r.body) + case <-time.After(700 * time.Millisecond): + } + if n := served.Load(); n != 0 { + t.Fatalf("the draining engine got %d request(s)", n) + } + + st.Store(&proto.Status{Engines: up, Leases: []proto.Lease{{ID: "strata-58", Engine: "strata", Group: "strata", State: "active"}}}) + select { + case r := <-done: + if r.code != http.StatusOK || !strings.Contains(r.body, "from the engine") { + t.Fatalf("after the drain: %d %s", r.code, r.body) + } + case <-time.After(5 * time.Second): + t.Fatal("the held request never went on after the lease was active again") + } + if n := served.Load(); n != 1 { + t.Fatalf("the engine served %d request(s), want 1", n) + } +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index d187d24..8f46b91 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -463,7 +463,7 @@ func (g *Gateway) resolve(ctx context.Context, model, card string) (target, bool for _, e := range g.live() { // A parked engine answers its list with the weights in RAM; the // resume, not the list, says when it may be proxied. - if e.Up && g.engineOn(st, e.Name, card) && !fit.Parked(st, e.Name) && slices.Contains(g.listed(ctx, e.URL), model) { + if e.Up && g.engineOn(st, e.Name, card) && !fit.Parked(st, e.Name) && !fit.Leaving(st, e.Name) && slices.Contains(g.listed(ctx, e.URL), model) { return target{url: e.URL, live: true}, true } } @@ -930,6 +930,10 @@ func (g *Gateway) comfyProxyTo(t target, w http.ResponseWriter, r *http.Request) // ensure starts e under a lease unless this gateway already has, then waits // until it answers. A start the broker refuses comes back as the reason. // switched marks a person's switch, which wins a juggle. +// leavingEvery is how often a request held for a draining engine reads the +// lease again. +var leavingEvery = 250 * time.Millisecond + func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) error { // ComfyUI answers no /v1/models; its queue endpoint is the liveness // probe instead, so a cold start waits for the real server. @@ -937,6 +941,15 @@ func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) e if e.Type == "comfyui" { probe = g.comfyAnswers } + // A draining engine answers until it stops: the request waits for the + // lease to settle, active again or gone and started anew. + for fit.Leaving(g.nodeStatus(), e.Name) { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(leavingEvery): + } + } // A parked engine answers while the weights sit in RAM: the resume // goes before any probe, or the request is proxied to RAM. if parks(e) {