From 2960433aef3bdd390e52ed4e5e077c9bd30df17d Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:21:04 -0700 Subject: [PATCH 1/2] test(gateway): hold a request while its engine drains --- internal/gateway/draining_test.go | 111 ++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 internal/gateway/draining_test.go diff --git a/internal/gateway/draining_test.go b/internal/gateway/draining_test.go new file mode 100644 index 0000000..fe7e2a0 --- /dev/null +++ b/internal/gateway/draining_test.go @@ -0,0 +1,111 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package gateway + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/config" + "github.com/T-Crypt/berth/internal/proto" + "github.com/T-Crypt/berth/internal/server" +) + +// liveStatusDaemon is a berthd whose status the test changes while the +// gateway reads it. +func liveStatusDaemon(t *testing.T, st *atomic.Pointer[proto.Status]) string { + t.Helper() + sock := filepath.Join(t.TempDir(), "berth.sock") + ln, err := server.Listen(sock) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + conn := proto.NewConn(c) + if m, err := conn.Recv(); err == nil && m.Op == proto.OpStatus { + _ = conn.Send(proto.Msg{Op: proto.OpStatus, Status: st.Load()}) + } + c.Close() + } + }() + return sock +} + +// An engine whose lease is draining still answers, and on aphotic Strata took +// a request in the seconds it spent stopping and ended it with 0 tokens. The +// gateway holds a request for a draining engine and sends it on only once +// the lease is active again (or gone, and the engine is started anew). +func TestGatewayHoldsARequestWhileItsEngineDrains(t *testing.T) { + const model = "qwen3.8-flash" + var served atomic.Int32 + engine := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/v1/models": + fmt.Fprintf(w, `{"object":"list","data":[{"id":%q}]}`, model) + case "/v1/chat/completions": + served.Add(1) + fmt.Fprint(w, `{"choices":[{"message":{"content":"from the engine"}}]}`) + default: + http.NotFound(w, r) + } + })) + defer engine.Close() + + var st atomic.Pointer[proto.Status] + up := []proto.Engine{{Name: "strata", Type: "strata", URL: engine.URL, Up: true, State: "up"}} + st.Store(&proto.Status{Engines: up, Leases: []proto.Lease{{ID: "strata-58", Engine: "strata", Group: "strata", State: "draining"}}}) + g := &Gateway{Socket: liveStatusDaemon(t, &st), Ready: 5 * time.Second, Key: testKey, + Engines: []config.Engine{{Name: "strata", Type: "strata", URL: engine.URL, Serves: []string{model}}}} + gw := httptest.NewServer(g.Handler()) + defer gw.Close() + + type reply struct { + code int + body string + } + done := make(chan reply, 1) + go func() { + resp := call(t, "POST", gw.URL+"/v1/chat/completions", fmt.Sprintf(`{"model":%q}`, model)) + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + done <- reply{resp.StatusCode, string(b)} + }() + + select { + case r := <-done: + t.Fatalf("answered while the engine drained: %d %s", r.code, r.body) + case <-time.After(700 * time.Millisecond): + } + if n := served.Load(); n != 0 { + t.Fatalf("the draining engine got %d request(s)", n) + } + + st.Store(&proto.Status{Engines: up, Leases: []proto.Lease{{ID: "strata-58", Engine: "strata", Group: "strata", State: "active"}}}) + select { + case r := <-done: + if r.code != http.StatusOK || !strings.Contains(r.body, "from the engine") { + t.Fatalf("after the drain: %d %s", r.code, r.body) + } + case <-time.After(5 * time.Second): + t.Fatal("the held request never went on after the lease was active again") + } + if n := served.Load(); n != 1 { + t.Fatalf("the engine served %d request(s), want 1", n) + } +} From 31434d78028056777050154f0ffe54ec5c9bfa0c Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Sun, 15 Nov 2026 14:21:04 -0700 Subject: [PATCH 2/2] fix(gateway): hold a request while its engine's lease drains A draining engine still answers its probe, so the gateway proxied the next request to it. On aphotic at 12:45:22 on 2026-11-15, the runner read Strata idle, began to unload it, and Strata took the next agent turn in the two seconds it spent stopping and ended it with 0 tokens (finish=length). resolve no longer treats an engine as live while its lease is draining, parking or releasing, and ensure waits for that lease to settle before it probes. Active again, the request goes on to the engine. Gone, the engine stops answering and the gateway starts it anew, waiting for the card as any start does. --- CHANGELOG.md | 3 +++ internal/fit/fit.go | 15 +++++++++++++++ internal/gateway/gateway.go | 15 ++++++++++++++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fd52e18..b702c5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ has a matching GitHub release whose body comes from its section below. ### Fixed +- The gateway no longer sends a request to an engine whose lease is draining, parking or releasing. It holds the + request until the lease is active again, or gone and the engine is started anew. Before, Strata took a request + in the seconds it spent stopping and ended it with 0 tokens. - A minimum turn (`--min-turn`) now protects only a lease whose engine has served a request. An engine started by a health probe or a warm-up, read up and idle with nothing served, no longer keeps the card for its full turn while a real request waits. diff --git a/internal/fit/fit.go b/internal/fit/fit.go index dc31a12..aebac9e 100644 --- a/internal/fit/fit.go +++ b/internal/fit/fit.go @@ -134,6 +134,21 @@ func Parked(st *proto.Status, name string) bool { return false } +// Leaving is whether name's lease is giving the card back: draining, parking +// or releasing. The engine may still answer, but a request sent to it now +// lands on an engine that is about to stop. +func Leaving(st *proto.Status, name string) bool { + if st == nil { + return false + } + for _, l := range st.Leases { + if l.Engine == name && (l.State == "draining" || l.State == "parking" || l.State == "releasing") { + return true + } + } + return false +} + // CardFree is whether nothing holds, uses or waits for the card and the card // serves this machine. A parked lease holds it: a batch job must not evict an // engine someone parked in RAM to come back to. A stowaway or a tracked app diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go index d187d24..8f46b91 100644 --- a/internal/gateway/gateway.go +++ b/internal/gateway/gateway.go @@ -463,7 +463,7 @@ func (g *Gateway) resolve(ctx context.Context, model, card string) (target, bool for _, e := range g.live() { // A parked engine answers its list with the weights in RAM; the // resume, not the list, says when it may be proxied. - if e.Up && g.engineOn(st, e.Name, card) && !fit.Parked(st, e.Name) && slices.Contains(g.listed(ctx, e.URL), model) { + if e.Up && g.engineOn(st, e.Name, card) && !fit.Parked(st, e.Name) && !fit.Leaving(st, e.Name) && slices.Contains(g.listed(ctx, e.URL), model) { return target{url: e.URL, live: true}, true } } @@ -930,6 +930,10 @@ func (g *Gateway) comfyProxyTo(t target, w http.ResponseWriter, r *http.Request) // ensure starts e under a lease unless this gateway already has, then waits // until it answers. A start the broker refuses comes back as the reason. // switched marks a person's switch, which wins a juggle. +// leavingEvery is how often a request held for a draining engine reads the +// lease again. +var leavingEvery = 250 * time.Millisecond + func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) error { // ComfyUI answers no /v1/models; its queue endpoint is the liveness // probe instead, so a cold start waits for the real server. @@ -937,6 +941,15 @@ func (g *Gateway) ensure(ctx context.Context, e *config.Engine, switched bool) e if e.Type == "comfyui" { probe = g.comfyAnswers } + // A draining engine answers until it stops: the request waits for the + // lease to settle, active again or gone and started anew. + for fit.Leaving(g.nodeStatus(), e.Name) { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(leavingEvery): + } + } // A parked engine answers while the weights sit in RAM: the resume // goes before any probe, or the request is proxied to RAM. if parks(e) {