Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@ has a matching GitHub release whose body comes from its section below.
moves one to the front of its priority class, `berth queue prio ID batch|interactive|foreground` moves it to
another class, and `berth queue drop ID` refuses it now. Holds and trainings cannot be changed this way. Each
change is a `lease.queue_override` event.
- The harbor's SWITCH tab has a QUEUE panel. It lists the waiters in grant order, and a paired device gets FIRST, a
priority picker and DROP on each one. These buttons go from the harbor straight to berthd, never through the
gateway, because agents hold the gateway's API key and a queue change is the operator's to make.
- `berth wire llama-swap` names each llama-swap group where llama-swap swaps two or more wired models itself.
There llama-swap unloads the old model before Berth sees the request, so `min_turn`, the engine ranking and the
juggle guard never apply between them. The JSON carries the groups as `swapping`. The llama-swap section of
Expand Down
3 changes: 2 additions & 1 deletion internal/harbor/harbor.go
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,7 @@ func (h *Harbor) Handler() http.Handler {
mux.HandleFunc("POST /v1/board/switch", h.boardAction("switch"))
mux.HandleFunc("POST /v1/board/stop", h.boardAction("stop"))
mux.HandleFunc("POST /v1/board/load", h.boardAction("load"))
mux.HandleFunc("POST /v1/board/queue", h.boardQueue)
mux.HandleFunc("POST /v1/pair/code", h.pairCode)
return h.guard(mux)
}
Expand Down Expand Up @@ -190,7 +191,7 @@ func (h *Harbor) guard(next http.Handler) http.Handler {
}

// writePaths are the only paths the harbor takes a POST on.
var writePaths = map[string]bool{"/pair": true, "/v1/board/switch": true, "/v1/board/stop": true, "/v1/board/load": true, "/v1/pair/code": true}
var writePaths = map[string]bool{"/pair": true, "/v1/board/switch": true, "/v1/board/stop": true, "/v1/board/load": true, "/v1/board/queue": true, "/v1/pair/code": true}

// mode tells the page which guard this harbor runs under. It is a harbor
// fact, stable for the process lifetime, so the page fetches it once: unlike
Expand Down
86 changes: 86 additions & 0 deletions internal/harbor/queue_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright 2026 Trevin Tindall (T-Crypt)
// SPDX-License-Identifier: Apache-2.0

package harbor

import (
"context"
"encoding/json"
"strings"
"testing"
"time"

"github.com/T-Crypt/berth/internal/broker"
)

// A paired device reorders and drops waiters through the harbor, straight to
// berthd; an unpaired one is refused and the queue is unchanged; nothing goes
// through the gateway, whose key agents hold.
func TestBoardQueue(t *testing.T) {
base, gw, token, b := brokerHarbor(t, testKey)
if _, err := b.Acquire(context.Background(), broker.Request{Engine: "game", Priority: broker.Hold, Pinned: true}, nil); err != nil {
t.Fatal(err)
}
refused := make(chan error, 2)
for _, e := range []string{"first", "second"} {
go func() {
_, err := b.Acquire(context.Background(), broker.Request{Engine: e, Priority: broker.Interactive, Wait: time.Minute}, nil)
refused <- err
}()
time.Sleep(20 * time.Millisecond)
}
order := func() (ids, names []string) {
for _, l := range b.Status().Waiting {
ids, names = append(ids, l.ID), append(names, l.Engine)
}
return
}
ids, names := order()
if strings.Join(names, ",") != "first,second" {
t.Fatalf("queue = %v", names)
}
paired := map[string]string{"Cookie": cookieName + "=" + token, "X-Berth": "1", "Sec-Fetch-Site": "same-origin"}
unpaired := map[string]string{"X-Berth": "1", "Sec-Fetch-Site": "same-origin"}

code, body := call(t, "POST", base+"/v1/board/queue", `{"lease":"`+ids[1]+`","action":"bump"}`, unpaired)
if code != 401 {
t.Fatalf("unpaired bump = %d %s, want 401", code, body)
}
if _, names = order(); strings.Join(names, ",") != "first,second" {
t.Fatalf("an unpaired bump moved the queue: %v", names)
}

code, body = call(t, "POST", base+"/v1/board/queue", `{"lease":"`+ids[1]+`","action":"bump"}`, paired)
var got struct{ Lease, Change string }
if code != 200 || json.Unmarshal([]byte(body), &got) != nil || got.Lease != ids[1] ||
got.Change != "moved to the front of its class by the operator" {
t.Fatalf("bump = %d %s", code, body)
}
if _, names = order(); strings.Join(names, ",") != "second,first" {
t.Fatalf("after bump: %v", names)
}

for _, bad := range []string{`{"lease":"x","action":"jump"}`, `{"lease":"x","action":"prio"}`, `{"action":"drop"}`} {
if code, body = call(t, "POST", base+"/v1/board/queue", bad, paired); code != 400 {
t.Fatalf("%s = %d %s, want 400", bad, code, body)
}
}
if code, body = call(t, "POST", base+"/v1/board/queue", `{"lease":"`+ids[0]+`","action":"prio","priority":"hold"}`, paired); code != 409 {
t.Fatalf("prio hold = %d %s, want 409", code, body)
}

if code, body = call(t, "POST", base+"/v1/board/queue", `{"lease":"`+ids[0]+`","action":"drop"}`, paired); code != 200 {
t.Fatalf("drop = %d %s", code, body)
}
select {
case err := <-refused:
if err == nil || !strings.Contains(err.Error(), "dropped from the queue by the operator") {
t.Fatalf("dropped waiter: %v", err)
}
case <-time.After(time.Second):
t.Fatal("dropped waiter still waiting")
}
if gw.last() != "" {
t.Fatalf("a queue action reached the gateway: %q", gw.last())
}
}
31 changes: 31 additions & 0 deletions internal/harbor/switchboard.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,37 @@ func (h *Harbor) boardAction(action string) http.HandlerFunc {
}
}

// boardQueue is POST /v1/board/queue: a paired device bumps, re-ranks or
// drops one waiter. It goes to berthd's socket, not the gateway: the
// gateway's API key is held by agents, and a queue change is the
// operator's, so only a paired device makes one.
func (h *Harbor) boardQueue(w http.ResponseWriter, r *http.Request) {
if !h.writeOK(w, r) {
return
}
var req struct {
Lease string `json:"lease"`
Action string `json:"action"`
Priority string `json:"priority,omitempty"`
}
body, err := io.ReadAll(io.LimitReader(r.Body, 1<<16))
if err != nil || json.Unmarshal(body, &req) != nil || req.Lease == "" ||
(req.Action != "bump" && req.Action != "prio" && req.Action != "drop") || (req.Action == "prio") != (req.Priority != "") {
apiError(w, http.StatusBadRequest, "bad_body", `the body must be JSON like {"lease":"ID","action":"bump|drop"} or {"lease":"ID","action":"prio","priority":"batch|interactive|foreground"}`)
return
}
m, err := h.ask(proto.Msg{Op: proto.OpQueue, Lease: req.Lease, Action: req.Action, Priority: req.Priority})
switch {
case err != nil:
apiError(w, http.StatusServiceUnavailable, "berthd_down", "berthd is not answering; the queue is unchanged")
case m.Op == proto.OpRefused:
apiError(w, http.StatusConflict, "refused", m.Reason)
default:
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]string{"lease": m.Lease, "change": m.Reason})
}
}

// toGateway sends one request to the gateway with the node's key and hands
// its status and body back to the page.
func (h *Harbor) toGateway(w http.ResponseWriter, r *http.Request, method, path string, body []byte) {
Expand Down
12 changes: 10 additions & 2 deletions internal/harbor/switchboard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,19 @@ func (f *fakeGateway) last() string {
// switchHarbor is a harbor in front of a real berthd that owns a pairing
// file and a fake gateway, with one paired device's token.
func switchHarbor(t *testing.T, key string) (base string, gw *fakeGateway, token string) {
t.Helper()
base, gw, token, _ = brokerHarbor(t, key)
return base, gw, token
}

// brokerHarbor is switchHarbor with the broker behind it, for a test that
// puts leases on the card.
func brokerHarbor(t *testing.T, key string) (base string, gw *fakeGateway, token string, b *broker.Broker) {
t.Helper()
dir := t.TempDir()
sock := filepath.Join(dir, "berth.sock")
log := slog.New(slog.NewTextHandler(io.Discard, nil))
b := broker.New(broker.Config{Poll: 20 * time.Millisecond, Log: log}, idleCard{}, procs.System{})
b = broker.New(broker.Config{Poll: 20 * time.Millisecond, Log: log}, idleCard{}, procs.System{})
ln, err := server.Listen(sock)
if err != nil {
t.Fatal(err)
Expand All @@ -88,7 +96,7 @@ func switchHarbor(t *testing.T, key string) (base string, gw *fakeGateway, token
t.Cleanup(gsrv.Close)
srv := httptest.NewServer((&Harbor{Socket: sock, Log: log, Gateway: gsrv.URL, GatewayKey: key}).Handler())
t.Cleanup(srv.Close)
return srv.URL, gw, token
return srv.URL, gw, token, b
}

func call(t *testing.T, method, url, body string, hdr map[string]string) (int, string) {
Expand Down
10 changes: 10 additions & 0 deletions internal/harbor/web/harbor.css
Original file line number Diff line number Diff line change
Expand Up @@ -1048,6 +1048,16 @@ body.ready #panel-engines > :nth-child(n+2) { animation-delay: 60ms; }
.sw-btn { min-height: var(--touch); min-width: 88px; }
.sw-btn:disabled { opacity: 0.5; cursor: progress; }
.sw-stop { color: var(--amber); border-color: var(--amber-dim); }
/* the queue's priority picker: a select drawn like .btn, not the browser's default */
.sw-prio {
min-height: var(--touch); padding: 6px 14px;
background: var(--panel-2); color: var(--ink);
border: 1px solid var(--edge); border-radius: var(--r2);
font-family: var(--mono); font-size: var(--fs-small); letter-spacing: 1px;
cursor: pointer;
}
.sw-prio:hover, .sw-prio:focus-visible { color: var(--ink-strong); border-color: var(--dim-2); }
.sw-prio:disabled { opacity: 0.5; cursor: progress; }
.sw-error { color: var(--magenta); }
.sw-detail { color: var(--amber); }
.sw-paired { font-family: var(--mono); font-size: var(--fs-small); color: var(--dim); display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: var(--s2); margin-bottom: var(--s3); }
Expand Down
2 changes: 1 addition & 1 deletion internal/harbor/web/harbor.js
Original file line number Diff line number Diff line change
Expand Up @@ -517,7 +517,7 @@ const ENGINES_SECTIONS = [
["sec-throughput", "Throughput"], ["sec-ledger", "Ledger"], ["sec-lb", "Leaderboard"],
["sec-recent", "Recent"], ["sec-residency", "Residency"], ["sec-storage", "Storage"],
];
const SWITCH_SECTIONS = [["sec-switch-cards", "Cards"], ["sec-switch-engines", "Engines"]];
const SWITCH_SECTIONS = [["sec-switch-cards", "Cards"], ["sec-switch-queue", "Queue"], ["sec-switch-engines", "Engines"]];
const ABOUT_SECTIONS = [
["about-top", "Berth"], ["about-cards", "Cards"], ["about-modes", "Modes"], ["about-nav", "Getting around"],
["about-keys", "Keyboard"], ["about-states", "Status words"], ["about-kpis", "KPIs"], ["about-counters", "Counters"],
Expand Down
7 changes: 7 additions & 0 deletions internal/harbor/web/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,13 @@ <h2 class="panel-title">CARDS</h2>
</div>
<div class="squatter-list" id="sw-cards"></div>
</section>
<section class="panel" id="sec-switch-queue" aria-label="the waiters in grant order">
<div class="panel-head">
<h2 class="panel-title">QUEUE</h2>
<span class="panel-meta">grant order: priority, then arrival</span>
</div>
<div class="squatter-list" id="sw-queue"></div>
</section>
<section class="panel" id="sec-switch-engines" aria-label="switch the card to an engine">
<div class="panel-head">
<h2 class="panel-title">ENGINES</h2>
Expand Down
85 changes: 85 additions & 0 deletions internal/harbor/web/switch.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,31 @@ export function initSwitch({ demo }) {
const cardsRoot = $("sw-cards");
const pickRoot = $("sw-picker");
const enginesRoot = $("sw-engines");
const queueRoot = $("sw-queue");
const noteRoot = $("sw-note");
const meta = $("sw-meta");
const pairDlg = $("pair-dlg");
let board = null;
let queue = null; // the waiters in grant order, from the node's status
let paired = demo;
let timer = 0;
let pick = localStorage.getItem(PICK_KEY) || "";
const busy = new Set();
const errors = new Map(); // engine -> the last action's error, until the next read
const open = new Set(); // engines whose model list is unfolded

async function readQueue() {
try {
const res = await fetch(demo ? "demo.json" : "/v1/status", { headers: { accept: "application/json" } });
const j = await res.json();
queue = res.ok ? (j.waiting || []) : null;
} catch (e) {
queue = null;
}
}

async function read() {
await readQueue();
try {
const res = await fetch(demo ? "demo-board.json" : "/v1/board", { headers: { accept: "application/json" } });
const j = await res.json();
Expand Down Expand Up @@ -66,6 +79,77 @@ export function initSwitch({ demo }) {
await read();
}

// queueAct bumps, re-ranks or drops one waiter. It goes to berthd through
// the harbor only (never the gateway), and only from a paired device.
async function queueAct(action, lease, priority) {
if (demo) { toast("demo: " + action + " " + lease + " is not sent", "info"); return; }
const key = "queue\u0000" + lease;
busy.add(key);
errors.delete(key);
paint();
try {
const res = await fetch("/v1/board/queue", {
method: "POST",
headers: { "X-Berth": "1", "Content-Type": "application/json", accept: "application/json" },
body: JSON.stringify(priority ? { lease, action, priority } : { lease, action }),
});
if (!res.ok) {
const j = await res.json().catch(() => ({}));
errors.set(key, j.message || res.statusText);
if (res.status === 401) paired = false;
}
} catch (e) {
errors.set(key, e.message);
}
busy.delete(key);
await read();
}

function paintQueue(many) {
if (queue === null) { queueRoot.replaceChildren(el("div", "empty-note", "queue not read")); return; }
if (queue.length === 0) { queueRoot.replaceChildren(el("div", "empty-note", "nobody is waiting")); return; }
queueRoot.replaceChildren(...queue.map((l, i) => queueRow(l, i, many)));
}

function queueRow(l, i, many) {
const key = "queue\u0000" + l.id;
const who = l.group && l.group !== l.engine ? l.group + " (" + l.engine + ")" : l.engine;
const right = el("span", "sw-state", el("span", "sw-chip", (l.priority || "interactive").toUpperCase()));
if (l.training) right.append(el("span", "sw-chip sw-pinned", "TRAINING"));
if (many && l.card) right.prepend(el("span", "sw-chip", l.card));
const row = el("div", "sq sw-engine sw-waiting");
row.append(el("div", "top", el("span", "name", (i + 1) + ". " + who), right));
row.append(el("div", "meta", l.id + ", waiting " + Math.round(l.age_seconds || 0) + " s"));
const err = errors.get(key);
if (err) row.append(el("div", "meta sw-error", err));
if (paired && !l.training && !l.pinned && l.priority !== "hold") {
const actions = el("div", "sw-actions");
const first = el("button", "btn sw-btn", busy.has(key) ? "…" : "FIRST");
first.type = "button";
first.disabled = busy.has(key) || i === 0;
first.setAttribute("aria-label", "move " + who + " to the front of its class");
first.addEventListener("click", () => queueAct("bump", l.id));
const prio = el("select", "sw-prio");
prio.setAttribute("aria-label", "priority of " + who);
for (const p of ["batch", "interactive", "foreground"]) {
const o = el("option", null, p);
o.value = p;
o.selected = (l.priority || "interactive") === p;
prio.append(o);
}
prio.disabled = busy.has(key);
prio.addEventListener("change", () => queueAct("prio", l.id, prio.value));
const drop = el("button", "btn sw-btn sw-stop", "DROP");
drop.type = "button";
drop.disabled = busy.has(key);
drop.setAttribute("aria-label", "drop " + who + " from the queue");
drop.addEventListener("click", () => queueAct("drop", l.id));
actions.append(first, prio, drop);
row.append(actions);
}
return row;
}

function pinnedHolders() {
const names = new Set();
for (const c of (board && board.cards) || []) for (const h of c.holders || []) if (h.pinned) names.add(h.name);
Expand All @@ -79,6 +163,7 @@ export function initSwitch({ demo }) {
if (pick && !cards.some((c) => c.card === pick)) pick = "";
meta.textContent = board.reachable ? (board.name || "this node") : "berthd is not answering; cards unknown";
cardsRoot.replaceChildren(...cards.map(cardItem));
paintQueue(cards.length > 1);
paintPicker(cards);
noteRoot.replaceChildren(paired ? pairButton() : el("div", "empty-note", "Read only. Run berth pair on the machine to switch from this device."));
const shown = pick ? engines.filter((e) => (e.cards || []).includes(pick)) : engines;
Expand Down
Loading