diff --git a/CHANGELOG.md b/CHANGELOG.md index d83d28c..a323991 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,15 @@ has a matching GitHub release whose body comes from its section below. ### Added +- `berth why ID` tells one lease's story in plain words: when it asked for the card, each reason it waited, the + grant, each drain and who it was for, the release, and the berthd restarts it lived through. `berth why ENGINE` + reads that engine's newest lease. A lease berthd still holds ends with where it stands now, and one a restart + cut off names the lease its engine re-attached as. +- berthd keeps its event stream in `events.jsonl` in the state directory (`--events`, `""` for memory only). A + restart reads the last 1000 events back, goes on numbering after them, and goes on with the lease numbers, so + `berth events --since N` and `berth why` read across restarts and deploys. The file moves to `events.jsonl.1` + at 8 MiB. Each start is a `node.started` event that names the version. + - A schedule for each card, up to 14 days ahead. `berth schedule` shows what holds each card, the waiters in grant order and every booking. `berth schedule book` books an engine's run (it goes first, runs at the booking's priority, and keeps the card against everything but a hold), a priority change for set hours, or downtime (no diff --git a/docs/harbor.md b/docs/harbor.md index fe00857..6aa5943 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -65,7 +65,11 @@ for a phone. first-seen time; optional desktop notification and webhook. Report-only by design. - **Contested grants:** a grant made while stowaways hold memory is flagged, not hidden. - **Event log:** every request, wait, drain, release, free, stuck lease, stowaway and - probe change, with gap-free sequence numbers; `berth events -f` follows it live. + probe change, with gap-free sequence numbers; `berth events -f` follows it live. berthd keeps the + stream in `~/.local/state/berth/events.jsonl` (`--events`), so the numbers and the last 1000 events carry + on across a restart. `berth why ID` tells one lease's part of it in plain words: why it waited, the + grant, each drain and who it was for, and the restarts it lived through. `berth why ENGINE` reads that + engine's newest lease. - **The collisions board:** collisions avoided, VRAM handed off, drains, stuck releases caught, stowaways caught, contested grants, refusals. Each count links back to a recorded event. berthd keeps the board in `~/.local/state/berth/counters.json` diff --git a/docs/interfaces.md b/docs/interfaces.md index a649dc3..feab98a 100644 --- a/docs/interfaces.md +++ b/docs/interfaces.md @@ -2,7 +2,7 @@ - **CLI:** one command tree with one output contract. `berth status`, `berth events -f`, `berth run`, `berth hold`, `berth stop`, `berth cordon|uncordon`, `berth notify`, `berth claims`, `berth pull`, `berth verify`, `berth models`, `berth gateway`, - `berth wire llama-swap`, `berth engine check`, `berth contract ENGINE`, `berth fit NAME`, `berth queue`, `berth config check`, `berth config show`, + `berth wire llama-swap`, `berth engine check`, `berth contract ENGINE`, `berth fit NAME`, `berth queue`, `berth why ID|ENGINE`, `berth config check`, `berth config show`, `berth export`, `berth key`, `berth skill install|show`, `berth update`, `berth version`, and for the node itself `berth daemon`, `berth harbor`, `berth service install|uninstall|status`, `berth reclaim`, `berth resume`, `berth prefetch`, `berth storage`, `berth civitai search|versions` diff --git a/internal/cli/commands.go b/internal/cli/commands.go index ea4243b..a1bbfb4 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -223,6 +223,22 @@ notify-send, so any desktop or notification daemon shows it. On Windows it is a tray icon whose notifications are Windows' own toasts; a click opens the harbor.`, Run: runNotify, }, + { + Name: "why", + Summary: "one lease's story: why it waited, was granted, drained or refused", + Args: "ID|ENGINE [--socket P] [--format F]", + Help: `Print every event berthd kept about one lease, oldest first, in plain words: +when it asked for the card, each reason it waited, the grant, each drain and +who it was for, the release, and the berthd restarts it lived through. ENGINE +picks that engine's newest lease. A lease that berthd still holds ends with +where it stands now; one a restart cut off names the lease its engine +re-attached as. + +berthd keeps its events in events.jsonl in the state directory (--events), so +the story reads across restarts and deploys, back to the last 1000 events.`, + Formats: []Format{FormatHuman, FormatJSON, FormatYAML}, + Run: runWhy, + }, { Name: "fit", Summary: "where a request for an engine or model would run now, and what the start costs", diff --git a/internal/cli/daemon.go b/internal/cli/daemon.go index 32a7b84..2a5a630 100644 --- a/internal/cli/daemon.go +++ b/internal/cli/daemon.go @@ -21,6 +21,7 @@ import ( "github.com/T-Crypt/berth/internal/claims" "github.com/T-Crypt/berth/internal/config" "github.com/T-Crypt/berth/internal/engines" + "github.com/T-Crypt/berth/internal/eventlog" "github.com/T-Crypt/berth/internal/gpu" "github.com/T-Crypt/berth/internal/history" "github.com/T-Crypt/berth/internal/host" @@ -54,6 +55,7 @@ func runDaemon(c *Context) int { scheduleFile := fs.String("schedule", defaultStateFile("schedule.json"), "the cards' bookings (berth schedule): runs, priority changes, downtime; \"\" = no schedule") usageFile := fs.String("usage", defaultStateFile("usage.json"), "card-hours per engine, the default juggle ranking; \"\" = memory only") footprintFile := fs.String("footprint", defaultStateFile("footprint.json"), "VRAM measured per engine across restarts, the estimate behind EST. VRAM; \"\" = memory only") + eventsFile := fs.String("events", defaultStateFile("events.jsonl"), "the event stream across restarts, what berth events and berth why read; \"\" = memory only") if code, done := c.Parse(fs); done { return code } @@ -127,6 +129,16 @@ func runDaemon(c *Context) int { allowList = splitList(*allow) } events, ids, seq, ledger := node.Shared() + if *eventsFile != "" { + if kept, err := eventlog.Open(*eventsFile); err != nil { + log.Error("event file; events are kept in memory only", "path", *eventsFile, "err", err) + } else { + events = kept + defer kept.Close() + ids.Store(kept.LastLease()) + } + } + events.Emit(proto.Event{At: time.Now(), Kind: proto.EvNodeStarted, Detail: "berth " + c.Version}) procTable := procs.Shared(procs.List, *poll/2) var cards []*node.Card var snaps []gpu.Snapshot diff --git a/internal/cli/why.go b/internal/cli/why.go new file mode 100644 index 0000000..1ae5379 --- /dev/null +++ b/internal/cli/why.go @@ -0,0 +1,206 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "slices" + "time" + + "github.com/T-Crypt/berth/internal/proto" + "github.com/T-Crypt/berth/internal/server" +) + +// whyDoc is one lease's story: every event berthd kept about it, oldest +// first, with the restarts it lived through, and where it stands now. +type whyDoc struct { + Lease string `json:"lease" yaml:"lease"` + Engine string `json:"engine" yaml:"engine"` + Card string `json:"card,omitempty" yaml:"card,omitempty"` + Events []proto.Event `json:"events" yaml:"events"` + // Now is the lease as berthd holds it now, nil once it has ended. + Now *proto.Lease `json:"now,omitempty" yaml:"now,omitempty"` + // Next is the lease its engine re-attached as after a berthd restart + // cut this one off, "" when none did. + Next string `json:"next,omitempty" yaml:"next,omitempty"` +} + +// whyVerb says what each kind of event did to the lease, in the words the +// story prints. A kind not listed prints as its name. +var whyVerb = map[string]string{ + proto.EvRequested: "asked for the card", + proto.EvWaiting: "waited", + proto.EvGranted: "granted", + proto.EvReattached: "re-attached after a berthd restart", + proto.EvDraining: "asked to give the card back", + proto.EvReleased: "released the card", + proto.EvFreed: "its memory is free", + proto.EvRefused: "refused", + proto.EvOrphaned: "its process was gone, so berthd ended the lease", + proto.EvStuck: "its memory stayed on the card after the release", + proto.EvParked: "parked in RAM", + proto.EvResumed: "resumed from RAM", + proto.EvUnparked: "loaded back onto the card without a resume", + proto.EvParkRefused: "could not park", + proto.EvParkEvicted: "stopped while parked", + proto.EvJuggle: "a juggle for the card", + proto.EvQueueOverride: "the operator changed its place in line", + proto.EvBookingPrio: "a booking changed its priority", + proto.EvLending: "asked to lend part of the card", + proto.EvLent: "lent part of the card", + proto.EvReturned: "took its lent memory back", + proto.EvLendFailed: "the lend failed", + proto.EvNodeStarted: "berthd restarted", +} + +// ends is the kinds after which a lease has no more to say. +var ends = []string{proto.EvFreed, proto.EvRefused, proto.EvOrphaned} + +// whyOf finds the lease name means, a lease ID or else the engine's newest +// lease, and tells its story from evs. live is the leases berthd holds now. +func whyOf(evs []proto.Event, live []proto.Lease, name string) (whyDoc, bool) { + id := "" + for i := len(evs) - 1; i >= 0 && id == ""; i-- { + if evs[i].Lease == name { + id = name + } + } + for i := len(evs) - 1; i >= 0 && id == ""; i-- { + if evs[i].Engine == name && evs[i].Lease != "" { + id = evs[i].Lease + } + } + if id == "" { + return whyDoc{}, false + } + doc := whyDoc{Lease: id} + for _, l := range live { + if l.ID == id { + doc.Now = &l + } + } + first, last := -1, -1 + for i, e := range evs { + if e.Lease == id { + if first < 0 { + first = i + } + last = i + doc.Engine, doc.Card = e.Engine, e.Card + } + } + ended := slices.Contains(ends, evs[last].Kind) + for i := first; i < len(evs); i++ { + e := evs[i] + switch { + case e.Lease == id: + doc.Events = append(doc.Events, e) + case e.Kind == proto.EvNodeStarted && (i < last || (!ended && doc.Now == nil)): + doc.Events = append(doc.Events, e) + if i > last { + doc.Next = reattachedAs(evs[i+1:], doc.Engine) + return doc, true + } + } + } + return doc, true +} + +// reattachedAs is the lease engine re-attached as in evs, which start just +// after a berthd restart, "" when it did not before the next restart. +func reattachedAs(evs []proto.Event, engine string) string { + for _, e := range evs { + if e.Kind == proto.EvNodeStarted { + return "" + } + if e.Kind == proto.EvReattached && e.Engine == engine { + return e.Lease + } + } + return "" +} + +type whyView struct{ doc whyDoc } + +func (v whyView) Data() any { return v.doc } + +func (v whyView) Human(c *Context, _ bool) error { + d := v.doc + head := d.Lease + if d.Engine != "" && d.Engine != d.Lease { + head += " " + d.Engine + } + if d.Card != "" { + head += " " + d.Card + } + c.Printf("%s\n", head) + for _, e := range d.Events { + verb, ok := whyVerb[e.Kind] + if !ok { + verb = e.Kind + } + line := verb + if e.Bytes != 0 { + line += ", " + server.MiB(e.Bytes) + } + if e.Detail != "" { + line += ": " + e.Detail + } + c.Printf(" %s %s\n", c.Style.Dim(e.At.Local().Format("Jan 02 15:04:05")), line) + } + switch { + case d.Now != nil: + c.Printf("now %s, %s", d.Now.State, d.Now.Priority) + if d.Now.Position > 0 { + c.Printf(", %d in line", d.Now.Position) + } + if d.Now.TurnS > 0 { + c.Printf(", %s of its minimum turn left", (time.Duration(d.Now.TurnS) * time.Second).Round(time.Second)) + } + c.Printf("\n") + case d.Next != "": + c.Printf("its engine re-attached as %s: berth why %s\n", d.Next, d.Next) + } + return nil +} + +func runWhy(c *Context) int { + fs := c.FlagSet() + sock := fs.String("socket", defaultSocket(), "berthd socket ($BERTH_SOCKET)") + if code, done := c.ParseInterspersed(fs); done { + return code + } + if len(c.Args) != 1 { + return c.fail(Systemf("berth why takes one lease ID or engine name"). + WithHint("for example: berth why strata-24, or berth why comfyui")) + } + conn, raw, err := server.Dial(*sock) + if err != nil { + return c.fail(Systemf("berthd unreachable on %s: %v", *sock, err)) + } + defer raw.Close() + if err := conn.Send(proto.Msg{Op: proto.OpEvents}); err != nil { + return c.fail(Systemf("berthd on %s: %v", *sock, err)) + } + var evs []proto.Event + for { + m, err := conn.Recv() + if err != nil { + break + } + if m.Event != nil { + evs = append(evs, *m.Event) + } + } + st, _ := contractStatus(*sock) + var live []proto.Lease + if st != nil { + live = append(st.Leases, st.Waiting...) + } + doc, found := whyOf(evs, live, c.Args[0]) + if !found { + return c.fail(Failf("no lease or engine named %q in berthd's last %d events", c.Args[0], len(evs)). + WithHint("berth status lists the leases now; berth events lists what berthd kept")) + } + return c.Write(whyView{doc: doc}, nil) +} diff --git a/internal/cli/why_test.go b/internal/cli/why_test.go new file mode 100644 index 0000000..63a68a2 --- /dev/null +++ b/internal/cli/why_test.go @@ -0,0 +1,108 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/proto" + "github.com/T-Crypt/berth/internal/server" +) + +func at(min int) time.Time { return time.Date(2026, 11, 12, 23, min, 0, 0, time.UTC) } + +// The 11-12 night: strata waited for OrcaSAQ to drain, held the card, and +// was drained for Dirk-27B. Then berthd restarted and Dirk re-attached. +var night = []proto.Event{ + {Seq: 1, At: at(22), Kind: proto.EvRequested, Lease: "strata-24", Engine: "strata", Detail: "foreground"}, + {Seq: 2, At: at(22), Kind: proto.EvDraining, Lease: "OrcaSAQ-23", Engine: "OrcaSAQ", Detail: "strata (foreground) needs the card"}, + {Seq: 3, At: at(22), Kind: proto.EvWaiting, Lease: "strata-24", Engine: "strata", Detail: "next in line: draining OrcaSAQ [foreground, active]"}, + {Seq: 4, At: at(23), Kind: proto.EvGranted, Lease: "strata-24", Engine: "strata", Detail: "waited 1.631s"}, + {Seq: 5, At: at(25), Kind: proto.EvRequested, Lease: "Dirk-27B-25", Engine: "Dirk-27B", Detail: "foreground"}, + {Seq: 6, At: at(25), Kind: proto.EvDraining, Lease: "strata-24", Engine: "strata", Detail: "Dirk-27B (foreground) needs the card"}, + {Seq: 7, At: at(25), Kind: proto.EvReleased, Lease: "strata-24", Engine: "strata"}, + {Seq: 8, At: at(26), Kind: proto.EvFreed, Lease: "strata-24", Engine: "strata", Bytes: 16 << 30}, + {Seq: 9, At: at(26), Kind: proto.EvGranted, Lease: "Dirk-27B-25", Engine: "Dirk-27B", Detail: "waited 40s"}, + {Seq: 10, At: at(30), Kind: proto.EvNodeStarted, Detail: "berth 0.6.0"}, + {Seq: 11, At: at(30), Kind: proto.EvReattached, Lease: "Dirk-27B-26", Engine: "Dirk-27B", Detail: "foreground"}, +} + +func kindsOf(evs []proto.Event) string { + var k []string + for _, e := range evs { + k = append(k, e.Kind) + } + return strings.Join(k, " ") +} + +func TestWhyTellsOneLeasesStory(t *testing.T) { + doc, ok := whyOf(night, nil, "strata") + if !ok { + t.Fatal("strata not found") + } + if doc.Lease != "strata-24" || doc.Engine != "strata" { + t.Fatalf("lease = %q engine = %q", doc.Lease, doc.Engine) + } + if got, want := kindsOf(doc.Events), "lease.requested lease.waiting lease.granted lease.draining lease.released lease.freed"; got != want { + t.Fatalf("events = %s, want %s", got, want) + } + if doc.Next != "" { + t.Fatalf("an ended lease names a re-attach: %q", doc.Next) + } +} + +// A lease a restart cut off ends at the restart and names the lease its +// engine re-attached as. +func TestWhyFollowsARestart(t *testing.T) { + doc, ok := whyOf(night, nil, "Dirk-27B-25") + if !ok { + t.Fatal("Dirk-27B-25 not found") + } + if got, want := kindsOf(doc.Events), "lease.requested lease.granted node.started"; got != want { + t.Fatalf("events = %s, want %s", got, want) + } + if doc.Next != "Dirk-27B-26" { + t.Fatalf("next = %q, want Dirk-27B-26", doc.Next) + } + + doc, _ = whyOf(night, []proto.Lease{{ID: "Dirk-27B-26", State: "active", Priority: "foreground"}}, "Dirk-27B") + if doc.Lease != "Dirk-27B-26" || doc.Now == nil || doc.Now.State != "active" { + t.Fatalf("the engine's newest lease = %+v", doc) + } +} + +func TestWhyAgainstADaemon(t *testing.T) { + isolate(t) + sock := startDaemonWith(t, server.Extras{}) + holder := ask(t, sock, proto.Msg{Engine: "comfyui", Priority: "interactive"}) + if m, err := holder.Recv(); err != nil || m.Op != proto.OpGranted { + t.Fatalf("holder: %+v, %v", m, err) + } + ask(t, sock, proto.Msg{Engine: "strata", Priority: "foreground", Wait: 30}) + waitUntil(t, func() bool { return len(queueJSON(t, sock)) == 1 }) + + code, out, errOut := runCLI(t, "why", "comfyui", "--socket", sock) + if code != 0 { + t.Fatalf("why (exit %d): %s", code, errOut) + } + for _, want := range []string{"asked for the card", "granted", "asked to give the card back: strata (foreground) needs the card", "now "} { + if !strings.Contains(out, want) { + t.Fatalf("why comfyui = %q, want it to contain %q", out, want) + } + } + + code, out, _ = runCLI(t, "why", "strata", "--socket", sock, "--format", "json") + var doc whyDoc + if code != 0 || json.Unmarshal([]byte(out), &doc) != nil || doc.Now == nil || doc.Now.State != "waiting" { + t.Fatalf("why strata json (exit %d): %s", code, out) + } + + code, _, errOut = runCLI(t, "why", "nobody", "--socket", sock) + if code != 1 || !strings.Contains(errOut, `no lease or engine named "nobody"`) { + t.Fatalf("why nobody (exit %d): %s", code, errOut) + } +} diff --git a/internal/eventlog/eventlog.go b/internal/eventlog/eventlog.go index b07c98e..27707b6 100644 --- a/internal/eventlog/eventlog.go +++ b/internal/eventlog/eventlog.go @@ -4,10 +4,20 @@ // Package eventlog is berthd's event stream: one numbered sequence for the // whole machine, kept in a ring and fanned out to watchers. Every card's // broker on a node writes to the same log, so `berth events` is one ordered -// stream however many cards the node governs. +// stream however many cards the node governs. Opened on a file, the stream +// outlives berthd: a restart reads the last Ring events back and numbers on +// from them. package eventlog import ( + "bufio" + "encoding/json" + "errors" + "io/fs" + "os" + "path/filepath" + "strconv" + "strings" "sync" "github.com/T-Crypt/berth/internal/proto" @@ -20,29 +30,173 @@ const Ring = 1000 // closed. const watchBuffer = 256 +// maxFile is how large the event file grows before it is moved to +// path.1, which keeps one file of history behind the live one. +const maxFile = 8 << 20 + // Log is one node's event stream. type Log struct { mu sync.Mutex seq uint64 events []proto.Event // ring, oldest first subs map[chan proto.Event]struct{} + // file is where each event is appended, nil for a log kept in memory + // only or one whose file failed a write. + file *os.File + path string + size int64 + maxFile int64 } -// New returns an empty log. +// New returns an empty log kept in memory only. func New() *Log { return &Log{subs: map[chan proto.Event]struct{}{}} } -// Emit numbers e, keeps it, and hands it to every watcher. A watcher whose -// buffer is full is dropped, so one slow client never stalls a broker. -func (l *Log) Emit(e proto.Event) { +// Open returns the log kept in path: the newest Ring events in path.1 and +// path come back, numbering goes on after the newest, and each new event is +// appended. A line a crash cut short is skipped. +func Open(path string) (*Log, error) { + l := New() + for _, p := range []string{path + ".1", path} { + if err := l.load(p); err != nil { + return nil, err + } + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return nil, err + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return nil, err + } + fi, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + l.file, l.path, l.size, l.maxFile = f, path, fi.Size(), maxFile + if torn(path, fi.Size()) { + n, _ := f.Write([]byte{'\n'}) + l.size += int64(n) + } + return l, nil +} + +// torn is whether the file's last line has no newline: a crash cut it, and +// the next event must start a line of its own. +func torn(path string, size int64) bool { + if size == 0 { + return false + } + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + b := make([]byte, 1) + _, err = f.ReadAt(b, size-1) + return err == nil && b[0] != '\n' +} + +func (l *Log) load(path string) error { + f, err := os.Open(path) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + defer f.Close() + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 64<<10), 1<<20) + for sc.Scan() { + var e proto.Event + if json.Unmarshal(sc.Bytes(), &e) != nil || e.Seq <= l.seq { + continue + } + l.seq = e.Seq + l.keep(e) + } + return sc.Err() +} + +// Close stops writing to the file. The log keeps working in memory. +func (l *Log) Close() error { l.mu.Lock() defer l.mu.Unlock() - l.seq++ - e.Seq = l.seq + if l.file == nil { + return nil + } + err := l.file.Close() + l.file = nil + return err +} + +// LastLease is the highest lease number the kept events name, so a +// restarted berthd does not hand out lease IDs its history already uses. +func (l *Log) LastLease() uint64 { + l.mu.Lock() + defer l.mu.Unlock() + var n uint64 + for _, e := range l.events { + i := strings.LastIndexByte(e.Lease, '-') + if i < 0 { + continue + } + if v, err := strconv.ParseUint(e.Lease[i+1:], 10, 64); err == nil && v > n { + n = v + } + } + return n +} + +func (l *Log) keep(e proto.Event) { if len(l.events) == Ring { copy(l.events, l.events[1:]) l.events = l.events[:Ring-1] } l.events = append(l.events, e) +} + +// write appends e to the file, moving the file aside first when it is +// full. A failed write ends the file: the stream goes on in memory. +func (l *Log) write(e proto.Event) { + if l.file == nil { + return + } + line, err := json.Marshal(e) + if err != nil { + return + } + line = append(line, '\n') + if l.size+int64(len(line)) > l.maxFile && l.size > 0 { + l.file.Close() + l.file = nil + if os.Rename(l.path, l.path+".1") != nil { + return + } + f, err := os.OpenFile(l.path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC|os.O_APPEND, 0o600) + if err != nil { + return + } + l.file, l.size = f, 0 + } + n, err := l.file.Write(line) + l.size += int64(n) + if err != nil { + l.file.Close() + l.file = nil + } +} + +// Emit numbers e, keeps it, and hands it to every watcher. A watcher whose +// buffer is full is dropped, so one slow client never stalls a broker. +func (l *Log) Emit(e proto.Event) { + l.mu.Lock() + defer l.mu.Unlock() + l.seq++ + e.Seq = l.seq + l.keep(e) + l.write(e) for ch := range l.subs { select { case ch <- e: diff --git a/internal/eventlog/eventlog_test.go b/internal/eventlog/eventlog_test.go new file mode 100644 index 0000000..c4fe16a --- /dev/null +++ b/internal/eventlog/eventlog_test.go @@ -0,0 +1,114 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package eventlog + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/proto" +) + +func open(t *testing.T, path string) *Log { + t.Helper() + l, err := Open(path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = l.Close() }) + return l +} + +func kinds(evs []proto.Event) []string { + var out []string + for _, e := range evs { + out = append(out, e.Kind) + } + return out +} + +// A restarted berthd shows the events from before it, and numbers its own +// after them, so `berth events --since N` and `berth why` read across a +// deploy. +func TestOpenKeepsTheStreamAcrossARestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "events.jsonl") + l := open(t, path) + l.Emit(proto.Event{Kind: proto.EvRequested, Lease: "strata-7", At: time.Unix(10, 0)}) + l.Emit(proto.Event{Kind: proto.EvGranted, Lease: "strata-7", At: time.Unix(11, 0)}) + if err := l.Close(); err != nil { + t.Fatal(err) + } + + l2 := open(t, path) + if got := l2.Last(); got != 2 { + t.Fatalf("Last after the restart = %d, want 2", got) + } + l2.Emit(proto.Event{Kind: proto.EvDraining, Lease: "strata-7"}) + evs, _, _ := l2.Events(0, false) + if got := kinds(evs); len(got) != 3 || got[0] != proto.EvRequested || got[2] != proto.EvDraining { + t.Fatalf("events after the restart = %v", got) + } + if evs[2].Seq != 3 { + t.Fatalf("the first event after the restart is seq %d, want 3", evs[2].Seq) + } + if got := l2.LastLease(); got != 7 { + t.Fatalf("LastLease = %d, want 7", got) + } +} + +// A line cut short by a crash is skipped, not the whole file. +func TestOpenSkipsATornLine(t *testing.T) { + path := filepath.Join(t.TempDir(), "events.jsonl") + data := `{"seq":1,"kind":"lease.requested","lease":"a-1"}` + "\n" + `{"seq":2,"kind":"lease.gr` + if err := os.WriteFile(path, []byte(data), 0o600); err != nil { + t.Fatal(err) + } + l := open(t, path) + l.Emit(proto.Event{Kind: proto.EvGranted, Lease: "a-1"}) + evs, _, _ := l.Events(0, false) + if got := kinds(evs); len(got) != 2 || got[1] != proto.EvGranted || evs[1].Seq != 2 { + t.Fatalf("events = %v (%+v)", got, evs) + } + l.Close() + l3 := open(t, path) + if got := l3.Last(); got != 2 { + t.Fatalf("Last after a torn line and a restart = %d, want 2", got) + } +} + +// The file is moved aside once it passes its cap, so it never grows without +// bound, and the events in the moved file still come back. +func TestOpenRotatesTheFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "events.jsonl") + l := open(t, path) + l.maxFile = 200 + for range 10 { + l.Emit(proto.Event{Kind: proto.EvRequested, Lease: "comfyui-1", Detail: "a detail line"}) + } + l.Close() + if fi, err := os.Stat(path); err != nil || fi.Size() > 400 { + t.Fatalf("the live file was not rotated: %v %v", fi, err) + } + if _, err := os.Stat(path + ".1"); err != nil { + t.Fatalf("no rotated file: %v", err) + } + l2 := open(t, path) + if got := l2.Last(); got != 10 { + t.Fatalf("Last after rotation = %d, want 10", got) + } +} + +// A first start has no state directory yet; Open makes it, as the other +// state files do. +func TestOpenMakesTheStateDirectory(t *testing.T) { + path := filepath.Join(t.TempDir(), "state", "berth", "events.jsonl") + l := open(t, path) + l.Emit(proto.Event{Kind: proto.EvNodeStarted}) + l.Close() + if fi, err := os.Stat(path); err != nil || fi.Size() == 0 { + t.Fatalf("no events written: %v %v", fi, err) + } +} diff --git a/internal/proto/proto.go b/internal/proto/proto.go index f2bb083..ef9cc04 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -343,6 +343,9 @@ const ( // host driver. Each fires once per transition. EvCardPassthrough = "card.passthrough" EvCardReturned = "card.returned" + // EvNodeStarted: berthd started. Its detail is the version. The events + // before it are the last daemon's, read back from the event file. + EvNodeStarted = "node.started" ) // Card states, the values Status.Passthrough.State takes. Normal is the diff --git a/internal/skill/SKILL.md b/internal/skill/SKILL.md index 72fa360..94feef9 100644 --- a/internal/skill/SKILL.md +++ b/internal/skill/SKILL.md @@ -23,6 +23,7 @@ berth fit MODEL --explain # where a request would run now (running, resume, fr berth queue # waiters in grant order; bump/prio/drop are the operator's, never an agent's own call berth schedule # what holds the card, the queue, and every booking for the next 14 days berth events -f # live: grants, drains, refusals, stowaways +berth why ID|ENGINE # one lease's story: why it waited, was drained or refused, across restarts ``` If `berth status` says berthd is unreachable, Berth is not coordinating this card: say so and @@ -150,7 +151,8 @@ builtin = true # false turns the built-in pack off 1. `berth status`: is something else leased, waiting, or a stowaway? 2. A refusal reason names who holds the card and why the request could not fit ("needs 20480 MiB; at most 14336 MiB can be freed; claims hold 10240 MiB"). -3. `berth events --format json` after the failure shows the drain and grant sequence. +3. `berth why ENGINE` after the failure tells that engine's last lease, wait by wait; `berth events --format json` + shows the whole drain and grant sequence. 4. Fix the cause (a smaller quant, `--needs-mib`, a priority, the user's hold ending), never by killing the holder. 5. A lease stuck after its engine exited: `berth reclaim` lists what the lease recorded. Only with