diff --git a/CHANGELOG.md b/CHANGELOG.md index f3d509b..662da7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ has a matching GitHub release whose body comes from its section below. ## [Unreleased] +### Added + +- `berth schedule book --claim` books a run that claims its hours. While it lasts, the booked engine's waiter + drains any holder but a hold or a pinned lease, whatever the holder's priority. Before, a `batch` run lost its + hours to an idle `foreground` holder, and the workaround was a downtime booked just before it. The harbor's + booking form has the same choice. Only the operator books a claim; the gateway refuses to move or cancel one. + (#345) + ### Fixed - On Windows, `berth service install --notify` no longer opens a console window at logon. The Run key started diff --git a/docs/gateway.md b/docs/gateway.md index 6651c1e..730c0b2 100644 --- a/docs/gateway.md +++ b/docs/gateway.md @@ -333,8 +333,9 @@ curl -s -X DELETE http://127.0.0.1:9799/v1/berth/schedule/b4 -H "$K" - `PATCH /v1/berth/schedule/ID` moves a run. It takes `start`, `for`, `priority` and `note`, keeps the rest and keeps the ID. `DELETE` cancels a run. Both answer `200` with the booking. -The gateway books, moves and cancels runs only. A priority change or downtime acts on everyone else's work, so it -is the operator's: `berth schedule`, or a paired device in the harbor. `PATCH` or `DELETE` on one is a `403`. Any +The gateway books, moves and cancels runs only. A priority change, downtime or a run that claims its hours acts on +everyone else's work, so it is the operator's: `berth schedule`, or a paired device in the harbor. The gateway +books no claim, and `PATCH` or `DELETE` on one of these is a `403`. Any holder of the key can change any run; per-device ownership comes with device keys. ## One URL per card diff --git a/docs/leasing.md b/docs/leasing.md index 84e50d9..1eac213 100644 --- a/docs/leasing.md +++ b/docs/leasing.md @@ -187,7 +187,10 @@ three kinds: - `run` is an engine's turn. While it lasts, the engine's waiter goes first in line and runs at the booking's priority (foreground when the booking names none). It wins a tie against a holder that has no booking. Once it holds the card, nothing but a hold drains it until the booking ends. A minimum turn (`min_turn`) still protects - the holder it takes over from. + the holder it takes over from. A `batch` run does not drain a `foreground` holder, so an engine left idle at a + higher class keeps the card through the booked hours. Book the run with `--claim` to take them back: while it + lasts, the booked engine's waiter drains any holder but a hold or a pinned lease, whatever its class. A training + stays too. Only the operator books a claim; the gateway does not. - `priority` moves an engine to another class for set hours, such as llama-swap at batch while a training runs. It claims no time on the card, so it may overlap a run. - `down` is downtime. No new work starts, and holders drain at their usual boundary when it begins. A hold and a @@ -200,6 +203,7 @@ everyone else, so a booking never keeps the card idle. berth schedule # what holds each card, the waiters in grant order, every booking berth schedule book --who pi-pocket --engine pocket-train --at 23:00 --for 5h --days 7 --priority batch berth schedule book --who training --engine sft --at "sun 18:00" --for 3h +berth schedule book --who reviews --engine strata --priority batch --claim --at 00:00 --for 85m --days 7 berth schedule book --who operator --kind priority --engine llama-swap --priority batch --at "sun 18:00" --for 3h berth schedule book --who operator --kind down --at "tue 04:00" --for 1h --note "driver update" berth schedule move b4 --at 01:00 # new hours, same ID; --for, --priority and --note change those diff --git a/internal/broker/booking.go b/internal/broker/booking.go index e3d3640..7f17156 100644 --- a/internal/broker/booking.go +++ b/internal/broker/booking.go @@ -6,7 +6,8 @@ package broker // The card's schedule. berthd's booking file says who has the card when, // and the broker reads it on every tick. A run booking is an engine's turn: // its waiter goes first, it runs at the booking's priority, and while it -// holds the card nothing but a hold drains it. A priority booking moves an +// holds the card nothing but a hold drains it. A run that claims its hours +// also drains a holder of a higher class for its waiter. A priority booking moves an // engine to another class for its hours. Downtime takes no new work and // drains its holders at the start. An hour nobody booked runs as it always // did, by priority then arrival, so an unused booking leaves the card idle @@ -57,6 +58,14 @@ func (b *Broker) inRun(l *lease, now time.Time) bool { return ok && x.Kind == proto.BookRun } +// claiming is whether a run booking that claims its hours holds for l at +// now: l's waiter may drain any holder but a hold or a pinned one, whatever +// the holder's class. +func (b *Broker) claiming(l *lease, now time.Time) bool { + x, ok := b.booked(l, now) + return ok && x.Kind == proto.BookRun && x.Claim +} + // bookedPrio is the class a booking gives: its own, and foreground for a // run that names none. func bookedPrio(x proto.Booking) Priority { diff --git a/internal/broker/booking_test.go b/internal/broker/booking_test.go index edb6a89..e950f4e 100644 --- a/internal/broker/booking_test.go +++ b/internal/broker/booking_test.go @@ -93,6 +93,42 @@ func TestRunBookingKeepsItsHolderUntilItEnds(t *testing.T) { } } +// A batch run booking does not drain a foreground holder; one that claims +// its hours does, and still leaves a hold alone. +func TestClaimRunBookingDrainsAHigherHolder(t *testing.T) { + b, _ := startTurnBroker(t, procs.Map{}) + b.SetBookings([]proto.Booking{book(b, "b1", proto.BookRun, "strata", "batch", 0, time.Hour)}) + holder := must(t, acquire(b, Request{Engine: "strata-sc117", Priority: Foreground, Needs: 20 * GiB})) + w := acquireW(b, Request{Engine: "strata", Priority: Foreground, Needs: 20 * GiB}) + pending(t, w.ch) + select { + case why := <-holder.Drains(): + t.Fatalf("a batch run without a claim drained a foreground holder: %s", why) + default: + } + + claim := book(b, "b1", proto.BookRun, "strata", "batch", 0, time.Hour) + claim.Claim = true + b.SetBookings([]proto.Booking{claim}) + if got, want := drained(t, holder), "strata (batch) needs the card"; got != want { + t.Errorf("drain = %q, want %q", got, want) + } + holder.Release() + b.Tick(context.Background()) + granted(t, b, "strata", w.ch) + + b2, _ := startTurnBroker(t, procs.Map{}) + b2.SetBookings([]proto.Booking{claim}) + hold := must(t, acquire(b2, Request{Engine: "game", Priority: Hold, Needs: 20 * GiB})) + w2 := acquireW(b2, Request{Engine: "strata", Priority: Batch, Needs: 20 * GiB}) + pending(t, w2.ch) + select { + case why := <-hold.Drains(): + t.Fatalf("a claim drained a hold: %s", why) + default: + } +} + // A run booking puts its waiter ahead of an earlier foreground one. func TestRunBookingGoesFirstInLine(t *testing.T) { b, _ := startTurnBroker(t, procs.Map{}) diff --git a/internal/broker/broker.go b/internal/broker/broker.go index e575419..f4becf6 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -1579,7 +1579,7 @@ func (b *Broker) evaluate(w *lease, now time.Time) (bool, string, []*lease) { // for it to become parked (or fall back to a drain); evicting // it mid-park would throw the park away. pending = append(pending, h) - case h.pinned || h.prio > w.prio, w.prio != Hold && b.inRun(h, now) && !b.inRun(w, now): + case h.pinned || h.prio > w.prio && (h.prio == Hold || !b.claiming(w, now)), w.prio != Hold && b.inRun(h, now) && !b.inRun(w, now): blockers = append(blockers, h) case w.prio != Hold && h.inTurn(now): // A minimum turn: the holder keeps the card until its turn diff --git a/internal/cli/schedule.go b/internal/cli/schedule.go index e063e26..78dfcdd 100644 --- a/internal/cli/schedule.go +++ b/internal/cli/schedule.go @@ -50,7 +50,11 @@ func (v scheduleView) Human(c *Context, ok bool) error { if x.Holds(v.b.Now) { when += " (now)" } - fmt.Fprintf(tw, "%s\t%s%s\t%s\t%s\t%s\t%s\t%s\n", x.ID, card(x.Card), x.Kind, x.Who, dash(x.Engine), dash(x.Priority), when, x.Note) + kind := string(x.Kind) + if x.Claim { + kind += " (claim)" + } + fmt.Fprintf(tw, "%s\t%s%s\t%s\t%s\t%s\t%s\t%s\n", x.ID, card(x.Card), kind, x.Who, dash(x.Engine), dash(x.Priority), when, x.Note) } return tw.Flush() } @@ -84,6 +88,7 @@ func runSchedule(c *Context) int { earliest := fs.Bool("earliest", false, "book: take the first free hours at or after --at") card := fs.String("card", "", "book: the card (default the first)") note := fs.String("note", "", "book, move: a note the board shows") + claim := fs.Bool("claim", false, "book: a run that claims its hours drains a holder of any class but a hold for its engine") if code, done := c.ParseInterspersed(fs); done { return code } @@ -99,7 +104,7 @@ func runSchedule(c *Context) int { return c.fail(Failf("berth schedule book needs --at and --for").WithHint("%s", usage)) } b := proto.Booking{Card: *card, Kind: proto.BookingKind(*kind), Who: *who, Engine: *engine, Priority: *prio, - Start: start, End: start.Add(*forFlag), Note: *note} + Start: start, End: start.Add(*forFlag), Note: *note, Claim: *claim} if err := b.Check(now); err != nil { return c.fail(Failf("%v", err).WithHint("%s", usage)) } diff --git a/internal/cli/schedule_test.go b/internal/cli/schedule_test.go index 6b26994..62696d0 100644 --- a/internal/cli/schedule_test.go +++ b/internal/cli/schedule_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/T-Crypt/berth/internal/proto" "github.com/T-Crypt/berth/internal/schedule" @@ -76,4 +77,20 @@ func TestScheduleAgainstADaemon(t *testing.T) { if code != 0 || !strings.Contains(out, "nothing booked") || !strings.Contains(out, "RUNNING") { t.Fatalf("board after cancel (exit %d): %s", code, out) } + + code, out, errOut = runCLI(t, "schedule", "book", "--who", "op", "--engine", "strata", "--priority", "batch", "--claim", + "--at", "now", "--for", "1h", "--socket", sock) + if code != 0 || !strings.Contains(out, "booked b2") { + t.Fatalf("book --claim (exit %d): %s %s", code, out, errOut) + } + if got := s.List(time.Now()); len(got) != 1 || !got[0].Claim { + t.Errorf("schedule after --claim = %+v, want one claiming run", got) + } + if code, out, _ = runCLI(t, "schedule", "--socket", sock); code != 0 || !strings.Contains(out, "run (claim)") { + t.Errorf("board (exit %d) does not show the claim: %s", code, out) + } + code, _, errOut = runCLI(t, "schedule", "book", "--who", "op", "--kind", "down", "--claim", "--at", "23:00", "--for", "1h", "--socket", sock) + if code != 1 || !strings.Contains(errOut, "only a run booking claims") { + t.Errorf("down --claim (exit %d): %s", code, errOut) + } } diff --git a/internal/gateway/schedule.go b/internal/gateway/schedule.go index 5d02a9a..4c3e048 100644 --- a/internal/gateway/schedule.go +++ b/internal/gateway/schedule.go @@ -80,6 +80,9 @@ func (g *Gateway) runBooking(w http.ResponseWriter, r *http.Request) (proto.Book case st.Bookings[i].Kind != proto.BookRun: apiError(w, http.StatusForbidden, fmt.Sprintf("%s is a %s booking: only the operator changes it (berth schedule, or the harbor)", id, st.Bookings[i].Kind)) return proto.Booking{}, false + case st.Bookings[i].Claim: + apiError(w, http.StatusForbidden, fmt.Sprintf("%s claims its hours: only the operator changes it (berth schedule, or the harbor)", id)) + return proto.Booking{}, false } return st.Bookings[i], true } diff --git a/internal/gateway/schedule_test.go b/internal/gateway/schedule_test.go index 72ea8b5..accb35f 100644 --- a/internal/gateway/schedule_test.go +++ b/internal/gateway/schedule_test.go @@ -105,6 +105,22 @@ func TestScheduleRoutesLeaveTheOperatorsBookings(t *testing.T) { t.Errorf("%s downtime = %d %s, want 403", method, resp.StatusCode, body) } } + + resp = call(t, "POST", srv.URL+"/v1/berth/schedule", `{"who":"agent","engine":"x","start":"`+now.Add(5*time.Hour).Format(time.RFC3339)+`","for":"30m","claim":true}`) + if made := decode[map[string][]proto.Booking](t, resp)["bookings"]; len(made) != 1 || made[0].Claim { + t.Fatalf("a claim in the body booked %+v (%d), want a run without one", made, resp.StatusCode) + } + claim, err := s.Book(proto.Booking{Card: "gpu0", Kind: proto.BookRun, Who: "operator", Engine: "x", Claim: true, + Start: now.Add(3 * time.Hour), End: now.Add(4 * time.Hour)}, 1, false, now) + if err != nil { + t.Fatal(err) + } + for _, method := range []string{"PATCH", "DELETE"} { + resp := call(t, method, srv.URL+"/v1/berth/schedule/"+claim[0].ID, `{"for":"3h"}`) + if body := readAll(resp); resp.StatusCode != http.StatusForbidden || !strings.Contains(body, "only the operator") { + t.Errorf("%s claim = %d %s, want 403", method, resp.StatusCode, body) + } + } } func readAll(resp *http.Response) string { diff --git a/internal/harbor/web/index.html b/internal/harbor/web/index.html index 7d68f28..1d70dee 100644 --- a/internal/harbor/web/index.html +++ b/internal/harbor/web/index.html @@ -586,6 +586,7 @@