Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ jobs:
for skew in 3600s -3600s; do
echo "== conn and root, CELERIS_MONO_SKEW=$skew"
CELERIS_MONO_SKEW=$skew go test -count=1 -timeout=120s -tags celeris_monoskew -run '^(TestArmHeaderDeadlineAt|TestHeaderDeadline_ClearArmCycle)$' ./internal/conn/
CELERIS_MONO_SKEW=$skew go test -count=1 -timeout=120s -tags celeris_monoskew -run '^TestDetachSnapshotElapsedFromStartTimeNs889$' .
CELERIS_MONO_SKEW=$skew go test -count=1 -timeout=120s -tags celeris_monoskew -run '^(TestDetachSnapshotElapsedFromStartTimeNs889|TestStartTimeIsTheTrueWallTimeOfTheRequestsStamp1178)$' .
done
# Step 5 (celeris#1178): the HTTP/2 deadlines and waits, whole packages: conn
# (the H2 sweep against the stream stamps) and the stream code itself. With
Expand Down
11 changes: 7 additions & 4 deletions context.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"time"

"github.com/goceleris/celeris/internal/ctxkit"
"github.com/goceleris/celeris/internal/mono"
"github.com/goceleris/celeris/internal/protocol/h2/stream"
"github.com/goceleris/celeris/internal/testhooks"
)
Expand Down Expand Up @@ -481,11 +482,13 @@ func (c *Context) WorkerID() int {
// the framework before the handler chain runs, so all middleware share the
// same timestamp without calling time.Now() independently.
func (c *Context) StartTime() time.Time {
// Materialize lazily from the raw ns set by routerAdapter.HandleStream.
// Most requests never call StartTime(), so the time.Unix division is
// deferred off the hot path.
// Materialize lazily from the raw stamp set by routerAdapter.HandleStream
// (internal/mono's clock). mono.Time gives the true wall time of the stamp with
// a monotonic reading, so (once the clock is monotonic, celeris#1178) time.Since(c.StartTime()) (the logger, metrics and
// otel latencies) ignores steps of the wall clock. Most requests never call
// StartTime(), so this one clock read is deferred off the hot path.
if c.startTime.IsZero() && c.startTimeNs != 0 {
c.startTime = time.Unix(0, c.startTimeNs)
c.startTime = mono.Time(c.startTimeNs)
}
return c.startTime
}
Expand Down
44 changes: 19 additions & 25 deletions internal/mono/mono.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,38 +36,32 @@
// first sweep, for the rest of the process's life): worse than today, where a
// step only cuts the connections that were open during it. So:
//
// 1. (this package) The clock, its seam, and the scan test. [Now] still reads
// the WALL clock, so nothing here changes a value.
// 2. Routing PRs send every wall stamp through this package, one package at a
// time. They never change a clock, so each is correct on its own and they
// merge in any order.
// 3. The FLIP changes [Now] to the monotonic form (the constant flipped) in
// ONE PR. It is the only change of a clock.
// 1. The clock, its seam, the scan and the canary. [Now] reads the WALL clock,
// so nothing changes a value.
// 2. Routing PRs (io_uring, epoll, conn and the root package, the HTTP/2 stream
// code, StartTime) sent every wall stamp through this package. They never
// changed a clock, so each was correct on its own.
// 3. The FLIP changes [Now] to the monotonic form (the constant flipped) in ONE
// PR. It is the only change of a clock.
//
// # The scan test and its PENDING list (scan_test.go)
// # The scan test (scan_test.go)
//
// TestNoWallStamps walks the directories (not a package list) of the engines,
// internal/conn, internal/protocol, the root package and the WebSocket and SSE
// middleware, and matches SYNTACTICALLY, without type information: every call
// of .UnixNano(), .UnixMilli(), .UnixMicro() or .Unix(), every time.Unix(...),
// and the calls that strip a monotonic reading (.Round(0), .UTC(), .Local(),
// .In(...)). Each match, by (file, enclosing function, what), must be listed in
// one of two tables, keyed by names and counts, never by line:
//
// - stays: a site that stays on the wall for good (the Date header, the
// public SetWSIdleDeadline contract, one-second flood windows), with its
// reason;
// - pending: every site that has still to move onto this clock. A routing
// PR deletes its entries (the test also fails on a STALE entry, so a
// deleted entry cannot be forgotten, and a re-added wall read is a match in
// neither table). The test files that write a wall reading into a stamp
// field are in pendingFixtures, deleted the same way.
//
// The flip is gated in code: TestPendingIsEmptyWhenFlipped fails while flipped
// is true and any pending entry is left. A routing PR reverted after the flip
// re-adds an unlisted match, and CI goes red; the flip cherry-picked without
// its routing PRs fails on the release branch. The flip PR also carries the
// behaviour tests (red against the wall, green after).
// .In(...)). Each match, by (file, enclosing function, what) with a count,
// never by line, must be in the table stays with its reason (the Date header,
// the public SetWSIdleDeadline contract, one-second flood windows); the test
// also fails on a stale entry. During the routing steps a second table listed
// every site still to move and the test functions that wrote a wall reading into
// a stamp field; it is gone, so a routing PR reverted after the flip re-adds a
// match that is listed nowhere and CI goes red, and the flip cherry-picked
// without its routing PRs fails on the release branch. TestNoWallFixtures
// refuses a test function that writes a wall reading into a stamp field or hands
// one to a function that takes a stamp. A statement that mixes a drainclock or
// deferlinger reading with one of this package fails too.
//
// # The skew canary (test builds only)
//
Expand Down
19 changes: 4 additions & 15 deletions internal/mono/scan_tables_test.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
package mono

// The two tables of TestNoWallStamps and the list of TestNoWallFixtures (see the
// package documentation). Generated once by TestScanDump; edited by hand since.
// The table of TestNoWallStamps (see the package documentation): the wall reads
// that stay wall for good. The pending tables of the routing steps are gone: every
// stamp of the engines, conn, the root package and the HTTP/2 stream code is read
// from this package (celeris#1178), and a new wall read has to be argued for here.

type entry struct {
Count int
Expand All @@ -19,16 +21,3 @@ var stays = map[site]entry{
{"middleware/websocket/conn.go", "Conn.ReadMessageReuse", ".UnixNano"}: {1, "the public Context.SetWSIdleDeadline contract (wall Unix nanoseconds); the engine converts it with mono.FromWall"},
{"router.go", "var nowNano", ".UnixNano"}: {1, "the adaptive promotion TTL: stamped and compared in router.go, with its own test hook"},
}

// pending: every wall stamp that has still to move onto mono.Now(). A routing PR
// deletes its entries; the flip PR requires the table to be empty.
var pending = map[site]entry{
// step 4: route internal/conn H1 and the root package (handler.go, context_response.go; context.go StartTime keeps time.Unix(0, ns) until the flip PR, which makes it mono.Time and deletes its entry)
{"context.go", "Context.StartTime", "time.Unix"}: {1, "step 4"},
}

// pendingFixtures: test functions that write a wall reading into a stamp field or
// hand one to a function that takes a stamp. Each moves to mono.Now() with the
// routing PR of its package (the skew canary fails them otherwise), and its entry
// goes with it.
var pendingFixtures = map[string]bool{}
67 changes: 15 additions & 52 deletions internal/mono/scan_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -343,36 +343,22 @@ func mustRead(t testing.TB, path string) []byte {
return b
}

// TestNoWallStamps: every wall read of the scanned tree is listed, once, with
// its count, in stays (for good) or pending (to move), and every entry of both
// is still there. A routing PR deletes its pending entries; a wall read added
// later, or re-added by a revert, is in neither table and fails here.
// TestNoWallStamps: every wall read of the scanned tree is listed, with its count,
// in stays (with its reason), and every entry is still there. A wall read added
// later, or re-added by a revert of a routing PR, is not listed and fails here.
func TestNoWallStamps(t *testing.T) {
prod, _ := scanTree(t, moduleRoot(t))
listed := map[site]bool{}
for s := range stays {
listed[s] = true
}
for s := range pending {
if listed[s] {
t.Errorf("%s is in stays and in pending: decide which", s)
}
listed[s] = true
}
var unlisted, stale, count []string
for s, n := range prod {
e, ok := stays[s]
if p, isP := pending[s]; isP {
e, ok = p, true
}
switch {
case !ok:
unlisted = append(unlisted, fmt.Sprintf("%s (x%d)", s, n))
case e.Count != n:
count = append(count, fmt.Sprintf("%s: found %d, listed %d", s, n, e.Count))
}
}
for s := range listed {
for s := range stays {
if _, ok := prod[s]; !ok {
stale = append(stale, s.String())
}
Expand All @@ -393,47 +379,24 @@ func TestNoWallStamps(t *testing.T) {
}
}

// TestNoWallFixtures is the same for the test files: a test function that
// writes a wall reading into a stamp field, or hands one to a function that
// takes a stamp, is correct only while the clock and the wall agree, which the
// skew canary does not allow. It moves to mono.Now() with the routing PR of
// its package, and its entry goes.
// TestNoWallFixtures: no test function of the scanned tree writes a wall reading
// into a stamp field or hands one to a function that takes a stamp. Such a test is
// correct only while the clock and the wall agree, which the skew canary does not
// allow; it writes mono.Now() instead.
func TestNoWallFixtures(t *testing.T) {
_, fixtures := scanTree(t, moduleRoot(t))
var unlisted, stale []string
var bad []string
for f := range fixtures {
if !pendingFixtures[f] {
unlisted = append(unlisted, f)
}
}
for f := range pendingFixtures {
if !fixtures[f] {
stale = append(stale, f)
}
}
sort.Strings(unlisted)
sort.Strings(stale)
if len(unlisted) > 0 {
t.Errorf("test functions that write a wall reading into a stamp field (%d): use mono.Now() instead (the stamps are on its clock):\n %s", len(unlisted), strings.Join(unlisted, "\n "))
}
if len(stale) > 0 {
t.Errorf("stale pendingFixtures entries (%d): delete them:\n %s", len(stale), strings.Join(stale, "\n "))
}
}

// TestPendingIsEmptyWhenFlipped holds the flip back in code: with flipped true,
// no site and no fixture may be left to move.
func TestPendingIsEmptyWhenFlipped(t *testing.T) {
if flipped && (len(pending) != 0 || len(pendingFixtures) != 0) {
t.Fatalf("flipped is true with %d pending sites and %d pending fixtures: route them first", len(pending), len(pendingFixtures))
bad = append(bad, f)
}
if !flipped {
t.Logf("routing phase: %d sites and %d fixtures still to move", len(pending), len(pendingFixtures))
sort.Strings(bad)
if len(bad) > 0 {
t.Errorf("test functions that write a wall reading into a stamp field (%d): use mono.Now() instead (the stamps are on its clock):\n %s", len(bad), strings.Join(bad, "\n "))
}
}

// TestScanDump prints what the scan sees, as the tables' lines, for writing or
// updating them: MONO_SCAN_DUMP=1 go test -run TestScanDump -v ./internal/mono
// TestScanDump prints what the scan sees, as the table's lines, for writing or
// updating it: MONO_SCAN_DUMP=1 go test -run TestScanDump -v ./internal/mono
func TestScanDump(t *testing.T) {
if os.Getenv("MONO_SCAN_DUMP") == "" {
return // not a skip: CI's tally fails the step on any SKIP line
Expand Down
56 changes: 56 additions & 0 deletions start_time_1178_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
package celeris

import (
"context"
"testing"
"time"

"github.com/goceleris/celeris/internal/mono"
)

// celeris#1178. A request's start is a stamp of the engines' clock, internal/mono's.
// StartTime() is that instant as a time.Time: the TRUE wall time (the logger prints
// it as the request's timestamp) with a monotonic reading (the logger, metrics and
// otel take time.Since of it as the latency). A conversion that took the stamp for
// a Unix time (time.Unix(0, ns)) is wrong by the distance between the two clocks,
// which the skew canary makes an hour.
func TestStartTimeIsTheTrueWallTimeOfTheRequestsStamp1178(t *testing.T) {
srv := New(Config{})
got := make(chan time.Time, 1)
srv.GET("/s", func(c *Context) error {
got <- c.StartTime()
return nil
})
adapter := &routerAdapter{server: srv}
s, _ := newTestStream("GET", "/s")
defer s.Release()
s.StartTimeNs = mono.Now() // what an engine does
before := time.Now()
if err := adapter.HandleStream(context.Background(), s); err != nil {
t.Fatal(err)
}
select {
case st := <-got:
if d := st.Sub(before); d < -time.Second || d > time.Second {
t.Errorf("StartTime() is %v from the real time of the request", d)
}
if age := time.Since(st); age < 0 || age > 5*time.Second {
t.Errorf("time.Since(StartTime()) = %v, want a few ms", age)
}
case <-time.After(5 * time.Second):
t.Fatal("the handler did not run")
}
}

// BenchmarkStartTime1178 is StartTime() of a request an engine stamped: the lazy
// conversion the logger, metrics and otel pay once per request.
func BenchmarkStartTime1178(b *testing.B) {
b.ReportAllocs()
c := &Context{}
ns := mono.Now()
for range b.N {
c.startTime = time.Time{}
c.startTimeNs = ns
_ = c.StartTime()
}
}
Loading