From 896d8835bc2dda795f996a1f57f3d67ab0c97664 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Sat, 29 Aug 2026 10:39:21 -0700 Subject: [PATCH 1/3] docs(sender): dropped_at_handoff has a producer now Co-Authored-By: Claude Opus 5 (1M context) --- sender/doc.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sender/doc.go b/sender/doc.go index bfdb8cc..c4d6dad 100644 --- a/sender/doc.go +++ b/sender/doc.go @@ -108,8 +108,9 @@ // and accepted ⊆ succeeded (only a successful send is accepted). The left side // is accepted rather than registered, because a tx refused at the cap is // counted by dropped_at_cap and never entered the registry. -// dropped_at_handoff is a term of the identity and nothing produces it yet; the -// hand-off channel is what will. The inclusion +// dropped_at_handoff is what the sender could not hand over: Submit never +// blocks, so a full channel drops and counts rather than stalling the send path +// it was called from. The inclusion // denominator is succeeded (txs_accepted), never a minted "registered" series; // dropped_at_cap txs are excluded from it. inflight_at_shutdown is read only // after both the senders and the tracker have joined. From a394739c42718f65948eb081ad9e02eb977bf55b Mon Sep 17 00:00:00 2001 From: bdchatham Date: Sat, 29 Aug 2026 10:40:16 -0700 Subject: [PATCH 2/3] refactor(stats): rename InclusionTracker to OutcomeTracker The type stopped being about inclusion when it started reading receipts. It resolves a transaction to committed or reverted, and inclusion is one of the states it reports rather than the thing it measures. Type and filenames only. Log strings keep "inclusion tracker", because they are what an operator greps and what the runbooks name. Co-Authored-By: Claude Opus 5 (1M context) --- main.go | 4 +- sender/doc.go | 4 +- sender/sender_test.go | 4 +- sender/sharded_sender.go | 4 +- stats/doc.go | 6 +- stats/inclusion_run_test.go | 10 ++-- stats/metrics.go | 20 +++---- stats/outcome.go | 2 +- ...nclusion_tracker.go => outcome_tracker.go} | 58 +++++++++---------- ...racker_test.go => outcome_tracker_test.go} | 12 ++-- 10 files changed, 62 insertions(+), 62 deletions(-) rename stats/{inclusion_tracker.go => outcome_tracker.go} (95%) rename stats/{inclusion_tracker_test.go => outcome_tracker_test.go} (97%) diff --git a/main.go b/main.go index dc33378..6f4468a 100644 --- a/main.go +++ b/main.go @@ -232,7 +232,7 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error { logger := stats.NewLogger(collector, cfg.Settings.StatsInterval.ToDuration(), cfg.Settings.ReportPath, cfg.Settings.Debug) rng := generator.ResolveSeed(cfg) var ramper *sender.Ramper - inclusion := utils.None[*stats.InclusionTracker]() + inclusion := utils.None[*stats.OutcomeTracker]() err = scope.Run(ctx, func(ctx context.Context, s scope.Scope) error { // The generator deploys as it is built, so resolve who signs those @@ -293,7 +293,7 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error { // would all reap as expired and pollute the inclusion stats. if len(cfg.Endpoints) > 0 && cfg.Settings.TrackReceipts && !cfg.Settings.DryRun { reapAfter := cfg.Settings.InclusionReapAfter.ToDuration() - inclusionTracker := stats.NewInclusionTracker( + inclusionTracker := stats.NewOutcomeTracker( cfg.SeiChainID, reapAfter, inclusionRegistryCap(cfg.Settings.MaxInFlight, cfg.Settings.TPS, reapAfter), diff --git a/sender/doc.go b/sender/doc.go index c4d6dad..ad6b7f1 100644 --- a/sender/doc.go +++ b/sender/doc.go @@ -6,7 +6,7 @@ // the sender queue contains unsigned transactions; the sender loop stamps the // attempt, signs the tx, and calls the go-ethereum client (eth_sendRawTransaction). // Inclusion, when tracked, is observed by the -// block-indexed [stats.InclusionTracker] (see Inclusion stage below), not by +// block-indexed [stats.OutcomeTracker] (see Inclusion stage below), not by // per-tx receipt polling. A shared [golang.org/x/time/rate.Limiter] is // the single rate authority for the whole pipeline; the [Ramper] drives its // limit up or down via SetLimit. @@ -89,7 +89,7 @@ // # Inclusion stage // // When enabled (--track-receipts), the sender hands each successful send to the -// [stats.InclusionTracker] at send-completion (after OnComplete, only on a nil +// [stats.OutcomeTracker] at send-completion (after OnComplete, only on a nil // send error). The tracker subscribes to new heads, reads each arriving block's // receipts once (one request per block, whatever the block's tx count), and // stamps InclusionTime on every matched in-flight tx with the block's diff --git a/sender/sender_test.go b/sender/sender_test.go index 8509c76..895e8fa 100644 --- a/sender/sender_test.go +++ b/sender/sender_test.go @@ -144,7 +144,7 @@ func TestShardedSender_DryRunWithoutEndpoints(t *testing.T) { ChainID: 1, SeiChainID: "test-chain", Settings: &settings, - }, rate.NewLimiter(rate.Inf, 1), stats.NewCollector(), utils.None[*stats.InclusionTracker]()) + }, rate.NewLimiter(rate.Inf, 1), stats.NewCollector(), utils.None[*stats.OutcomeTracker]()) account := types.NewAccount(true) require.NoError(t, scope.Run(t.Context(), func(ctx context.Context, s scope.Scope) error { @@ -345,7 +345,7 @@ func newTestShardedSender(endpoints []string) *ShardedSender { ChainID: 1, Endpoints: endpoints, Settings: &settings, - }, rate.NewLimiter(rate.Inf, 1), stats.NewCollector(), utils.None[*stats.InclusionTracker]()) + }, rate.NewLimiter(rate.Inf, 1), stats.NewCollector(), utils.None[*stats.OutcomeTracker]()) } func testGeneratorConfigWithAccounts(endpoints []string, accountCount int, newAccountRate float64) *config.LoadConfig { diff --git a/sender/sharded_sender.go b/sender/sharded_sender.go index 7873940..dce47bd 100644 --- a/sender/sharded_sender.go +++ b/sender/sharded_sender.go @@ -24,12 +24,12 @@ type ShardedSender struct { queue *TxsQueue limiter *rate.Limiter // Shared rate limiter for transaction sending collector *stats.Collector - inclusion utils.Option[*stats.InclusionTracker] + inclusion utils.Option[*stats.OutcomeTracker] } // NewShardedSender creates a new sharded sender. // Txs of each shard are sent sequentially, using a single eth client. -func NewShardedSender(cfg *config.LoadConfig, limiter *rate.Limiter, collector *stats.Collector, inclusion utils.Option[*stats.InclusionTracker]) *ShardedSender { +func NewShardedSender(cfg *config.LoadConfig, limiter *rate.Limiter, collector *stats.Collector, inclusion utils.Option[*stats.OutcomeTracker]) *ShardedSender { return &ShardedSender{ cfg: cfg, queue: NewTxsQueue(cfg.Settings.MaxInFlight), diff --git a/stats/doc.go b/stats/doc.go index 1686b91..1feadb3 100644 --- a/stats/doc.go +++ b/stats/doc.go @@ -11,7 +11,7 @@ // report reads. // - Outcome — the six terminal states a transaction reaches, and the sentinel // that catches one nobody classified. -// - InclusionTracker — the registry of transactions sent and not yet +// - OutcomeTracker — the registry of transactions sent and not yet // accounted for, and the loops that match blocks and reap stragglers. // - RunSummary, InclusionSummary — the end-of-run tallies. // - BlockCollector — per-block chain data, gathered independently of the @@ -42,7 +42,7 @@ // RecordOutcome take it themselves; recordOperation is called with it // already held. // - TPSWindow.mu guards the rolling rate window. -// - InclusionTracker.state guards the in-flight registry and the watermarks a +// - OutcomeTracker.state guards the in-flight registry and the watermarks a // reap consults: the first height the run read, the highest head it has // taken off the wire, and the highest whose block it has read. // @@ -60,7 +60,7 @@ // and reads it after they join, so a read of its counts is only final at that // point. // -// InclusionTracker runs for the length of the run. Run dials the receipt +// OutcomeTracker runs for the length of the run. Run dials the receipt // endpoint and proves it answers. That proof can refuse the run: an endpoint // that never answers, refuses the method, or replies with something other than // JSON-RPC ends the run with ErrEndpointUnusable rather than letting it report diff --git a/stats/inclusion_run_test.go b/stats/inclusion_run_test.go index c6257c6..34ed207 100644 --- a/stats/inclusion_run_test.go +++ b/stats/inclusion_run_test.go @@ -17,7 +17,7 @@ import ( ) // headServer serves eth_subscribe("newHeads") over WebSocket, so a test can -// drive InclusionTracker.Run itself rather than the helpers underneath it. +// drive OutcomeTracker.Run itself rather than the helpers underneath it. // // Run is where the ordering lives: the pump stamps a head's arrival, the head // loop reads its block, the reap decides what to call a transaction, and a dead @@ -112,10 +112,10 @@ func (h *headServer) kill() { // and returns the tracker plus a stop function. func runTracker( t *testing.T, hs *headServer, src receiptSource, reapAfter time.Duration, -) (*InclusionTracker, context.CancelFunc, <-chan error) { +) (*OutcomeTracker, context.CancelFunc, <-chan error) { t.Helper() - tr := newInclusionTrackerWithSource( - NewInclusionTracker("test-chain", reapAfter, 1000, testTPS, true, NewCollector()), src) + tr := newOutcomeTrackerWithSource( + NewOutcomeTracker("test-chain", reapAfter, 1000, testTPS, true, NewCollector()), src) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) go func() { done <- tr.Run(ctx, hs.endpoint()) }() @@ -238,7 +238,7 @@ func TestADeadSubscriptionStopsTheRunSayingAnything(t *testing.T) { } // headsSeen reports the highest head the tracker has taken off the wire. -func headsSeen(tr *InclusionTracker) uint64 { +func headsSeen(tr *OutcomeTracker) uint64 { for s := range tr.state.Lock() { return s.headsReceived } diff --git a/stats/metrics.go b/stats/metrics.go index 6b6c51e..38cca04 100644 --- a/stats/metrics.go +++ b/stats/metrics.go @@ -108,17 +108,17 @@ var ( metric.WithUnit("{transactions}"))) ) -// meteredInclusionTrackers backs the inclusion_inflight gauge: each tracker +// meteredOutcomeTrackers backs the inclusion_inflight gauge: each tracker // registers so the callback can sample its in-flight map under lock. -var meteredInclusionTrackers = struct { +var meteredOutcomeTrackers = struct { lock sync.RWMutex - trackers []*InclusionTracker + trackers []*OutcomeTracker }{} -func meterInclusionInflight(t *InclusionTracker) { - meteredInclusionTrackers.lock.Lock() - defer meteredInclusionTrackers.lock.Unlock() - meteredInclusionTrackers.trackers = append(meteredInclusionTrackers.trackers, t) +func meterInclusionInflight(t *OutcomeTracker) { + meteredOutcomeTrackers.lock.Lock() + defer meteredOutcomeTrackers.lock.Unlock() + meteredOutcomeTrackers.trackers = append(meteredOutcomeTrackers.trackers, t) } func init() { @@ -127,9 +127,9 @@ func init() { metric.WithDescription("Current size of the inclusion tracker's in-flight tx registry"), metric.WithUnit("{transactions}"), metric.WithInt64Callback(func(_ context.Context, observer metric.Int64Observer) error { - meteredInclusionTrackers.lock.RLock() - defer meteredInclusionTrackers.lock.RUnlock() - for _, t := range meteredInclusionTrackers.trackers { + meteredOutcomeTrackers.lock.RLock() + defer meteredOutcomeTrackers.lock.RUnlock() + for _, t := range meteredOutcomeTrackers.trackers { for s := range t.state.Lock() { observer.Observe(int64(len(s.inflight)), metric.WithAttributes(attribute.String("chain_id", t.seiChainID))) diff --git a/stats/outcome.go b/stats/outcome.go index c45a483..1020778 100644 --- a/stats/outcome.go +++ b/stats/outcome.go @@ -2,7 +2,7 @@ package stats // Outcome is what became of one transaction the endpoint accepted. // -// InclusionTracker reports one of these for every transaction it registers, to +// OutcomeTracker reports one of these for every transaction it registers, to // the inclusion_outcome metric and to Collector.RecordOutcome. The run report // does not read the collector's counts yet. // diff --git a/stats/inclusion_tracker.go b/stats/outcome_tracker.go similarity index 95% rename from stats/inclusion_tracker.go rename to stats/outcome_tracker.go index bdda29f..42a191f 100644 --- a/stats/inclusion_tracker.go +++ b/stats/outcome_tracker.go @@ -185,12 +185,12 @@ type inclusionState struct { inflightAtShutdown uint64 } -// InclusionTracker matches arriving blocks against in-flight txs, stamps +// OutcomeTracker matches arriving blocks against in-flight txs, stamps // InclusionTime, and resolves each one to a terminal [Outcome]. // // sender/doc.go owns the conservation identity these outcomes satisfy, and it is // not restated here. -type InclusionTracker struct { +type OutcomeTracker struct { seiChainID string reapAfter time.Duration maxInflight int @@ -214,7 +214,7 @@ type InclusionTracker struct { // the metric SDK's collection goroutine. It takes no collector lock, and // the rule above is why it must not start. // - // Never nil. NewInclusionTracker takes it because every run has one. + // Never nil. NewOutcomeTracker takes it because every run has one. collector *Collector } @@ -243,18 +243,18 @@ func handoffDepth(tps float64) int { // crash reapLoop. Defense-in-depth; the config default is already 30s. const defaultInclusionReapAfter = 30 * time.Second -// NewInclusionTracker builds a tracker bounded at maxInflight in-flight txs that +// NewOutcomeTracker builds a tracker bounded at maxInflight in-flight txs that // reaps un-included txs after reapAfter. openLoop gates the inclusion_latency // sample (included/expired counts are tracked in both models). The block source -// is the production ethclient impl; tests inject via newInclusionTrackerWithSource. -func NewInclusionTracker(seiChainID string, reapAfter time.Duration, maxInflight int, tps float64, openLoop bool, collector *Collector) *InclusionTracker { +// is the production ethclient impl; tests inject via newOutcomeTrackerWithSource. +func NewOutcomeTracker(seiChainID string, reapAfter time.Duration, maxInflight int, tps float64, openLoop bool, collector *Collector) *OutcomeTracker { if maxInflight <= 0 { maxInflight = defaultMaxInflight } if reapAfter <= 0 { reapAfter = defaultInclusionReapAfter } - t := &InclusionTracker{ + t := &OutcomeTracker{ seiChainID: seiChainID, reapAfter: reapAfter, maxInflight: maxInflight, @@ -269,7 +269,7 @@ func NewInclusionTracker(seiChainID string, reapAfter time.Duration, maxInflight return t } -func newInclusionTrackerWithSource(t *InclusionTracker, source receiptSource) *InclusionTracker { +func newOutcomeTrackerWithSource(t *OutcomeTracker, source receiptSource) *OutcomeTracker { t.source = source return t } @@ -283,7 +283,7 @@ func newInclusionTrackerWithSource(t *InclusionTracker, source receiptSource) *I // hand-off that blocks brings back the stall it exists to remove, and one that // drops in silence leaves an accepted transaction with no terminal state // (TOT-014). -func (t *InclusionTracker) Submit(ctx context.Context, tx *types.LoadTx) { +func (t *OutcomeTracker) Submit(ctx context.Context, tx *types.LoadTx) { select { case t.submit <- tx: default: @@ -301,7 +301,7 @@ func (t *InclusionTracker) Submit(ctx context.Context, tx *types.LoadTx) { // head loop would inherit that loop's block read, and no depth absorbs a // multi-second stall at a few thousand transactions per second. Admission alone // stalls for as long as the registry lock is held, which is microseconds. -func (t *InclusionTracker) drainLoop(ctx context.Context) error { +func (t *OutcomeTracker) drainLoop(ctx context.Context) error { for { tx, err := utils.Recv(ctx, t.submit) if err != nil { @@ -314,7 +314,7 @@ func (t *InclusionTracker) drainLoop(ctx context.Context) error { // admit places one transaction in the registry. It runs on the drain goroutine // alone, so the checks below need no ordering beyond the registry lock they // already take. At cap the tx is dropped and counted. -func (t *InclusionTracker) admit(ctx context.Context, tx *types.LoadTx) { +func (t *OutcomeTracker) admit(ctx context.Context, tx *types.LoadTx) { hash := tx.EthTx.Hash() var outcome Outcome for s := range t.state.Lock() { @@ -362,7 +362,7 @@ func (t *InclusionTracker) admit(ctx context.Context, tx *types.LoadTx) { // value this label can take. Passing a literal here would leave the metric and // Outcome.String() as two independent sources for one wire contract, and a // rename of either would orphan a dashboard while the other's test stayed green. -func (t *InclusionTracker) meterOutcome(ctx context.Context, outcome Outcome, scenario *types.TxScenario) { +func (t *OutcomeTracker) meterOutcome(ctx context.Context, outcome Outcome, scenario *types.TxScenario) { inclusionOutcome.Add(ctx, 1, metric.WithAttributes( attribute.String("chain_id", t.seiChainID), attribute.String("scenario", scenario.Name), @@ -386,7 +386,7 @@ func (t *InclusionTracker) meterOutcome(ctx context.Context, outcome Outcome, sc // the node has not finished writing it. It never asks for an older height, so // the serving node's receipt retention does not bound it. A change that reaches // further back does. -func (t *InclusionTracker) Run(ctx context.Context, endpoint string) error { +func (t *OutcomeTracker) Run(ctx context.Context, endpoint string) error { wsEndpoint := utils.GetWSEndpoint(endpoint) if t.source == nil { client, err := ethclient.Dial(endpoint) @@ -468,7 +468,7 @@ func (t *InclusionTracker) Run(ctx context.Context, endpoint string) error { // processHead handles one arriving head: counts any gap (no backfill), matches // the block, and returns the new lastSeen. lastSeen==0 seeds on the first head. -func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64, arrival time.Time, lastSeen uint64) uint64 { +func (t *OutcomeTracker) processHead(ctx context.Context, num, gasUsed uint64, arrival time.Time, lastSeen uint64) uint64 { if lastSeen != 0 && num <= lastSeen { return lastSeen // duplicate or out-of-order head: no re-fetch, no spurious gap. } @@ -496,7 +496,7 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64, // tracker whose reads outrun the block interval builds a backlog, and a stamp // taken at processing would carry that backlog into InclusionTime and report // this run's own lateness as the chain's inclusion latency. -func (t *InclusionTracker) pumpHeads( +func (t *OutcomeTracker) pumpHeads( ctx context.Context, headers <-chan *ethtypes.Header, arrivals chan<- headArrival, ) error { for ctx.Err() == nil { @@ -519,7 +519,7 @@ func (t *InclusionTracker) pumpHeads( // noteHeadReceived records that a head reached this process, before anything // reads its block. -func (t *InclusionTracker) noteHeadReceived(num uint64) { +func (t *OutcomeTracker) noteHeadReceived(num uint64) { for s := range t.state.Lock() { if num > s.headsReceived { s.headsReceived = num @@ -528,7 +528,7 @@ func (t *InclusionTracker) noteHeadReceived(num uint64) { } // noteHeadResolved records that a head's block has been read. -func (t *InclusionTracker) noteHeadResolved(num uint64) { +func (t *OutcomeTracker) noteHeadResolved(num uint64) { for s := range t.state.Lock() { if num > s.headsResolved { s.headsResolved = num @@ -543,7 +543,7 @@ func (t *InclusionTracker) noteHeadResolved(num uint64) { // reaps after this: without a drain the registry keeps filling from senders that // are still working, and every later transaction reports dropped_at_cap. An // operator reads that as "raise the cap" rather than "the subscription died". -func (t *InclusionTracker) stopTracking(ctx context.Context) { +func (t *OutcomeTracker) stopTracking(ctx context.Context) { var stranded []resolvedOutcome for s := range t.state.Lock() { s.trackingStopped = true @@ -573,7 +573,7 @@ func (t *InclusionTracker) stopTracking(ctx context.Context) { // nothing about the node's sync state: a node merely behind still passes. // Nothing but a broken endpoint fails it, which is why any error that survives // every attempt refuses the run and no error classifier is needed here. -func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error { +func (t *OutcomeTracker) preflight(ctx context.Context, endpoint string) error { // Several attempts, because one verdict stands for the whole run. A single // reset or timeout from a node that is merely busy must not end a run before // it starts. @@ -608,7 +608,7 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error // a missing block. Retrying in place costs a phase shift and no backlog: the // wait for height N ends when N becomes readable, which on a healthy node is // long before head N+1 arrives. -func (t *InclusionTracker) readReceipts( +func (t *OutcomeTracker) readReceipts( ctx context.Context, num uint64, budget time.Duration, ) (receipts []blockReceipt, nulls int, waited time.Duration, err error) { readCtx, cancel := context.WithTimeout(ctx, budget) @@ -632,7 +632,7 @@ func (t *InclusionTracker) readReceipts( // through a watermark that also waits on the receipt store's async writer, so a // height can answer null for the moment between the two. readReceipts waits that // out; anything the read budget does not resolve is a hole. -func (t *InclusionTracker) matchBlock(ctx context.Context, num, gasUsed uint64, arrival time.Time) { +func (t *OutcomeTracker) matchBlock(ctx context.Context, num, gasUsed uint64, arrival time.Time) { // TOT-023. What is left of this height's deadline is what the read gets, and // a height with nothing left is not read at all: every transaction it could // carry has already reaped, so the read answers a question the run has @@ -783,7 +783,7 @@ type resolvedOutcome struct { // The run report does not read those counts yet. // // Call it outside the registry lock. -func (t *InclusionTracker) report(ctx context.Context, outcome Outcome, scenario *types.TxScenario) { +func (t *OutcomeTracker) report(ctx context.Context, outcome Outcome, scenario *types.TxScenario) { t.meterOutcome(ctx, outcome, scenario) t.collector.RecordOutcome(OperationKey{ Scenario: scenario.Name, @@ -801,7 +801,7 @@ type inclusionSample struct { // reapLoop sweeps every reapAfter; worst-case eviction latency is ~2×reapAfter // (a tx registered just after a tick waits a full period for the next sweep) — // a calibration nuance, not a conservation concern. -func (t *InclusionTracker) reapLoop(ctx context.Context) error { +func (t *OutcomeTracker) reapLoop(ctx context.Context) error { ticker := time.NewTicker(t.reapAfter) defer ticker.Stop() for ctx.Err() == nil { @@ -817,7 +817,7 @@ func (t *InclusionTracker) reapLoop(ctx context.Context) error { // reap evicts txs in-flight longer than reapAfter as expired. Delete-on-touch // under the lock races safely against matchBlock: whoever holds the lock first // wins, no double count. -func (t *InclusionTracker) reap(ctx context.Context) { +func (t *OutcomeTracker) reap(ctx context.Context) { cutoff := time.Now().Add(-t.reapAfter) var expired []resolvedOutcome for s := range t.state.Lock() { @@ -863,7 +863,7 @@ func (t *InclusionTracker) reap(ctx context.Context) { // recordBlindGap marks that the run never saw the heads from first to last, so // it never read those blocks. -func (t *InclusionTracker) recordBlindGap(ctx context.Context, first, last uint64) { +func (t *OutcomeTracker) recordBlindGap(ctx context.Context, first, last uint64) { for s := range t.state.Lock() { s.blindHeights++ } @@ -885,7 +885,7 @@ func (t *InclusionTracker) recordBlindGap(ctx context.Context, first, last uint6 // past it until it catches up, and at a sub-second block interval one line per // height would push the run's own summary out of any bounded log tail — the // same reason recordBlindGap logs a range once. -func (t *InclusionTracker) skipPastDeadline(ctx context.Context, num uint64) { +func (t *OutcomeTracker) skipPastDeadline(ctx context.Context, num uint64) { var entered bool for s := range t.state.Lock() { s.blindHeights++ @@ -906,7 +906,7 @@ func (t *InclusionTracker) skipPastDeadline(ctx context.Context, num uint64) { // recordBlindFetch marks that the run could not read one block's receipts. Every // tx in flight now reaps as status_unavailable rather than expired, because the // run cannot tell the two apart for a block it never read. -func (t *InclusionTracker) recordBlindFetch(ctx context.Context, num uint64, reason string, err error) { +func (t *OutcomeTracker) recordBlindFetch(ctx context.Context, num uint64, reason string, err error) { for s := range t.state.Lock() { s.blindHeights++ } @@ -969,7 +969,7 @@ const minReadBudget = notFoundBackoff // gives each later head a smaller budget, and once a head has none left it // costs no request at all, so the loop catches up rather than paying readTimeout // per height forever. -func (t *InclusionTracker) readBudget(arrival time.Time) time.Duration { +func (t *OutcomeTracker) readBudget(arrival time.Time) time.Duration { return min(readTimeout, time.Until(arrival.Add(t.reapAfter))) } @@ -1062,7 +1062,7 @@ type InclusionSummary struct { } // Summary snapshots the final tally; call once at shutdown after joins. -func (t *InclusionTracker) Summary() InclusionSummary { +func (t *OutcomeTracker) Summary() InclusionSummary { for s := range t.state.Lock() { s.inflightAtShutdown = uint64(len(s.inflight)) return InclusionSummary{ diff --git a/stats/inclusion_tracker_test.go b/stats/outcome_tracker_test.go similarity index 97% rename from stats/inclusion_tracker_test.go rename to stats/outcome_tracker_test.go index 46c9187..ce60811 100644 --- a/stats/inclusion_tracker_test.go +++ b/stats/outcome_tracker_test.go @@ -115,7 +115,7 @@ func (m *MockBlockSource) FetchCount() int64 { return m.fetches.Load() } // newTestTracker builds an open-loop tracker wired to a mock source, skipping // the live dial. Open-loop is the default so latency-bearing tests exercise the // inclusion_latency path; closed-loop is covered explicitly via newTestTrackerLoop. -func newTestTracker(t *testing.T, reapAfter time.Duration, maxInflight int, src receiptSource) *InclusionTracker { +func newTestTracker(t *testing.T, reapAfter time.Duration, maxInflight int, src receiptSource) *OutcomeTracker { t.Helper() return newTestTrackerLoop(t, reapAfter, maxInflight, src, true) } @@ -128,10 +128,10 @@ func newTestTracker(t *testing.T, reapAfter time.Duration, maxInflight int, src // happens once the run is underway, so they start there. // TestATxAcceptedBeforeTheFirstReadIsNotAChainVerdict covers the other case // through Run itself. -func newTestTrackerLoop(t *testing.T, reapAfter time.Duration, maxInflight int, src receiptSource, openLoop bool) *InclusionTracker { +func newTestTrackerLoop(t *testing.T, reapAfter time.Duration, maxInflight int, src receiptSource, openLoop bool) *OutcomeTracker { t.Helper() - tr := newInclusionTrackerWithSource( - NewInclusionTracker("test-chain", reapAfter, maxInflight, testTPS, openLoop, NewCollector()), src) + tr := newOutcomeTrackerWithSource( + NewOutcomeTracker("test-chain", reapAfter, maxInflight, testTPS, openLoop, NewCollector()), src) for s := range tr.state.Lock() { s.firstReadAt = time.Now().Add(-time.Hour) } @@ -162,7 +162,7 @@ func sentAt() time.Time { return time.Now().Add(-fixtureLatency) } // reapAfter is also the deadline matchBlock reads a height inside, so a tracker // built to reap instantly is a tracker that reads nothing, and a test that wants // both would assert against a block the tracker skipped. -func expireInflight(t *testing.T, tr *InclusionTracker) { +func expireInflight(t *testing.T, tr *OutcomeTracker) { t.Helper() for s := range tr.state.Lock() { for _, e := range s.inflight { @@ -199,7 +199,7 @@ func loadTx(nonce uint64, intended time.Time) *types.LoadTx { } } -func inflightLen(t *testing.T, tr *InclusionTracker) int { +func inflightLen(t *testing.T, tr *OutcomeTracker) int { t.Helper() for s := range tr.state.Lock() { return len(s.inflight) From 10e1a5ff5afaf90f8a5c54c6679c3c04647a791b Mon Sep 17 00:00:00 2001 From: bdchatham Date: Sat, 29 Aug 2026 10:47:47 -0700 Subject: [PATCH 3/3] test(main): pin which node the tracker reads from T018 and T020, the last two open tasks in the spec. The behaviour was already right and nothing held it there. Extracting the choice into trackingEndpoint makes both halves of TOT-021 testable: reads land on a peer when one is configured, and fall back to the load node when it is the only one, saying so rather than competing with the workload in silence. TOT-022 follows from the shape. The tracker takes one endpoint and derives the head stream from it, so a caller cannot take heads from one node and status from another. Co-Authored-By: Claude Opus 5 (1M context) --- main.go | 20 ++++++++++++++++--- main_test.go | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 6f4468a..00430e2 100644 --- a/main.go +++ b/main.go @@ -309,9 +309,8 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error { // and a node trailing its peers by less than the readiness threshold // stays in service while it does. Reading both from one node makes // the run internally consistent, which is what TOT-022 requires. - trackingEndpoint := cfg.ReceiptEndpoint - if trackingEndpoint == "" { - trackingEndpoint = cfg.Endpoints[0] + trackingEndpoint, shared := trackingEndpoint(cfg) + if shared { log.Printf("⚠️ Tracking from the load endpoint %s. Set "+ "receiptEndpoint to a node taking no send load: a receipts "+ "read costs the serving node work that grows with the block's "+ @@ -461,6 +460,21 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error { return err } +// trackingEndpoint picks the node the tracker reads status and heads from, and +// reports whether that is the same node the run sends load to. +// +// Both signals come from this one value, so the caller cannot take heads from +// one node and status from another (TOT-022). Falling back to the load endpoint +// is what makes the tracker work on a single-node deployment (TOT-021); the +// caller says so out loud, because the reads then land on the system under test +// and count against the capacity the run exists to measure. +func trackingEndpoint(cfg *config.LoadConfig) (endpoint string, sharedWithLoad bool) { + if cfg.ReceiptEndpoint != "" { + return cfg.ReceiptEndpoint, false + } + return cfg.Endpoints[0], true +} + // endedOnRunContext reports whether err is just the run finishing: its duration // elapsed, or the operator signalled it. Both are success. // diff --git a/main_test.go b/main_test.go index 7ecdb88..94b6b94 100644 --- a/main_test.go +++ b/main_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "github.com/sei-protocol/sei-load/config" + "github.com/sei-protocol/sei-load/utils" "os" "path/filepath" "testing" @@ -55,3 +57,57 @@ func TestEndedOnRunContext(t *testing.T) { }) } } + +// TestStatusReadsLandOnTheTrackingNode covers T018 / TOT-021. +// +// The tracker's own reads otherwise land on the system under test, where they +// count against the capacity the run exists to measure. A receipts call costs +// the serving node work that grows with the block's transaction count, which is +// exactly what a load run makes large. +func TestStatusReadsLandOnTheTrackingNode(t *testing.T) { + got, shared := trackingEndpoint(&config.LoadConfig{ + Endpoints: []string{"http://load-node:8545"}, + ReceiptEndpoint: "http://peer-node:8545", + }) + + require.Equal(t, "http://peer-node:8545", got, + "the tracker read from the node under load while a peer holding the same blocks sat idle") + require.False(t, shared) +} + +// TestOneNodeStillWorks is the other half of TOT-021: the run MUST still work +// when only one node is available. A tracker that refused to start without a +// second endpoint would make outcome tracking unavailable on every single-node +// deployment, which is most of them. +func TestOneNodeStillWorks(t *testing.T) { + got, shared := trackingEndpoint(&config.LoadConfig{ + Endpoints: []string{"http://only-node:8545"}, + }) + + require.Equal(t, "http://only-node:8545", got) + require.True(t, shared, + "the run took its reads from the load node without saying so, so nobody knows the "+ + "measurement is competing with the workload") +} + +// TestBothSignalsShareOneNode covers T020 / TOT-022. +// +// A head notification carries the raw committed height. A status read resolves +// through a watermark behind it. The two disagree on one node already, and +// taking them from different nodes adds peer lag on top, which can reach minutes +// while both nodes still report healthy. +// +// The tracker takes one endpoint and derives the head stream from it, so the +// caller cannot split them. This pins that: the WebSocket endpoint follows the +// tracking node, never the load node. +func TestBothSignalsShareOneNode(t *testing.T) { + tracking, _ := trackingEndpoint(&config.LoadConfig{ + Endpoints: []string{"http://load-node:8545"}, + ReceiptEndpoint: "http://peer-node:8545", + }) + + require.Equal(t, "ws://peer-node:8546", utils.GetWSEndpoint(tracking), + "the head stream came from a different node than the status reads") + require.NotEqual(t, utils.GetWSEndpoint("http://load-node:8545"), utils.GetWSEndpoint(tracking), + "the two nodes resolve to one WebSocket endpoint, so this test proves nothing") +}