feat(stats): read execution status from receipts, not hashes

README.md

@@ -21,6 +21,7 @@ Edit `my-config.json`:
 ```json
 {
   "endpoints": ["http://localhost:8545"],
+  "receiptEndpoint": "http://localhost:8546",
   "chainId": 713714,
   "scenarios": [
     {"name": "EVMTransfer", "weight": 100}
@@ -38,6 +39,21 @@ Edit `my-config.json`:
 }
 ```
 
+`endpoints` take the load. `receiptEndpoint` is the node the inclusion tracker
+uses, and it should be a node taking no send load. The tracker takes both its
+head subscription and its receipt reads from that one node: a head carries the
+raw committed height while a status read resolves through a watermark behind it,
+so heads from one node and reads from another disagree by that pair's peer lag. On seid, a
+receipts read costs the serving node work that grows with the square of the
+block's transaction count, so a node doing both degrades, and the tracker
+reports that degradation as a chain result. Leave it out and the tracker reads
+from `endpoints[0]`, which is fine for a local chain and not for a load run.
+
+Time `eth_getBlockReceipts` against the chain you are testing, on a block as
+wide as the run will produce, before you turn `trackReceipts` on. A read slower
+than the block interval makes the tracker blind, and a blind tracker reports
+`status_unavailable` rather than a number.
+
 ### 3. Run
 
 ```bash
@@ -54,7 +70,7 @@ Edit `my-config.json`:
 | `--buffer-size, -b` | 1000 | Sender queue size |
 | `--dry-run` | false | Simulate without sending |
 | `--debug` | false | Log each transaction |
-| `--track-receipts` | false | Enable the block-indexed tx→inclusion tracker (stamps InclusionTime; reports included/expired/inflight-at-shutdown) |
+| `--track-receipts` | false | Read each block's receipts and record what the chain did with every transaction: committed, reverted, expired, status-unavailable, dropped-at-cap, or still in flight. The counts reach the `inclusion_outcome` metric; the closing log line carries the inclusion tally. The run report does not carry them yet. Set `receiptEndpoint` in the config with it |
 | `--inclusion-reap-after` | 30s | How long an un-included tx waits before reaping as expired (tune to expected inclusion time on congested chains) |
 | `--track-blocks` | false | Track block statistics |
 | `--track-user-latency` | false | Track user latency metrics |

config/config.go

@@ -31,12 +31,28 @@ type LoadConfig struct {
 	// operator reviews and commits. Empty writes nothing.
 	ChainRecordPath string `json:"chainRecordPath,omitempty"`
 	// SeiChainID is the textual chain ID used for tagging metric collection.
-	SeiChainID string         `json:"seiChainID,omitempty"`
-	Endpoints  []string       `json:"endpoints"`
-	Accounts   *AccountConfig `json:"accounts,omitempty"`
-	Scenarios  []Scenario     `json:"scenarios,omitempty"`
-	MockDeploy bool           `json:"mockDeploy,omitempty"`
-	Settings   *Settings      `json:"settings,omitempty"`
+	SeiChainID string   `json:"seiChainID,omitempty"`
+	Endpoints  []string `json:"endpoints"`
+	// ReceiptEndpoint is the node the inclusion tracker uses. It takes both the
+	// head subscription and the receipt reads from it, because a head from one
+	// node and a status read from another disagree by that pair's peer lag.
+	// Empty uses Endpoints[0].
+	//
+	// Name a node that takes no send load. On seid, eth_getBlockReceipts costs
+	// the node answering it work that grows with the square of the block's
+	// transaction count. A node that both takes the send load and answers that
+	// read degrades, and the tracker reports the degradation as a chain result.
+	//
+	// This bounds what the tracking node can keep up with. Time
+	// eth_getBlockReceipts against the target chain, on a block as wide as the
+	// run will produce, before enabling trackReceipts. A read slower than the
+	// chain's block interval makes the tracker blind, and a blind tracker
+	// reports status_unavailable rather than a number.
+	ReceiptEndpoint string         `json:"receiptEndpoint,omitempty"`
+	Accounts        *AccountConfig `json:"accounts,omitempty"`
+	Scenarios       []Scenario     `json:"scenarios,omitempty"`
+	MockDeploy      bool           `json:"mockDeploy,omitempty"`
+	Settings        *Settings      `json:"settings,omitempty"`
 	// Funding, when set, funds the generated account pool from a root key at
 	// startup so the run works against a real chain. See funding.go.
 	Funding *FundingConfig `json:"funding,omitempty"`

main.go

@@ -288,8 +288,7 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 			})
 		}
 
-		// The --track-receipts flag now enables the block-indexed inclusion
-		// tracker (the lossy per-tx receipt path is retired).
+		// --track-receipts enables the block-indexed inclusion tracker.
 		// Not wired under --dry-run: simulated sends never hit the chain, so they
 		// would all reap as expired and pollute the inclusion stats.
 		if len(cfg.Endpoints) > 0 && cfg.Settings.TrackReceipts && !cfg.Settings.DryRun {
@@ -299,10 +298,26 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 				reapAfter,
 				inclusionRegistryCap(cfg.Settings.MaxInFlight, cfg.Settings.TPS, reapAfter),
 				cfg.Settings.ArrivalModel == config.ArrivalModelOpenLoop,
+				collector,
 			)
 			inclusion = utils.Some(inclusionTracker)
+			// Heads and receipts come from the same node, always. A head
+			// notification carries the raw committed height while a status read
+			// resolves through a watermark behind it, so the two disagree even on
+			// one node; taking them from different nodes adds peer lag on top,
+			// and a node trailing its peers by less than the readiness threshold
+			// stays in service while it does. Reading both from one node makes
+			// the run internally consistent, which is what TOT-022 requires.
+			trackingEndpoint := cfg.ReceiptEndpoint
+			if trackingEndpoint == "" {
+				trackingEndpoint = cfg.Endpoints[0]
+				log.Printf("⚠️  Tracking from the load endpoint %s. Set "+
+					"receiptEndpoint to a node taking no send load: a receipts "+
+					"read costs the serving node work that grows with the block's "+
+					"transaction count.", trackingEndpoint)
+			}
 			s.SpawnBgNamed("inclusion tracker", func() error {
-				return inclusionTracker.Run(ctx, cfg.Endpoints[0])
+				return inclusionTracker.Run(ctx, trackingEndpoint)
 			})
 		}
 
@@ -419,10 +434,16 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 		summary.InclusionTracked = true
 		summary.Included = incl.Included
 		summary.Expired = incl.Expired
+		summary.StatusUnavailable = incl.StatusUnavailable
 		summary.DroppedAtCap = incl.DroppedAtCap
 		summary.InflightAtShutdown = incl.InflightAtShutdown
-		log.Printf("📦 Inclusion: included=%d expired=%d dropped_at_cap=%d inflight_at_shutdown=%d",
-			incl.Included, incl.Expired, incl.DroppedAtCap, incl.InflightAtShutdown)
+		log.Printf("📦 Inclusion: included=%d expired=%d status_unavailable=%d dropped_at_cap=%d inflight_at_shutdown=%d",
+			incl.Included, incl.Expired, incl.StatusUnavailable, incl.DroppedAtCap,
+			incl.InflightAtShutdown)
+		if incl.StatusUnavailable > 0 {
+			log.Printf("⚠️  %d txs were in flight while a receipt read failed. "+
+				"The run cannot say whether the chain took them.", incl.StatusUnavailable)
+		}
 	}
 	collector.EmitRunSummary(ctx, summary)
 	if d := cfg.Settings.PostSummaryFlushDelay.ToDuration(); d > 0 {

sender/doc.go

@@ -90,29 +90,52 @@
 //
 // When enabled (--track-receipts), the sender hands each successful send to the
 // [stats.InclusionTracker] at send-completion (after OnComplete, only on a nil
-// send error). The tracker subscribes to new heads, fetches each arriving
-// block's body once (O(blocks), not O(txs)), and stamps InclusionTime on every
-// matched in-flight tx with the block's header-ARRIVAL time. Un-included txs are
-// reaped as expired after reapAfter. inclusion_latency (arrival minus
+// send error). The tracker subscribes to new heads, reads each arriving block's
+// receipts once (one request per block, whatever the block's tx count), and
+// stamps InclusionTime on every matched in-flight tx with the block's
+// header-ARRIVAL time. A receipt carries the execution status, so a matched tx
+// resolves to committed or reverted rather than to one state covering both.
+// Un-included txs are reaped after reapAfter. inclusion_latency (arrival minus
 // IntendedSendTime) is an open-loop-only measure; in closed-loop IntendedSendTime
 // is enqueue time, so the latency sample is omitted (counts are tracked in both).
 //
-// Conservation. registered == included + expired + inflight_at_shutdown, and
-// registered ⊆ succeeded (only successful sends are registered). The inclusion
+// Conservation. Over [stats.Outcome]'s terminal states,
+//
+//	accepted == committed + reverted + status_unavailable
+//	          + expired + dropped_at_cap + dropped_at_handoff
+//	          + inflight_at_shutdown
+//
+// and accepted ⊆ succeeded (only a successful send is accepted). The left side
+// is accepted rather than registered, because a tx refused at the cap is
+// counted by dropped_at_cap and never entered the registry.
+// dropped_at_handoff is a term of the identity and nothing produces it yet; the
+// hand-off channel is what will. The inclusion
 // denominator is succeeded (txs_accepted), never a minted "registered" series;
 // dropped_at_cap txs are excluded from it. inflight_at_shutdown is read only
 // after both the senders and the tracker have joined.
 //
 // Accepted boundaries. (1) WS gaps degrade conservatively: a missed head is
-// counted (block_gaps) but never backfilled, so its txs reap as expired —
-// an undercount of inclusions, never a miscount. (2) Reorgs use
-// first-observation-wins (stamp + delete); the inclusion-time error is bounded
-// by reorg_depth × block_time, with no canonical reconciliation. (3) A single
-// fetch endpoint (Endpoints[0], shared with the block collector) adds a small
-// read load. (4) InclusionTime is the header-arrival wall clock, not fetch
-// completion and not header.Time. (5) A failed block-body fetch is counted
-// (block_fetch_errors) and not retried — that block's txs reap as expired, the
-// same conservative undercount as a WS gap. (6) A tx registered after its
+// counted (block_gaps) and never backfilled. The run never read those blocks,
+// so a tx in flight across the gap reaches status_unavailable rather than
+// expired: expired is a claim about the chain, and the run has no grounds for
+// one about a block it did not read. (2) Reorgs use
+// first-observation-wins (stamp + delete). This fixes an execution status as
+// well as a time: a tx that reverted on an orphaned block and committed on the
+// canonical one keeps the first answer. Sei's finality makes the window small,
+// and there is no canonical reconciliation. (3) The tracker reads receipts from
+// receiptEndpoint when a run names one, and from Endpoints[0] otherwise. A
+// receipts read costs the serving node work that grows with the block's tx
+// count, so a run at high TPS should name a second node. (4) InclusionTime is
+// the header-arrival wall clock, not fetch completion and not header.Time.
+// (5) A height the serving node has not reached is read again until a budget
+// runs out, because a receipt node trailing the head node is ordinary. One head
+// spends a bounded total on those re-reads, so a height the sweep does not reach
+// waits for the next head rather than becoming a hole. Any other
+// read failure is counted by reason (block_fetch_errors) and not retried, since
+// a retry piles requests onto an endpoint already failing. Either way a tx in
+// flight across an unread height reaches status_unavailable rather than expired.
+// An empty array is not an unread height: a block carrying no EVM transaction
+// answers that way, and most blocks on an idle chain do. (6) A tx registered after its
 // including block was already scanned is missed and reaps as expired — bounded
 // by the microsecond register window versus block time, a rare conservative
 // undercount, the same direction as a WS gap.

sender/sharded_sender.go

@@ -124,7 +124,7 @@ func (ss *ShardedSender) Run(ctx context.Context) error {
 
 				// Queue for inclusion check.
 				if inclusion, ok := ss.inclusion.Get(); ok {
-					inclusion.Register(tx)
+					inclusion.Register(ctx, tx)
 				}
 				ss.queue.PopSent(addr)
 				return nil

stats/collector.go

@@ -3,6 +3,7 @@ package stats
 import (
 	"cmp"
 	"fmt"
+	"log"
 	"slices"
 	"sort"
 	"sync"
@@ -13,6 +14,10 @@ import (
 type Collector struct {
 	mu sync.RWMutex
 
+	// loggedBadOutcome makes RecordOutcome's complaint fire once per run. A
+	// systematic classification bug puts every transaction through that branch.
+	loggedBadOutcome bool
+
 	// Transaction counts by scenario
 	txCounts map[string]uint64
 
@@ -88,6 +93,42 @@ func (c *Collector) RecordTransaction(scenario, operation string, latency time.D
 	c.recordWindowStats(latency)
 }
 
+// RecordOutcome counts one terminal outcome under the key the send path already
+// labels its metrics with. Call it once per transaction, for that transaction's
+// terminal state only. It adds rather than overwrites, because many
+// transactions share one key.
+//
+// Call it outside the tracker's registry lock. This method takes the collector
+// mutex, and the sender blocks on the registry lock at every send completion, so
+// a call made under both puts collector contention into the latency this package
+// reports. It is safe for concurrent callers; the nesting is what is not.
+//
+// An unset or out-of-range outcome counts as Unrecorded rather than vanishing,
+// so the conservation identity stays closed and the defect stays visible. The
+// first one also logs. A run never fails over a counting bug, and a systematic
+// one would otherwise write a line per transaction.
+func (c *Collector) RecordOutcome(key OperationKey, outcome Outcome) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if outcome == outcomeUnset || outcome >= outcomeCount {
+		if !c.loggedBadOutcome {
+			c.loggedBadOutcome = true
+			log.Printf("stats: outcome %d for %s/%s is not a terminal state; "+
+				"counting it as %s. This is a bug in sei-load.",
+				outcome, key.Scenario, key.Operation, outcomeNames[outcomeUnset])
+		}
+		outcome = outcomeUnset
+	}
+
+	samples := c.perOperation[key]
+	if samples == nil {
+		samples = &operationSamples{}
+		c.perOperation[key] = samples
+	}
+	samples.outcomes[outcome]++
+}
+
 // recordOperation counts one attempt for key and, on success, adds its latency
 // to that operation's samples. The bound is the same one recordLatency applies
 // to the pooled window.
@@ -269,10 +310,17 @@ func (c *Collector) GetOperationStats() map[OperationKey]OperationStats {
 	out := make(map[OperationKey]OperationStats, len(c.perOperation))
 	for key, samples := range c.perOperation {
 		op := OperationStats{
-			Count:       samples.count,
-			Successes:   samples.successes,
-			SampleCount: len(samples.samples),
-			Window:      samples.window(),
+			Count:             samples.count,
+			Successes:         samples.successes,
+			SampleCount:       len(samples.samples),
+			Window:            samples.window(),
+			Committed:         samples.outcomes[OutcomeCommitted],
+			Reverted:          samples.outcomes[OutcomeReverted],
+			Expired:           samples.outcomes[OutcomeExpired],
+			DroppedAtCap:      samples.outcomes[OutcomeDroppedAtCap],
+			DroppedAtHandoff:  samples.outcomes[OutcomeDroppedAtHandoff],
+			StatusUnavailable: samples.outcomes[OutcomeStatusUnavailable],
+			Unrecorded:        samples.outcomes[outcomeUnset],
 		}
 		if len(samples.samples) > 0 {
 			sorted := make([]time.Duration, len(samples.samples))
@@ -356,6 +404,31 @@ type OperationStats struct {
 	P99Latency  time.Duration
 	SampleCount int
 	Window      time.Duration
+
+	// Three layers, not one. Committed and Reverted are what the chain did.
+	// Expired, DroppedAtCap and DroppedAtHandoff are what this run did to its
+	// own transactions, and none of the three is evidence about the chain.
+	// StatusUnavailable is what the run failed to see. Count and Successes above
+	// are the send path. Total them across layers and a generator throttling
+	// itself looks like a chain rejecting work.
+	//
+	// Reverted is the execution status a receipt reported: the chain took the
+	// transaction and it did nothing. RunSummary.Failed is a send that returned
+	// an error, which is a different layer, and the two carry different names so
+	// an operator reading one series beside the other cannot conflate them.
+	//
+	// Unrecorded has no legitimate producer. A non-zero value means sei-load
+	// classified a transaction wrongly, and nothing else produces one.
+	//
+	// All of them stay zero for a run with --track-receipts off, because the
+	// inclusion tracker is the only producer.
+	Committed         uint64
+	Reverted          uint64
+	Expired           uint64
+	DroppedAtCap      uint64
+	DroppedAtHandoff  uint64
+	StatusUnavailable uint64
+	Unrecorded        uint64
 }
 
 // operationSamples accumulates one operation's counts and its most recent
@@ -369,6 +442,11 @@ type operationSamples struct {
 	count     uint64
 	successes uint64
 	samples   []latencySample
+
+	// outcomes counts terminal states by Outcome. An array rather than a map:
+	// the set is closed, and indexing by the constant costs no allocation on a
+	// path the tracker walks once per transaction.
+	outcomes [outcomeCount]uint64
 }
 
 // latencySample is one successful transaction's latency and when it happened.