From 974a13ecd6441b82d3a37fc9ee22e3019bbcec73 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Wed, 26 Aug 2026 16:02:08 -0700
Subject: [PATCH 01/17] feat(stats): read execution status from receipts, not
 hashes

The tracker derived inclusion from the transaction hashes in a block, and a hash
carries no execution status. One state therefore covered a transaction that
committed and one that failed and burned its gas. A run could report a million
accepted, near-perfect inclusion and a healthy p99 while every transaction
failed, and nothing in the output said so.

blockSource becomes receiptSource, backed by ethclient.BlockReceipts. One call
per block either way, so the request count does not move: an earlier design
fetched a receipt per transaction and its cost grew with the load the run
offered, which is the constraint that shaped this one.

blockReceipt is this package's own type rather than a go-ethereum receipt. A
receipt carries eleven more fields the tracker has no business reading, and a
test supplies a hash and a status without constructing one.

matchBlock resolves each matched transaction to Committed or Failed and reports
it, and the two existing outcome sites now route through the same reporter, so
the metric and the collector stay in step.

The tracker holds a collector. The reference runs one way: the tracker may take
the collector's lock, the collector must never take the tracker's state lock.
Nothing takes both, and the field comment is where that is written down.

Reports land outside the registry lock. The sender blocks on it at every send
completion, so work held under it lands in the latency this package reports.

Guards proven by breaking what they cover: every receipt treated as committed,
the operation label dropped, a per-transaction fetch reintroduced, reaped
transactions no longer reported.

Requirements: TOT-001, TOT-002, TOT-009, TOT-015, TOT-016, TOT-017.
Tasks T007, T009, T010, T011, T012.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 main.go                         |   1 +
 stats/inclusion_outcome_test.go | 144 ++++++++++++++++++++++++++++++++
 stats/inclusion_tracker.go      | 114 +++++++++++++++++++------
 stats/inclusion_tracker_test.go |  29 +++++--
 4 files changed, 254 insertions(+), 34 deletions(-)
 create mode 100644 stats/inclusion_outcome_test.go

diff --git a/main.go b/main.go
index f22beb0..3d95d88 100644
--- a/main.go
+++ b/main.go
@@ -292,6 +292,7 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 				reapAfter,
 				inclusionRegistryCap(cfg.Settings.MaxInFlight, cfg.Settings.TPS, reapAfter),
 				cfg.Settings.ArrivalModel == config.ArrivalModelOpenLoop,
+				collector,
 			)
 			inclusion = utils.Some(inclusionTracker)
 			s.SpawnBgNamed("inclusion tracker", func() error {
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
new file mode 100644
index 0000000..6ddfb43
--- /dev/null
+++ b/stats/inclusion_outcome_test.go
@@ -0,0 +1,144 @@
+package stats
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	ethtypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAllFailedReceiptsYieldZeroCommitted is the defect this phase removes. A
+// run whose every transaction reverted used to report the same inclusion count
+// as a run where every transaction worked, because a hash carries no execution
+// status.
+//
+// It fails when a failed transaction and a committed one land in the same
+// terminal state.
+func TestAllFailedReceiptsYieldZeroCommitted(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+	key := OperationKey{Scenario: "storagerw", Operation: "write"}
+
+	var receipts []blockReceipt
+	for i := uint64(1); i <= 5; i++ {
+		tx := loadTx(i, time.Unix(1000, 0))
+		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+		tr.Register(tx)
+		receipts = append(receipts, blockReceipt{
+			Hash:   tx.EthTx.Hash(),
+			Status: ethtypes.ReceiptStatusFailed,
+		})
+	}
+	src.SetReceipts(7, receipts...)
+	tr.matchBlock(context.Background(), 7, time.Unix(1002, 0))
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Zero(t, got.Committed, "a failed transaction counted as committed")
+	require.Equal(t, uint64(5), got.Failed)
+	require.Equal(t, uint64(5), tr.Summary().Included,
+		"they were included; inclusion and execution are different questions")
+}
+
+// TestReceiptsSeparateCommittedFromFailed covers the mixed block, which is the
+// shape a real run produces. A block carrying both must split them.
+func TestReceiptsSeparateCommittedFromFailed(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	var receipts []blockReceipt
+	for i := uint64(1); i <= 4; i++ {
+		tx := loadTx(i, time.Unix(1000, 0))
+		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+		tr.Register(tx)
+		status := ethtypes.ReceiptStatusSuccessful
+		if i%2 == 0 {
+			status = ethtypes.ReceiptStatusFailed
+		}
+		receipts = append(receipts, blockReceipt{Hash: tx.EthTx.Hash(), Status: status})
+	}
+	src.SetReceipts(9, receipts...)
+	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(2), got.Committed)
+	require.Equal(t, uint64(2), got.Failed)
+	require.Zero(t, got.Unrecorded, "every matched transaction reached a real state")
+}
+
+// TestOutcomesCarryTheOperation fails when the tracker labels an outcome by
+// scenario alone. Two operations in one scenario would then share a count, and
+// neither the report nor a dashboard could say which one degraded.
+func TestOutcomesCarryTheOperation(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	read := loadTx(1, time.Unix(1000, 0))
+	read.Scenario.Name, read.Scenario.Operation = "storagerw", "read"
+	write := loadTx(2, time.Unix(1000, 0))
+	write.Scenario.Name, write.Scenario.Operation = "storagerw", "write"
+	tr.Register(read)
+	tr.Register(write)
+
+	src.SetReceipts(3,
+		blockReceipt{Hash: read.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful},
+		blockReceipt{Hash: write.EthTx.Hash(), Status: ethtypes.ReceiptStatusFailed},
+	)
+	tr.matchBlock(context.Background(), 3, time.Unix(1002, 0))
+
+	stats := tr.collector.GetOperationStats()
+	require.Equal(t, uint64(1), stats[OperationKey{Scenario: "storagerw", Operation: "read"}].Committed)
+	require.Equal(t, uint64(1), stats[OperationKey{Scenario: "storagerw", Operation: "write"}].Failed)
+	require.Zero(t, stats[OperationKey{Scenario: "storagerw", Operation: "read"}].Failed,
+		"one operation's outcome reached another's count")
+}
+
+// TestRequestsPerBlockDoNotTrackVolume is the cost constraint. An earlier design
+// fetched a receipt per transaction, and its cost grew with the load the run
+// offered, so the tool scaled its own request rate with the thing it measured.
+//
+// It fails when the request count rises with the transaction count, or when a
+// second per-block call survives beside the receipts call.
+func TestRequestsPerBlockDoNotTrackVolume(t *testing.T) {
+	for _, volume := range []int{1, 50, 500} {
+		src := NewMockBlockSource()
+		tr := newTestTracker(t, time.Minute, 10_000, src)
+
+		var receipts []blockReceipt
+		for i := 1; i <= volume; i++ {
+			tx := loadTx(uint64(i), time.Unix(1000, 0))
+			tr.Register(tx)
+			receipts = append(receipts, blockReceipt{
+				Hash:   tx.EthTx.Hash(),
+				Status: ethtypes.ReceiptStatusSuccessful,
+			})
+		}
+		src.SetReceipts(11, receipts...)
+		tr.matchBlock(context.Background(), 11, time.Unix(1002, 0))
+
+		require.Equal(t, int64(1), src.FetchCount(),
+			"a block carrying %d transactions cost %d requests, not one",
+			volume, src.FetchCount())
+	}
+}
+
+// TestReapedTransactionsReachTheCollector covers the path that does not go
+// through a receipt. A transaction nothing named before the deadline is expired,
+// and the collector has to hear about it or the states stop partitioning.
+func TestReapedTransactionsReachTheCollector(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(tx)
+	time.Sleep(time.Millisecond)
+	tr.reap()
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Expired)
+	require.Zero(t, got.Committed)
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 378c087..9db34dd 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -4,12 +4,12 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"math/big"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
 
@@ -18,26 +18,43 @@ import (
 	"github.com/sei-protocol/sei-load/utils/scope"
 )
 
-// blockSource yields the tx hashes of a single block by number. Consumer-side
-// interface so tests can drive matching without a live chain.
-type blockSource interface {
-	BlockTxHashes(ctx context.Context, n uint64) ([]common.Hash, error)
+// blockReceipt is what the tracker reads out of one transaction's receipt: the
+// hash it joins against the registry, and the status that says what execution
+// did.
+//
+// This package's own type rather than *ethtypes.Receipt. A receipt carries
+// eleven more fields the tracker has no business reading, and a test supplies a
+// pair without constructing one.
+type blockReceipt struct {
+	Hash   common.Hash
+	Status uint64
 }
 
-// ethBlockSource is the production blockSource backed by an ethclient.
-type ethBlockSource struct{ client *ethclient.Client }
+// receiptSource yields one block's receipts by number. Consumer-side interface
+// so tests can drive matching without a live chain.
+//
+// It replaces a source that yielded hashes alone. A hash says a transaction
+// reached a block and nothing about what it did there, so one terminal state
+// covered a transaction that committed and one that reverted and burned its gas.
+// One call per block either way: the request count does not move.
+type receiptSource interface {
+	BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error)
+}
+
+// ethReceiptSource is the production receiptSource backed by an ethclient.
+type ethReceiptSource struct{ client *ethclient.Client }
 
-func (s ethBlockSource) BlockTxHashes(ctx context.Context, n uint64) ([]common.Hash, error) {
-	block, err := s.client.BlockByNumber(ctx, new(big.Int).SetUint64(n))
+func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error) {
+	number := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(n)) //nolint:gosec
+	receipts, err := s.client.BlockReceipts(ctx, number)
 	if err != nil {
 		return nil, err
 	}
-	txs := block.Transactions()
-	hashes := make([]common.Hash, len(txs))
-	for i, tx := range txs {
-		hashes[i] = tx.Hash()
+	out := make([]blockReceipt, len(receipts))
+	for i, r := range receipts {
+		out[i] = blockReceipt{Hash: r.TxHash, Status: r.Status}
 	}
-	return hashes, nil
+	return out, nil
 }
 
 type entry struct {
@@ -65,8 +82,16 @@ type InclusionTracker struct {
 	// enqueue time, so arrival-IntendedSendTime would be an enqueue→inclusion
 	// latency that must not be mixed into the histogram.
 	openLoop bool
-	source   blockSource
+	source   receiptSource
 	state    utils.Mutex[*inclusionState]
+
+	// collector receives every terminal outcome. The reference runs one way: the
+	// tracker may take the collector's lock, and the collector must never take
+	// this tracker's state lock. Nothing takes both today, and this is where the
+	// rule is written down so nothing starts.
+	//
+	// Nil when a run keeps no collector, which the tests do.
+	collector *Collector
 }
 
 // defaultMaxInflight bounds the registry when the caller passes a non-positive
@@ -83,7 +108,7 @@ const defaultInclusionReapAfter = 30 * time.Second
 // reaps un-included txs after reapAfter. openLoop gates the inclusion_latency
 // sample (included/expired counts are tracked in both models). The block source
 // is the production ethclient impl; tests inject via newInclusionTrackerWithSource.
-func NewInclusionTracker(seiChainID string, reapAfter time.Duration, maxInflight int, openLoop bool) *InclusionTracker {
+func NewInclusionTracker(seiChainID string, reapAfter time.Duration, maxInflight int, openLoop bool, collector *Collector) *InclusionTracker {
 	if maxInflight <= 0 {
 		maxInflight = defaultMaxInflight
 	}
@@ -95,6 +120,7 @@ func NewInclusionTracker(seiChainID string, reapAfter time.Duration, maxInflight
 		reapAfter:   reapAfter,
 		maxInflight: maxInflight,
 		openLoop:    openLoop,
+		collector:   collector,
 		state: utils.NewMutex(&inclusionState{
 			inflight: make(map[common.Hash]*entry),
 		}),
@@ -103,7 +129,7 @@ func NewInclusionTracker(seiChainID string, reapAfter time.Duration, maxInflight
 	return t
 }
 
-func newInclusionTrackerWithSource(t *InclusionTracker, source blockSource) *InclusionTracker {
+func newInclusionTrackerWithSource(t *InclusionTracker, source receiptSource) *InclusionTracker {
 	t.source = source
 	return t
 }
@@ -124,7 +150,7 @@ func (t *InclusionTracker) Register(tx *types.LoadTx) {
 		s.inflight[hash] = &entry{tx: tx, registeredAt: time.Now()}
 	}
 	if droppedAtCap {
-		t.recordOutcome(OutcomeDroppedAtCap, tx.Scenario)
+		t.report(OutcomeDroppedAtCap, tx.Scenario)
 	}
 }
 
@@ -154,7 +180,7 @@ func (t *InclusionTracker) Run(ctx context.Context, firstEndpoint string) error
 			return fmt.Errorf("inclusion tracker: dial %s: %w", firstEndpoint, err)
 		}
 		defer client.Close()
-		t.source = ethBlockSource{client: client}
+		t.source = ethReceiptSource{client: client}
 	}
 	return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error {
 		client, err := ethclient.Dial(wsEndpoint)
@@ -207,7 +233,7 @@ func (t *InclusionTracker) processHead(ctx context.Context, num uint64, arrival
 func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival time.Time) {
 	// Explicit per-iteration cancel (not deferred-in-loop): bound the fetch.
 	fetchCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	hashes, err := t.source.BlockTxHashes(fetchCtx, num)
+	receipts, err := t.source.BlockReceipts(fetchCtx, num)
 	cancel()
 	if err != nil {
 		// No retry (avoids piling RPC onto a struggling SUT): the block's txs go
@@ -217,18 +243,29 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 			attribute.String("chain_id", t.seiChainID)))
 		return
 	}
-	matched := make([]inclusionSample, 0, len(hashes))
+	matched := make([]inclusionSample, 0, len(receipts))
+	resolved := make([]resolvedOutcome, 0, len(receipts))
 	for s := range t.state.Lock() {
-		for _, h := range hashes {
-			e, ok := s.inflight[h]
+		for _, r := range receipts {
+			e, ok := s.inflight[r.Hash]
 			if !ok {
 				continue
 			}
 			// Single writer of InclusionTime, under the lock; first observation
 			// wins (delete-on-touch) — see reorg note in sender/doc.go.
 			e.tx.InclusionTime = arrival
-			delete(s.inflight, h)
+			delete(s.inflight, r.Hash)
 			s.included++
+			// A receipt carries one status bit, so the outcome names what the run
+			// observed. Every failure cause shares the failed status.
+			outcome := OutcomeFailed
+			if r.Status == ethtypes.ReceiptStatusSuccessful {
+				outcome = OutcomeCommitted
+			}
+			resolved = append(resolved, resolvedOutcome{
+				outcome:  outcome,
+				scenario: e.tx.Scenario,
+			})
 			// Open-loop only: IntendedSendTime is a true arrival schedule there, so
 			// arrival-IntendedSendTime is a real inclusion latency. In closed-loop
 			// it is enqueue time and the sample is omitted. A zero IntendedSendTime
@@ -242,6 +279,9 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 			}
 		}
 	}
+	// Both loops run outside the registry lock. The sender blocks on that lock at
+	// every send completion, so work held under it lands in the latency this
+	// package reports.
 	for _, m := range matched {
 		inclusionLatency.Record(ctx, m.latency, metric.WithAttributes(
 			attribute.String("chain_id", t.seiChainID),
@@ -249,6 +289,30 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 			attribute.String("operation", m.scenario.Operation),
 		))
 	}
+	for _, r := range resolved {
+		t.report(r.outcome, r.scenario)
+	}
+}
+
+// resolvedOutcome is one matched tx and what its receipt said, carried out of
+// the registry lock so the report lands outside the critical section.
+type resolvedOutcome struct {
+	outcome  Outcome
+	scenario *types.TxScenario
+}
+
+// report sends one terminal outcome to both ledgers: the metric an operator
+// watches live, and the collector the run report reads.
+//
+// Call it outside the registry lock.
+func (t *InclusionTracker) report(outcome Outcome, scenario *types.TxScenario) {
+	t.recordOutcome(outcome, scenario)
+	if t.collector != nil {
+		t.collector.RecordOutcome(OperationKey{
+			Scenario:  scenario.Name,
+			Operation: scenario.Operation,
+		}, outcome)
+	}
 }
 
 // inclusionSample is one matched tx, carried out of the registry lock so its
@@ -293,7 +357,7 @@ func (t *InclusionTracker) reap() {
 	// A stalled chain can expire the whole registry in one sweep, so the counter
 	// per victim runs after the lock is released.
 	for _, scenario := range expired {
-		t.recordOutcome(OutcomeExpired, scenario)
+		t.report(OutcomeExpired, scenario)
 	}
 }
 
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index 7efbc13..ff3b72c 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -15,27 +15,38 @@ import (
 	"github.com/sei-protocol/sei-load/types"
 )
 
-// MockBlockSource is a deterministic blockSource for tests. Setter style mirrors
-// MockBlockStats: SetBlock seeds a block's tx hashes; fetches are counted.
+// MockBlockSource is a deterministic receiptSource for tests. Setter style
+// mirrors MockBlockStats: SetBlock seeds a block's receipts; fetches are counted.
+//
+// SetBlock takes hashes and marks each committed, which is what most tests want.
+// SetReceipts takes the pairs, for the tests that care about status.
 type MockBlockSource struct {
 	mu       sync.Mutex
-	blocks   map[uint64][]common.Hash
+	blocks   map[uint64][]blockReceipt
 	fetches  atomic.Int64
 	fetchErr error
 }
 
 func NewMockBlockSource() *MockBlockSource {
-	return &MockBlockSource{blocks: make(map[uint64][]common.Hash)}
+	return &MockBlockSource{blocks: make(map[uint64][]blockReceipt)}
 }
 
 func (m *MockBlockSource) SetBlock(n uint64, hashes ...common.Hash) *MockBlockSource {
+	rs := make([]blockReceipt, len(hashes))
+	for i, h := range hashes {
+		rs[i] = blockReceipt{Hash: h, Status: ethtypes.ReceiptStatusSuccessful}
+	}
+	return m.SetReceipts(n, rs...)
+}
+
+func (m *MockBlockSource) SetReceipts(n uint64, rs ...blockReceipt) *MockBlockSource {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	m.blocks[n] = hashes
+	m.blocks[n] = rs
 	return m
 }
 
-func (m *MockBlockSource) BlockTxHashes(_ context.Context, n uint64) ([]common.Hash, error) {
+func (m *MockBlockSource) BlockReceipts(_ context.Context, n uint64) ([]blockReceipt, error) {
 	m.fetches.Add(1)
 	if m.fetchErr != nil {
 		return nil, m.fetchErr
@@ -50,15 +61,15 @@ func (m *MockBlockSource) FetchCount() int64 { return m.fetches.Load() }
 // newTestTracker builds an open-loop tracker wired to a mock source, skipping
 // the live dial. Open-loop is the default so latency-bearing tests exercise the
 // inclusion_latency path; closed-loop is covered explicitly via newTestTrackerLoop.
-func newTestTracker(t *testing.T, reapAfter time.Duration, maxInflight int, src blockSource) *InclusionTracker {
+func newTestTracker(t *testing.T, reapAfter time.Duration, maxInflight int, src receiptSource) *InclusionTracker {
 	t.Helper()
 	return newTestTrackerLoop(t, reapAfter, maxInflight, src, true)
 }
 
-func newTestTrackerLoop(t *testing.T, reapAfter time.Duration, maxInflight int, src blockSource, openLoop bool) *InclusionTracker {
+func newTestTrackerLoop(t *testing.T, reapAfter time.Duration, maxInflight int, src receiptSource, openLoop bool) *InclusionTracker {
 	t.Helper()
 	return newInclusionTrackerWithSource(
-		NewInclusionTracker("test-chain", reapAfter, maxInflight, openLoop), src)
+		NewInclusionTracker("test-chain", reapAfter, maxInflight, openLoop, NewCollector()), src)
 }
 
 // loadTx builds a LoadTx with a deterministic hash from nonce and an intended

From 2d9359e0692edf24f22e2abe38e85f2fef2f94e2 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Wed, 26 Aug 2026 16:30:28 -0700
Subject: [PATCH 02/17] fix(stats): separate what the chain did from what the
 run could not see

Four independent reviewers read the receipts change. Every one of them found
the same hole: OutcomeStatusUnavailable was defined, documented as the state
that keeps a measurement problem from being reported as a chain problem, and
had no producer. A receipt read that failed left its block's transactions to
age out as expired, which is a claim about the chain that the run had no
grounds to make.

The registry now records a watermark. Each entry stamps the count of unreadable
blocks at registration; a reap compares it against the count now. A higher
count means a block that could have carried this transaction was never read, so
the transaction reaches status_unavailable instead of expired. A transaction
registered after the hole still expires normally.

An empty array is one of those unreadable answers and used to arrive silently.
A node that holds a block but has lost its receipt bodies returns an empty list
with no error, so a whole block of transactions aged out with no log line and
no metric. It now takes the same path as an error, and block_fetch_errors
carries a reason so an operator reads the cause off a dashboard rather than the
pod log.

A receipt carrying a post-state root instead of a status says the transaction
executed and does not say how it ended. Reading that as a failure would invent
a chain result, so blockReceipt carries whether the status was there.

Run proves the endpoint answers before the run starts. A node in validator or
seed mode serves no EVM HTTP at all, and without the probe such a run completes,
reports every transaction un-included and exits zero, which reads as a chain
that accepted nothing.

The tracker can now read receipts from a node other than the one it loads.
receiptEndpoint defaults to Endpoints[0], so a single-node run is unchanged,
and a run that names a second node keeps the read work off the box under load.
That matters more than the request count: a receipts read costs the serving
node work that grows with the block's transaction count.

An endpoint decides what it puts in a receipts array. A null element would have
panicked the head loop, and nothing recovers there, so the run would have died
and lost every result it had gathered.

sender/doc.go states the conservation identity over the terminal states rather
than the older three-term one, and corrects the reorg boundary: first
observation now fixes an execution status, not only a time. stats/doc.go gains
the Lifecycle and Ownership sections it deferred to this change, and the
partition claim it documented now has a test.

Also: context threads through the report path instead of being dropped;
recordOutcome becomes meterOutcome, which is what it does; the two metric
descriptions that contradicted their own series are rewritten; the collector is
required rather than nil-checked, since no caller passed nil; and the comments
that narrated this change rather than stating the present are gone.

Guards proven by breaking what they cover: the reap attribution, the
registration watermark, the empty-array branch, the status-presence branch, the
preflight, the nil guard, the cap-drop report, and a double report.

Requirements: TOT-004, TOT-013, TOT-020, TOT-021. Tasks T008, T013.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 config/config.go                |  20 ++-
 main.go                         |  12 +-
 sender/doc.go                   |  39 ++++--
 stats/collector.go              |   5 +-
 stats/doc.go                    |  44 ++++--
 stats/inclusion_outcome_test.go | 227 +++++++++++++++++++++++++-----
 stats/inclusion_tracker.go      | 238 ++++++++++++++++++++++++--------
 stats/inclusion_tracker_test.go |  27 ++--
 stats/metrics.go                |   4 +-
 stats/outcome.go                |  11 +-
 10 files changed, 486 insertions(+), 141 deletions(-)

diff --git a/config/config.go b/config/config.go
index 72ba289..f5d79d7 100644
--- a/config/config.go
+++ b/config/config.go
@@ -31,12 +31,20 @@ type LoadConfig struct {
 	// operator reviews and commits. Empty writes nothing.
 	ChainRecordPath string `json:"chainRecordPath,omitempty"`
 	// SeiChainID is the textual chain ID used for tagging metric collection.
-	SeiChainID string         `json:"seiChainID,omitempty"`
-	Endpoints  []string       `json:"endpoints"`
-	Accounts   *AccountConfig `json:"accounts,omitempty"`
-	Scenarios  []Scenario     `json:"scenarios,omitempty"`
-	MockDeploy bool           `json:"mockDeploy,omitempty"`
-	Settings   *Settings      `json:"settings,omitempty"`
+	SeiChainID string   `json:"seiChainID,omitempty"`
+	Endpoints  []string `json:"endpoints"`
+	// ReceiptEndpoint is the node the inclusion tracker reads receipts from.
+	// Empty uses Endpoints[0].
+	//
+	// A run that names a second node keeps the measurement off the box it is
+	// loading. Reading receipts costs the serving node real work, and a node
+	// that both takes the send load and answers the read degrades in a way the
+	// tracker reports as a chain result.
+	ReceiptEndpoint string         `json:"receiptEndpoint,omitempty"`
+	Accounts        *AccountConfig `json:"accounts,omitempty"`
+	Scenarios       []Scenario     `json:"scenarios,omitempty"`
+	MockDeploy      bool           `json:"mockDeploy,omitempty"`
+	Settings        *Settings      `json:"settings,omitempty"`
 	// Funding, when set, funds the generated account pool from a root key at
 	// startup so the run works against a real chain. See funding.go.
 	Funding *FundingConfig `json:"funding,omitempty"`
diff --git a/main.go b/main.go
index 3d95d88..924d9ec 100644
--- a/main.go
+++ b/main.go
@@ -281,8 +281,7 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 			})
 		}
 
-		// The --track-receipts flag now enables the block-indexed inclusion
-		// tracker (the lossy per-tx receipt path is retired).
+		// --track-receipts enables the block-indexed inclusion tracker.
 		// Not wired under --dry-run: simulated sends never hit the chain, so they
 		// would all reap as expired and pollute the inclusion stats.
 		if len(cfg.Endpoints) > 0 && cfg.Settings.TrackReceipts && !cfg.Settings.DryRun {
@@ -295,8 +294,15 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 				collector,
 			)
 			inclusion = utils.Some(inclusionTracker)
+			// Heads come from the load endpoint, which is already subscribed
+			// and cheap to read. Receipts come from receiptEndpoint when the
+			// run names one, so the read work stays off the box under load.
+			receiptEndpoint := cfg.ReceiptEndpoint
+			if receiptEndpoint == "" {
+				receiptEndpoint = cfg.Endpoints[0]
+			}
 			s.SpawnBgNamed("inclusion tracker", func() error {
-				return inclusionTracker.Run(ctx, cfg.Endpoints[0])
+				return inclusionTracker.Run(ctx, cfg.Endpoints[0], receiptEndpoint)
 			})
 		}
 
diff --git a/sender/doc.go b/sender/doc.go
index b8fc3f0..9d03c60 100644
--- a/sender/doc.go
+++ b/sender/doc.go
@@ -90,15 +90,22 @@
 //
 // When enabled (--track-receipts), the sender hands each successful send to the
 // [stats.InclusionTracker] at send-completion (after OnComplete, only on a nil
-// send error). The tracker subscribes to new heads, fetches each arriving
-// block's body once (O(blocks), not O(txs)), and stamps InclusionTime on every
-// matched in-flight tx with the block's header-ARRIVAL time. Un-included txs are
-// reaped as expired after reapAfter. inclusion_latency (arrival minus
+// send error). The tracker subscribes to new heads, reads each arriving block's
+// receipts once (one request per block, whatever the block's tx count), and
+// stamps InclusionTime on every matched in-flight tx with the block's
+// header-ARRIVAL time. A receipt carries the execution status, so a matched tx
+// resolves to committed or failed rather than to one state covering both.
+// Un-included txs are reaped after reapAfter. inclusion_latency (arrival minus
 // IntendedSendTime) is an open-loop-only measure; in closed-loop IntendedSendTime
 // is enqueue time, so the latency sample is omitted (counts are tracked in both).
 //
-// Conservation. registered == included + expired + inflight_at_shutdown, and
-// registered ⊆ succeeded (only successful sends are registered). The inclusion
+// Conservation. Over [stats.Outcome]'s terminal states,
+//
+//	registered == committed + failed + status_unavailable
+//	            + expired + dropped_at_cap + inflight_at_shutdown
+//
+// and registered ⊆ succeeded (only successful sends are registered).
+// dropped_at_handoff joins the identity with the hand-off channel. The inclusion
 // denominator is succeeded (txs_accepted), never a minted "registered" series;
 // dropped_at_cap txs are excluded from it. inflight_at_shutdown is read only
 // after both the senders and the tracker have joined.
@@ -106,13 +113,19 @@
 // Accepted boundaries. (1) WS gaps degrade conservatively: a missed head is
 // counted (block_gaps) but never backfilled, so its txs reap as expired —
 // an undercount of inclusions, never a miscount. (2) Reorgs use
-// first-observation-wins (stamp + delete); the inclusion-time error is bounded
-// by reorg_depth × block_time, with no canonical reconciliation. (3) A single
-// fetch endpoint (Endpoints[0], shared with the block collector) adds a small
-// read load. (4) InclusionTime is the header-arrival wall clock, not fetch
-// completion and not header.Time. (5) A failed block-body fetch is counted
-// (block_fetch_errors) and not retried — that block's txs reap as expired, the
-// same conservative undercount as a WS gap. (6) A tx registered after its
+// first-observation-wins (stamp + delete). This fixes an execution status as
+// well as a time: a tx that failed on an orphaned block and committed on the
+// canonical one keeps the first answer. Sei's finality makes the window small,
+// and there is no canonical reconciliation. (3) The tracker reads receipts from
+// receiptEndpoint when a run names one, and from Endpoints[0] otherwise. A
+// receipts read costs the serving node work that grows with the block's tx
+// count, so a run at high TPS should name a second node. (4) InclusionTime is
+// the header-arrival wall clock, not fetch completion and not header.Time.
+// (5) A receipt read that returns nothing usable — an error, or an empty array
+// from a node that lost the bodies — is counted by reason
+// (block_fetch_errors) and not retried. Every tx in flight across it reaches
+// status_unavailable rather than expired, because the run cannot tell a chain
+// that left it out from a block it never read. (6) A tx registered after its
 // including block was already scanned is missed and reaps as expired — bounded
 // by the microsecond register window versus block time, a rare conservative
 // undercount, the same direction as a WS gap.
diff --git a/stats/collector.go b/stats/collector.go
index 5f199b2..177b60b 100644
--- a/stats/collector.go
+++ b/stats/collector.go
@@ -418,7 +418,8 @@ type OperationStats struct {
 	// Unrecorded has no legitimate producer. A non-zero value means sei-load
 	// classified a transaction wrongly, and nothing else produces one.
 	//
-	// All of them stay zero until a tracker reports outcomes.
+	// All of them stay zero for a run with --track-receipts off, because the
+	// inclusion tracker is the only producer.
 	Committed         uint64
 	Failed            uint64
 	Expired           uint64
@@ -440,7 +441,7 @@ type operationSamples struct {
 	successes uint64
 	samples   []latencySample
 
-	// results counts terminal states by Result. An array rather than a map:
+	// outcomes counts terminal states by Outcome. An array rather than a map:
 	// the set is closed, and indexing by the constant costs no allocation on a
 	// path the tracker walks once per transaction.
 	outcomes [outcomeCount]uint64
diff --git a/stats/doc.go b/stats/doc.go
index 3d45bdc..57412f2 100644
--- a/stats/doc.go
+++ b/stats/doc.go
@@ -44,18 +44,42 @@
 // lock. The sender blocks on that lock at every send completion, so work held
 // under both puts collector contention into the latency this package reports.
 //
-// # Invariants
+// A third goroutine takes the registry lock: the inclusion_inflight gauge
+// callback, on the metric SDK's collection goroutine. It takes no collector
+// lock. Nothing may take both.
 //
-// sender/doc.go owns the conservation identity and states it there.
-// TestInclusion_Conservation asserts it.
+// # Lifecycle
+//
+// Collector outlives every producer. main builds it before the senders start
+// and reads it after they join, so a read of its counts is only final at that
+// point.
+//
+// InclusionTracker runs for the length of the run. Run dials the receipt
+// endpoint, proves it answers, subscribes to new heads, and then two goroutines
+// drive it: the head loop matches each arriving block once, and the reap loop
+// evicts transactions that outlived reapAfter. Both end when the run context
+// does. A transaction still in the registry at that point reached no terminal
+// state and is counted as inflight_at_shutdown, which is why the conservation
+// identity is only readable after both have joined.
+//
+// # Ownership boundaries
 //
-// The claim that Outcome's states partition every accepted transaction is
-// documented and not yet guarded. The test belongs with the change that wires
-// the tracker.
+// The tracker owns every transaction's terminal state. It decides, and it
+// reports once. The collector owns the counts and never decides.
 //
-// # Not documented yet
+// The reference runs one way. The tracker holds a *Collector; the collector
+// knows nothing of the tracker. That is what keeps the lock rule above
+// enforceable by structure rather than by convention.
 //
-// Lifecycle, and ownership boundaries between the collector and the tracker.
-// Both describe the tracker loop that reports outcomes, which does not exist.
-// They arrive with it.
+// The tracker's registry holds a *types.LoadTx it does not own. The sender
+// writes it once before the hand-off and never again, which is what makes it
+// safe to read a scenario out of the registry and report on it after the lock
+// is released.
+//
+// # Invariants
+//
+// sender/doc.go owns the conservation identity and states it there.
+// TestInclusion_Conservation asserts the registry identity, and
+// TestOutcomesPartitionEveryRegisteredTx asserts that Outcome's terminal states
+// partition every registered transaction.
 package stats
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 6ddfb43..691c194 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -2,6 +2,8 @@ package stats
 
 import (
 	"context"
+	"errors"
+	"strconv"
 	"testing"
 	"time"
 
@@ -9,13 +11,8 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-// TestAllFailedReceiptsYieldZeroCommitted is the defect this phase removes. A
-// run whose every transaction reverted used to report the same inclusion count
-// as a run where every transaction worked, because a hash carries no execution
-// status.
-//
-// It fails when a failed transaction and a committed one land in the same
-// terminal state.
+// TestAllFailedReceiptsYieldZeroCommitted fails when a failed transaction and a
+// committed one land in the same terminal state.
 func TestAllFailedReceiptsYieldZeroCommitted(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
@@ -27,8 +24,9 @@ func TestAllFailedReceiptsYieldZeroCommitted(t *testing.T) {
 		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
 		tr.Register(tx)
 		receipts = append(receipts, blockReceipt{
-			Hash:   tx.EthTx.Hash(),
-			Status: ethtypes.ReceiptStatusFailed,
+			Hash:      tx.EthTx.Hash(),
+			Status:    ethtypes.ReceiptStatusFailed,
+			HasStatus: true,
 		})
 	}
 	src.SetReceipts(7, receipts...)
@@ -57,7 +55,7 @@ func TestReceiptsSeparateCommittedFromFailed(t *testing.T) {
 		if i%2 == 0 {
 			status = ethtypes.ReceiptStatusFailed
 		}
-		receipts = append(receipts, blockReceipt{Hash: tx.EthTx.Hash(), Status: status})
+		receipts = append(receipts, blockReceipt{Hash: tx.EthTx.Hash(), Status: status, HasStatus: true})
 	}
 	src.SetReceipts(9, receipts...)
 	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
@@ -83,8 +81,8 @@ func TestOutcomesCarryTheOperation(t *testing.T) {
 	tr.Register(write)
 
 	src.SetReceipts(3,
-		blockReceipt{Hash: read.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful},
-		blockReceipt{Hash: write.EthTx.Hash(), Status: ethtypes.ReceiptStatusFailed},
+		blockReceipt{Hash: read.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true},
+		blockReceipt{Hash: write.EthTx.Hash(), Status: ethtypes.ReceiptStatusFailed, HasStatus: true},
 	)
 	tr.matchBlock(context.Background(), 3, time.Unix(1002, 0))
 
@@ -95,32 +93,35 @@ func TestOutcomesCarryTheOperation(t *testing.T) {
 		"one operation's outcome reached another's count")
 }
 
-// TestRequestsPerBlockDoNotTrackVolume is the cost constraint. An earlier design
-// fetched a receipt per transaction, and its cost grew with the load the run
-// offered, so the tool scaled its own request rate with the thing it measured.
+// TestRequestsPerBlockDoNotTrackVolume fails when the request count rises with
+// the transaction count.
 //
-// It fails when the request count rises with the transaction count, or when a
-// second per-block call survives beside the receipts call.
+// It measures requests this process issues. It says nothing about what one
+// request costs the node that answers it, which is a separate constraint and a
+// separate measurement.
 func TestRequestsPerBlockDoNotTrackVolume(t *testing.T) {
 	for _, volume := range []int{1, 50, 500} {
-		src := NewMockBlockSource()
-		tr := newTestTracker(t, time.Minute, 10_000, src)
+		t.Run(strconv.Itoa(volume), func(t *testing.T) {
+			src := NewMockBlockSource()
+			tr := newTestTracker(t, time.Minute, 10_000, src)
 
-		var receipts []blockReceipt
-		for i := 1; i <= volume; i++ {
-			tx := loadTx(uint64(i), time.Unix(1000, 0))
-			tr.Register(tx)
-			receipts = append(receipts, blockReceipt{
-				Hash:   tx.EthTx.Hash(),
-				Status: ethtypes.ReceiptStatusSuccessful,
-			})
-		}
-		src.SetReceipts(11, receipts...)
-		tr.matchBlock(context.Background(), 11, time.Unix(1002, 0))
+			var receipts []blockReceipt
+			for i := 1; i <= volume; i++ {
+				tx := loadTx(uint64(i), time.Unix(1000, 0))
+				tr.Register(tx)
+				receipts = append(receipts, blockReceipt{
+					Hash:      tx.EthTx.Hash(),
+					Status:    ethtypes.ReceiptStatusSuccessful,
+					HasStatus: true,
+				})
+			}
+			src.SetReceipts(11, receipts...)
+			tr.matchBlock(context.Background(), 11, time.Unix(1002, 0))
 
-		require.Equal(t, int64(1), src.FetchCount(),
-			"a block carrying %d transactions cost %d requests, not one",
-			volume, src.FetchCount())
+			require.Equal(t, int64(1), src.FetchCount(),
+				"a block carrying %d transactions cost %d requests, not one",
+				volume, src.FetchCount())
+		})
 	}
 }
 
@@ -136,9 +137,167 @@ func TestReapedTransactionsReachTheCollector(t *testing.T) {
 	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
 	tr.Register(tx)
 	time.Sleep(time.Millisecond)
-	tr.reap()
+	tr.reap(context.Background())
 
 	got := tr.collector.GetOperationStats()[key]
 	require.Equal(t, uint64(1), got.Expired)
 	require.Zero(t, got.Committed)
 }
+
+// TestUnreadableBlockIsNotAChainVerdict fails when a block the run could not
+// read is reported as a chain that left the transaction out. Expired is a claim
+// about the chain; a fetch that returned nothing supports no such claim.
+func TestUnreadableBlockIsNotAChainVerdict(t *testing.T) {
+	cases := []struct {
+		name  string
+		drive func(*MockBlockSource)
+	}{
+		{"fetch_error", func(s *MockBlockSource) { s.SetFetchErr(errors.New("connection refused")) }},
+		{"empty_array", func(s *MockBlockSource) { s.SetReceipts(9) }},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			src := NewMockBlockSource()
+			tr := newTestTracker(t, time.Nanosecond, 100, src)
+			key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+			tx := loadTx(1, time.Unix(1000, 0))
+			tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+			tr.Register(tx)
+
+			tc.drive(src)
+			tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+			tr.reap(context.Background())
+
+			got := tr.collector.GetOperationStats()[key]
+			require.Equal(t, uint64(1), got.StatusUnavailable,
+				"a block the run never read was reported as a chain result")
+			require.Zero(t, got.Expired)
+		})
+	}
+}
+
+// TestTransactionsRegisteredAfterTheHoleStillExpire fails when one unreadable
+// block turns every later transaction into status_unavailable. The hole covers
+// the transactions in flight across it, and nothing after.
+func TestTransactionsRegisteredAfterTheHoleStillExpire(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	src.SetFetchErr(errors.New("connection refused"))
+	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+
+	tx := loadTx(2, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(tx)
+	tr.reap(context.Background())
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Expired,
+		"a transaction registered after the hole inherited it")
+	require.Zero(t, got.StatusUnavailable)
+}
+
+// TestReceiptWithoutStatusIsNotAFailure fails when a receipt that carries a
+// post-state root instead of a status is read as an execution failure. It says
+// the transaction executed; it does not say how it ended.
+func TestReceiptWithoutStatusIsNotAFailure(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(tx)
+	src.SetReceipts(4, blockReceipt{Hash: tx.EthTx.Hash(), HasStatus: false})
+	tr.matchBlock(context.Background(), 4, time.Unix(1002, 0))
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"an unreadable status was reported as a chain failure")
+	require.Zero(t, got.Failed)
+}
+
+// TestOutcomesPartitionEveryRegisteredTx is the guard stats/doc.go promises for
+// the seven-term identity. It fails when a registered transaction reaches no
+// terminal state, or reaches two.
+func TestOutcomesPartitionEveryRegisteredTx(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 4, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	// Four land in a block, split committed and failed. Two more are refused at
+	// the cap. One is left in flight and then reaped.
+	var receipts []blockReceipt
+	for i := uint64(1); i <= 4; i++ {
+		tx := loadTx(i, time.Unix(1000, 0))
+		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+		tr.Register(tx)
+		status := ethtypes.ReceiptStatusSuccessful
+		if i > 2 {
+			status = ethtypes.ReceiptStatusFailed
+		}
+		receipts = append(receipts, blockReceipt{Hash: tx.EthTx.Hash(), Status: status, HasStatus: true})
+	}
+	for i := uint64(5); i <= 6; i++ {
+		tx := loadTx(i, time.Unix(1000, 0))
+		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+		tr.Register(tx)
+	}
+	src.SetReceipts(5, receipts[:3]...)
+	tr.matchBlock(context.Background(), 5, time.Unix(1002, 0))
+
+	got := tr.collector.GetOperationStats()[key]
+	sum := got.Committed + got.Failed + got.Expired + got.DroppedAtCap +
+		got.DroppedAtHandoff + got.StatusUnavailable + got.Unrecorded
+	inflight := tr.Summary().InflightAtShutdown
+	require.Equal(t, uint64(6), sum+inflight,
+		"six registered, %d reached a terminal state and %d are still in flight",
+		sum, inflight)
+	require.Zero(t, got.Unrecorded, "an outcome no caller may report was recorded")
+}
+
+// TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer fails when a run against
+// a node that serves no receipts starts anyway. Such a run completes, reports
+// every transaction un-included, and exits zero, which reads as a chain that
+// accepted nothing.
+func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
+	cases := []struct {
+		name    string
+		err     error
+		wantErr bool
+	}{
+		{"method_absent", errors.New("the method eth_getBlockReceipts does not exist/is not available"), true},
+		{"node_busy", errors.New("context deadline exceeded"), false},
+		{"node_behind", errors.New("not found"), false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			src := NewMockBlockSource().SetFetchErr(tc.err)
+			tr := newTestTracker(t, time.Minute, 100, src)
+
+			err := tr.preflight(context.Background(), "http://node:8545")
+			if !tc.wantErr {
+				require.NoError(t, err, "a busy endpoint failed the run instead of being retried")
+				return
+			}
+			require.ErrorContains(t, err, "does not serve eth_getBlockReceipts")
+			require.ErrorContains(t, err, "http://node:8545", "the error does not name the endpoint")
+		})
+	}
+}
+
+// TestNilReceiptDoesNotEndTheRun fails when an endpoint that returns a null
+// array element panics the head loop. Nothing recovers there, so the run dies
+// and loses every result it gathered.
+func TestNilReceiptDoesNotEndTheRun(t *testing.T) {
+	hash := loadTx(1, time.Unix(1000, 0)).EthTx.Hash()
+	src := ethReceiptSource{}
+	got := src.narrow([]*ethtypes.Receipt{
+		nil,
+		{TxHash: hash, Status: ethtypes.ReceiptStatusSuccessful},
+		nil,
+	})
+	require.Equal(t, []blockReceipt{{Hash: hash, Status: 1, HasStatus: true}}, got)
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 9db34dd..fc62255 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -2,8 +2,10 @@ package stats
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -22,21 +24,22 @@ import (
 // hash it joins against the registry, and the status that says what execution
 // did.
 //
-// This package's own type rather than *ethtypes.Receipt. A receipt carries
-// eleven more fields the tracker has no business reading, and a test supplies a
+// This package's own type rather than *ethtypes.Receipt. A receipt carries a
+// dozen more fields the tracker has no business reading, and a test supplies a
 // pair without constructing one.
+//
+// HasStatus is false for a receipt that carries a post-state root instead of a
+// status, which is a receipt whose execution result cannot be read. Status is
+// meaningless when it is false.
 type blockReceipt struct {
-	Hash   common.Hash
-	Status uint64
+	Hash      common.Hash
+	Status    uint64
+	HasStatus bool
 }
 
 // receiptSource yields one block's receipts by number. Consumer-side interface
-// so tests can drive matching without a live chain.
-//
-// It replaces a source that yielded hashes alone. A hash says a transaction
-// reached a block and nothing about what it did there, so one terminal state
-// covered a transaction that committed and one that reverted and burned its gas.
-// One call per block either way: the request count does not move.
+// so tests can drive matching without a live chain. One call per block,
+// whatever the block's transaction count.
 type receiptSource interface {
 	BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error)
 }
@@ -45,24 +48,51 @@ type receiptSource interface {
 type ethReceiptSource struct{ client *ethclient.Client }
 
 func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error) {
-	number := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(n)) //nolint:gosec
+	//nolint:gosec // A block height never approaches MaxInt64, where the
+	// conversion would land on rpc.BlockNumber's negative sentinels.
+	number := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(n))
 	receipts, err := s.client.BlockReceipts(ctx, number)
 	if err != nil {
 		return nil, err
 	}
-	out := make([]blockReceipt, len(receipts))
-	for i, r := range receipts {
-		out[i] = blockReceipt{Hash: r.TxHash, Status: r.Status}
+	return s.narrow(receipts), nil
+}
+
+// narrow keeps the two fields the tracker reads and drops the rest.
+//
+// An endpoint decides what it puts in this array. A nil element would panic the
+// head loop, which ends the run and loses every result it gathered, so drop it
+// rather than trust the shape.
+func (ethReceiptSource) narrow(receipts []*ethtypes.Receipt) []blockReceipt {
+	out := make([]blockReceipt, 0, len(receipts))
+	for _, r := range receipts {
+		if r == nil {
+			continue
+		}
+		out = append(out, blockReceipt{
+			Hash:      r.TxHash,
+			Status:    r.Status,
+			HasStatus: len(r.PostState) == 0,
+		})
 	}
-	return out, nil
+	return out
 }
 
 type entry struct {
 	tx           *types.LoadTx
 	registeredAt time.Time
+	// blindFetches is inclusionState.blindFetches at registration. A reap
+	// compares it against the current count: a higher count means a receipt
+	// fetch failed while this tx was in flight, so the run cannot say the chain
+	// left it out. See reap.
+	blindFetches uint64
 }
 
 type inclusionState struct {
+	// blindFetches counts receipt fetches that returned nothing usable. It only
+	// ever grows, so an entry's terminal state follows from comparing this
+	// against the value it recorded at registration.
+	blindFetches       uint64
 	inflight           map[common.Hash]*entry
 	included           uint64
 	expired            uint64
@@ -88,9 +118,12 @@ type InclusionTracker struct {
 	// collector receives every terminal outcome. The reference runs one way: the
 	// tracker may take the collector's lock, and the collector must never take
 	// this tracker's state lock. Nothing takes both today, and this is where the
-	// rule is written down so nothing starts.
+	// rule is written down so nothing starts. A third holder exists: the
+	// inclusion_inflight gauge callback in metrics.go takes the state lock on
+	// the metric SDK's collection goroutine. It takes no collector lock, and
+	// the rule above is why it must not start.
 	//
-	// Nil when a run keeps no collector, which the tests do.
+	// Never nil. NewInclusionTracker takes it because every run has one.
 	collector *Collector
 }
 
@@ -147,23 +180,26 @@ func (t *InclusionTracker) Register(tx *types.LoadTx) {
 			droppedAtCap = true
 			break
 		}
-		s.inflight[hash] = &entry{tx: tx, registeredAt: time.Now()}
+		s.inflight[hash] = &entry{tx: tx, registeredAt: time.Now(), blindFetches: s.blindFetches}
 	}
 	if droppedAtCap {
-		t.report(OutcomeDroppedAtCap, tx.Scenario)
+		// The sender calls this from the send path and carries no context here,
+		// so this one report has none to pass.
+		t.report(context.Background(), OutcomeDroppedAtCap, tx.Scenario)
 	}
 }
 
-// recordOutcome counts one tx that left the registry un-included. Callers emit
-// outside the registry lock: the sender blocks on that lock at every send
-// completion, so time spent holding it lands in the latency this package
-// reports.
+// meterOutcome emits one terminal outcome to the inclusion_outcome counter.
+// Callers emit outside the registry lock: the sender blocks on that lock at
+// every send completion, so time spent holding it lands in the latency this
+// package reports.
+//
 // The parameter is an Outcome rather than a string so that one type owns every
 // value this label can take. Passing a literal here would leave the metric and
 // Outcome.String() as two independent sources for one wire contract, and a
 // rename of either would orphan a dashboard while the other's test stayed green.
-func (t *InclusionTracker) recordOutcome(outcome Outcome, scenario *types.TxScenario) {
-	inclusionOutcome.Add(context.Background(), 1, metric.WithAttributes(
+func (t *InclusionTracker) meterOutcome(ctx context.Context, outcome Outcome, scenario *types.TxScenario) {
+	inclusionOutcome.Add(ctx, 1, metric.WithAttributes(
 		attribute.String("chain_id", t.seiChainID),
 		attribute.String("scenario", scenario.Name),
 		attribute.String("operation", scenario.Operation),
@@ -171,16 +207,25 @@ func (t *InclusionTracker) recordOutcome(outcome Outcome, scenario *types.TxScen
 	))
 }
 
-// Run subscribes to new heads and matches each arriving block once.
-func (t *InclusionTracker) Run(ctx context.Context, firstEndpoint string) error {
-	wsEndpoint := utils.GetWSEndpoint(firstEndpoint)
+// Run subscribes to new heads on headEndpoint and reads each arriving block's
+// receipts from receiptEndpoint. Pass the same string for both to run against
+// one node.
+//
+// The tracker only ever reads the height it just received as a head. It never
+// backfills, so the serving node's receipt retention does not bound it. A change
+// that reaches further back does.
+func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoint string) error {
+	wsEndpoint := utils.GetWSEndpoint(headEndpoint)
 	if t.source == nil {
-		client, err := ethclient.Dial(firstEndpoint)
+		client, err := ethclient.Dial(receiptEndpoint)
 		if err != nil {
-			return fmt.Errorf("inclusion tracker: dial %s: %w", firstEndpoint, err)
+			return fmt.Errorf("inclusion tracker: dial %s: %w", receiptEndpoint, err)
 		}
 		defer client.Close()
 		t.source = ethReceiptSource{client: client}
+		if err := t.preflight(ctx, receiptEndpoint); err != nil {
+			return err
+		}
 	}
 	return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error {
 		client, err := ethclient.Dial(wsEndpoint)
@@ -228,6 +273,33 @@ func (t *InclusionTracker) processHead(ctx context.Context, num uint64, arrival
 	return num
 }
 
+// preflight proves the receipt endpoint answers before the run starts. A node in
+// validator or seed mode serves no EVM HTTP at all, and a node may deny the
+// method by name. Without this the run completes, reports every transaction
+// un-included, and exits zero, which reads as a chain that accepted nothing.
+func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error {
+	probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	head, err := t.source.BlockReceipts(probeCtx, 0)
+	// Height 0 is the genesis block: every node that serves the method answers
+	// it, and no node needs to hold recent state to do so. An empty list is a
+	// pass, because genesis carries no EVM transactions.
+	_ = head
+	if err == nil {
+		return nil
+	}
+	if reason := fetchFailureReason(err); reason == "method_unavailable" {
+		return fmt.Errorf(
+			"inclusion tracker: %s does not serve eth_getBlockReceipts (%w). "+
+				"Point --receipt-endpoint at a node in fullNode or archive mode",
+			endpoint, err)
+	}
+	// Any other error is the endpoint being busy or behind, not the endpoint
+	// being wrong. The run proceeds and reports what it sees.
+	log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
+	return nil
+}
+
 // matchBlock fetches block num once and stamps every in-flight tx it includes
 // with the header-arrival time.
 func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival time.Time) {
@@ -236,11 +308,19 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 	receipts, err := t.source.BlockReceipts(fetchCtx, num)
 	cancel()
 	if err != nil {
-		// No retry (avoids piling RPC onto a struggling SUT): the block's txs go
-		// unmatched and reap as expired. Surfaced so the undercount is visible.
-		log.Printf("inclusion tracker: fetch block %d: %v", num, err)
-		inclusionBlockFetchErrors.Add(ctx, 1, metric.WithAttributes(
-			attribute.String("chain_id", t.seiChainID)))
+		// No retry: a retry piles RPC onto an endpoint already failing. The
+		// block goes unmatched, and blindFetches records that this run has a
+		// hole, so a tx in flight across it reaps as status_unavailable rather
+		// than as a verdict about the chain.
+		t.recordBlindFetch(ctx, num, fetchFailureReason(err), err)
+		return
+	}
+	if len(receipts) == 0 {
+		// An empty array and an error are the same answer to this tracker and
+		// arrive differently: a node that holds the block but has lost its
+		// receipt bodies returns an empty array with no error. Untreated it
+		// reaps a whole block of transactions as expired, silently.
+		t.recordBlindFetch(ctx, num, "empty", nil)
 		return
 	}
 	matched := make([]inclusionSample, 0, len(receipts))
@@ -258,9 +338,16 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 			s.included++
 			// A receipt carries one status bit, so the outcome names what the run
 			// observed. Every failure cause shares the failed status.
-			outcome := OutcomeFailed
-			if r.Status == ethtypes.ReceiptStatusSuccessful {
+			outcome := OutcomeStatusUnavailable
+			switch {
+			case !r.HasStatus:
+				// A receipt with a post-state root instead of a status says the
+				// tx executed and does not say how it ended. Reading that as a
+				// failure would invent a chain result.
+			case r.Status == ethtypes.ReceiptStatusSuccessful:
 				outcome = OutcomeCommitted
+			default:
+				outcome = OutcomeFailed
 			}
 			resolved = append(resolved, resolvedOutcome{
 				outcome:  outcome,
@@ -290,7 +377,7 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 		))
 	}
 	for _, r := range resolved {
-		t.report(r.outcome, r.scenario)
+		t.report(ctx, r.outcome, r.scenario)
 	}
 }
 
@@ -301,18 +388,17 @@ type resolvedOutcome struct {
 	scenario *types.TxScenario
 }
 
-// report sends one terminal outcome to both ledgers: the metric an operator
-// watches live, and the collector the run report reads.
+// report sends one terminal outcome to both ledgers: the inclusion_outcome
+// metric an operator watches live, and the collector's per-operation counts.
+// The run report does not read those counts yet.
 //
 // Call it outside the registry lock.
-func (t *InclusionTracker) report(outcome Outcome, scenario *types.TxScenario) {
-	t.recordOutcome(outcome, scenario)
-	if t.collector != nil {
-		t.collector.RecordOutcome(OperationKey{
-			Scenario:  scenario.Name,
-			Operation: scenario.Operation,
-		}, outcome)
-	}
+func (t *InclusionTracker) report(ctx context.Context, outcome Outcome, scenario *types.TxScenario) {
+	t.meterOutcome(ctx, outcome, scenario)
+	t.collector.RecordOutcome(OperationKey{
+		Scenario:  scenario.Name,
+		Operation: scenario.Operation,
+	}, outcome)
 }
 
 // inclusionSample is one matched tx, carried out of the registry lock so its
@@ -333,7 +419,7 @@ func (t *InclusionTracker) reapLoop(ctx context.Context) error {
 			log.Printf("inclusion tracker: reap loop: %v", err)
 			continue
 		}
-		t.reap()
+		t.reap(ctx)
 	}
 	return ctx.Err()
 }
@@ -341,9 +427,9 @@ func (t *InclusionTracker) reapLoop(ctx context.Context) error {
 // reap evicts txs in-flight longer than reapAfter as expired. Delete-on-touch
 // under the lock races safely against matchBlock: whoever holds the lock first
 // wins, no double count.
-func (t *InclusionTracker) reap() {
+func (t *InclusionTracker) reap(ctx context.Context) {
 	cutoff := time.Now().Add(-t.reapAfter)
-	var expired []*types.TxScenario
+	var expired []resolvedOutcome
 	for s := range t.state.Lock() {
 		for h, e := range s.inflight {
 			if e.registeredAt.After(cutoff) {
@@ -351,13 +437,57 @@ func (t *InclusionTracker) reap() {
 			}
 			delete(s.inflight, h)
 			s.expired++
-			expired = append(expired, e.tx.Scenario)
+			// A fetch failed while this tx was in flight, so the block that
+			// would have carried it was never read. The chain may well have
+			// included it. Expired would report a chain problem where the truth
+			// is a measurement problem, which is what Outcome's two states are
+			// for.
+			outcome := OutcomeExpired
+			if s.blindFetches > e.blindFetches {
+				outcome = OutcomeStatusUnavailable
+			}
+			expired = append(expired, resolvedOutcome{outcome: outcome, scenario: e.tx.Scenario})
 		}
 	}
 	// A stalled chain can expire the whole registry in one sweep, so the counter
 	// per victim runs after the lock is released.
-	for _, scenario := range expired {
-		t.report(OutcomeExpired, scenario)
+	for _, r := range expired {
+		t.report(ctx, r.outcome, r.scenario)
+	}
+}
+
+// recordBlindFetch marks that the run could not read one block's receipts. Every
+// tx in flight now reaps as status_unavailable rather than expired, because the
+// run cannot tell the two apart for a block it never read.
+func (t *InclusionTracker) recordBlindFetch(ctx context.Context, num uint64, reason string, err error) {
+	for s := range t.state.Lock() {
+		s.blindFetches++
+	}
+	if err != nil {
+		log.Printf("inclusion tracker: fetch block %d (%s): %v", num, reason, err)
+	} else {
+		log.Printf("inclusion tracker: block %d returned no receipts (%s)", num, reason)
+	}
+	inclusionBlockFetchErrors.Add(ctx, 1, metric.WithAttributes(
+		attribute.String("chain_id", t.seiChainID),
+		attribute.String("reason", reason)))
+}
+
+// fetchFailureReason buckets a receipt-fetch error so an operator reads the
+// cause off a dashboard instead of the pod log. The strings are label values:
+// keep them few, and keep them stable.
+func fetchFailureReason(err error) string {
+	switch msg := strings.ToLower(err.Error()); {
+	case strings.Contains(msg, "does not exist") || strings.Contains(msg, "not available"):
+		return "method_unavailable"
+	case strings.Contains(msg, "pruned"):
+		return "pruned"
+	case strings.Contains(msg, "not found"):
+		return "not_found"
+	case errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded"):
+		return "timeout"
+	default:
+		return "other"
 	}
 }
 
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index ff3b72c..0b19c41 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -34,7 +34,7 @@ func NewMockBlockSource() *MockBlockSource {
 func (m *MockBlockSource) SetBlock(n uint64, hashes ...common.Hash) *MockBlockSource {
 	rs := make([]blockReceipt, len(hashes))
 	for i, h := range hashes {
-		rs[i] = blockReceipt{Hash: h, Status: ethtypes.ReceiptStatusSuccessful}
+		rs[i] = blockReceipt{Hash: h, Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true}
 	}
 	return m.SetReceipts(n, rs...)
 }
@@ -46,13 +46,22 @@ func (m *MockBlockSource) SetReceipts(n uint64, rs ...blockReceipt) *MockBlockSo
 	return m
 }
 
+// SetFetchErr makes every later fetch fail, so a test can drive the branch where
+// the run cannot read a block at all.
+func (m *MockBlockSource) SetFetchErr(err error) *MockBlockSource {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.fetchErr = err
+	return m
+}
+
 func (m *MockBlockSource) BlockReceipts(_ context.Context, n uint64) ([]blockReceipt, error) {
 	m.fetches.Add(1)
+	m.mu.Lock()
+	defer m.mu.Unlock()
 	if m.fetchErr != nil {
 		return nil, m.fetchErr
 	}
-	m.mu.Lock()
-	defer m.mu.Unlock()
 	return m.blocks[n], nil
 }
 
@@ -152,7 +161,7 @@ func TestInclusion_ReapExpires(t *testing.T) {
 	require.Equal(t, 1, inflightLen(t, tr))
 
 	time.Sleep(20 * time.Millisecond)
-	tr.reap()
+	tr.reap(context.Background())
 
 	require.Equal(t, 0, inflightLen(t, tr), "reaped tx leaves the registry (no leak)")
 	require.True(t, tx.InclusionTime.IsZero(), "reaped tx is never stamped")
@@ -169,7 +178,7 @@ func TestInclusion_ReapVsLateInclusion(t *testing.T) {
 		tx := loadTx(1, time.Unix(1000, 0))
 		tr.Register(tx)
 		time.Sleep(time.Millisecond)
-		tr.reap() // wins: expired
+		tr.reap(context.Background()) // wins: expired
 		src.SetBlock(5, tx.EthTx.Hash())
 		tr.matchBlock(context.Background(), 5, time.Unix(1002, 0)) // no-op
 		s := tr.Summary()
@@ -185,7 +194,7 @@ func TestInclusion_ReapVsLateInclusion(t *testing.T) {
 		src.SetBlock(5, tx.EthTx.Hash())
 		tr.matchBlock(context.Background(), 5, time.Unix(1002, 0)) // wins: included
 		time.Sleep(time.Millisecond)
-		tr.reap() // no-op
+		tr.reap(context.Background()) // no-op
 		s := tr.Summary()
 		require.Equal(t, uint64(1), s.Included)
 		require.Equal(t, uint64(0), s.Expired)
@@ -277,7 +286,7 @@ func TestInclusion_Conservation(t *testing.T) {
 					}
 				}
 			}
-			tr.reap()
+			tr.reap(context.Background())
 
 			s := tr.Summary()
 			// dropped_at_cap is excluded from the registered set, so every Register
@@ -333,12 +342,12 @@ func TestInclusion_ConcurrentRaceSafe(t *testing.T) {
 	go func() {
 		defer wg.Done()
 		for range 50 {
-			tr.reap()
+			tr.reap(context.Background())
 			time.Sleep(100 * time.Microsecond)
 		}
 	}()
 	wg.Wait()
-	tr.reap()
+	tr.reap(context.Background())
 
 	s := tr.Summary()
 	require.Equal(t, uint64(n), s.Included+s.Expired+s.InflightAtShutdown,
diff --git a/stats/metrics.go b/stats/metrics.go
index ba5f8ac..452d208 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -68,7 +68,7 @@ var (
 
 	inclusionOutcome = must(meter.Int64Counter(
 		"inclusion_outcome",
-		metric.WithDescription("In-flight txs that left the registry un-included, by outcome. See stats.Outcome for the values."),
+		metric.WithDescription("Terminal outcome of every tx the inclusion tracker registered. See stats.Outcome for the values."),
 		metric.WithUnit("{transactions}")))
 
 	inclusionBlockGaps = must(meter.Int64Counter(
@@ -78,7 +78,7 @@ var (
 
 	inclusionBlockFetchErrors = must(meter.Int64Counter(
 		"block_fetch_errors",
-		metric.WithDescription("Block-body fetches that failed; the block's txs go unmatched and reap as expired (no retry)"),
+		metric.WithDescription("Receipt reads that returned nothing usable, by reason; the block goes unmatched and its txs reach status_unavailable (no retry)"),
 		metric.WithUnit("{blocks}")))
 
 	// Run-summary: the only inclusion tally with no live series, since it is the
diff --git a/stats/outcome.go b/stats/outcome.go
index 402024e..0631fd7 100644
--- a/stats/outcome.go
+++ b/stats/outcome.go
@@ -2,11 +2,9 @@ package stats
 
 // Outcome is what became of one transaction the endpoint accepted.
 //
-// Nothing reports an Outcome yet. The tracker still matches blocks by hash and
-// counts its own terminal states; wiring it to Collector.RecordOutcome, and
-// reading execution status from a per-block receipts call, both come later.
-// Until then every count stays zero, and the states below describe the design
-// rather than the run.
+// InclusionTracker reports one of these for every transaction it registers, to
+// the inclusion_outcome metric and to Collector.RecordOutcome. The run report
+// does not read the collector's counts yet.
 //
 // Two distinctions carry the point of the type. Committed and Failed separate a
 // transaction that did what the workload asked from one that burned its gas
@@ -21,9 +19,6 @@ package stats
 //	accepted = committed + failed + status_unavailable
 //	         + expired + dropped_at_cap + dropped_at_handoff
 //	         + inflight_at_shutdown
-//
-// sender/doc.go still states an older three-term identity over included and
-// expired. Rewriting it belongs to the change that wires the tracker.
 type Outcome uint8
 
 const (

From 61b1c4c67462676057727732bbf25321a7a96ba2 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Wed, 26 Aug 2026 16:42:56 -0700
Subject: [PATCH 03/17] refactor(stats): name the execution failure reverted,
 and state what a read costs

Two decisions the review surfaced and could not make for itself.

An operator reading seiload_inclusion_outcome_total{outcome="failed"} beside
seiload_run_txs_failed_total sees one word for two layers of the same run. One
is a transaction the chain took and ran to no effect, the other is a send that
returned an error. reverted is what the EVM calls the first, and it does not
collide. succeeded was not available as the other half of the pair, because
OperationStats already carries Successes for the send path.

The label values are declared frozen, and this is the last moment the rename is
free: nothing in the platform repo binds them yet.

The second decision stands rather than changes the code. eth_getBlockReceipts
costs the node answering it work that grows with the square of the block's
transaction count, because it resolves each receipt's index by walking the
whole block and recovers every sender again while doing so. That is a property
of the node, not of this change, and every caller of the method pays it.

The run keeps that cost off the box it is loading by pointing the tracker at a
node that takes no send load, which is what receiptEndpoint is for. The cost
does not disappear: it bounds what the tracking node can keep up with. Both
config.ReceiptEndpoint and receiptSource now say so, and say to measure against
the target chain before turning receipt tracking on.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 config/config.go                | 15 +++++++++++----
 stats/collector.go              | 12 +++++++-----
 stats/collector_outcome_test.go | 10 +++++-----
 stats/inclusion_outcome_test.go | 12 ++++++------
 stats/inclusion_tracker.go      |  7 ++++++-
 stats/outcome.go                | 12 +++++++-----
 6 files changed, 42 insertions(+), 26 deletions(-)

diff --git a/config/config.go b/config/config.go
index f5d79d7..50b124a 100644
--- a/config/config.go
+++ b/config/config.go
@@ -36,10 +36,17 @@ type LoadConfig struct {
 	// ReceiptEndpoint is the node the inclusion tracker reads receipts from.
 	// Empty uses Endpoints[0].
 	//
-	// A run that names a second node keeps the measurement off the box it is
-	// loading. Reading receipts costs the serving node real work, and a node
-	// that both takes the send load and answers the read degrades in a way the
-	// tracker reports as a chain result.
+	// Name a node that takes no send load. eth_getBlockReceipts costs the node
+	// answering it work that grows with the square of the block's transaction
+	// count, because it resolves each receipt's index by walking the whole
+	// block. A node that both takes the send load and answers that read
+	// degrades, and the tracker reports the degradation as a chain result.
+	//
+	// This bounds what the tracking node can keep up with. Measure
+	// eth_getBlockReceipts against the target chain at the run's block width
+	// before enabling --track-receipts: a read that outruns the block interval
+	// makes the tracker blind, and a blind tracker reports status_unavailable
+	// rather than a number.
 	ReceiptEndpoint string         `json:"receiptEndpoint,omitempty"`
 	Accounts        *AccountConfig `json:"accounts,omitempty"`
 	Scenarios       []Scenario     `json:"scenarios,omitempty"`
diff --git a/stats/collector.go b/stats/collector.go
index 177b60b..109b974 100644
--- a/stats/collector.go
+++ b/stats/collector.go
@@ -315,7 +315,7 @@ func (c *Collector) GetOperationStats() map[OperationKey]OperationStats {
 			SampleCount:       len(samples.samples),
 			Window:            samples.window(),
 			Committed:         samples.outcomes[OutcomeCommitted],
-			Failed:            samples.outcomes[OutcomeFailed],
+			Reverted:          samples.outcomes[OutcomeReverted],
 			Expired:           samples.outcomes[OutcomeExpired],
 			DroppedAtCap:      samples.outcomes[OutcomeDroppedAtCap],
 			DroppedAtHandoff:  samples.outcomes[OutcomeDroppedAtHandoff],
@@ -405,15 +405,17 @@ type OperationStats struct {
 	SampleCount int
 	Window      time.Duration
 
-	// Three layers, not one. Committed and Failed are what the chain did.
+	// Three layers, not one. Committed and Reverted are what the chain did.
 	// Expired, DroppedAtCap and DroppedAtHandoff are what this run did to its
 	// own transactions, and none of the three is evidence about the chain.
 	// StatusUnavailable is what the run failed to see. Count and Successes above
 	// are the send path. Total them across layers and a generator throttling
 	// itself looks like a chain rejecting work.
 	//
-	// Failed here is the execution status a receipt reported. RunSummary.Failed
-	// is a send that returned an error. Opposite layers, same word.
+	// Reverted is the execution status a receipt reported: the chain took the
+	// transaction and it did nothing. RunSummary.Failed is a send that returned
+	// an error, which is a different layer, and the two carry different names so
+	// an operator reading one series beside the other cannot conflate them.
 	//
 	// Unrecorded has no legitimate producer. A non-zero value means sei-load
 	// classified a transaction wrongly, and nothing else produces one.
@@ -421,7 +423,7 @@ type OperationStats struct {
 	// All of them stay zero for a run with --track-receipts off, because the
 	// inclusion tracker is the only producer.
 	Committed         uint64
-	Failed            uint64
+	Reverted          uint64
 	Expired           uint64
 	DroppedAtCap      uint64
 	DroppedAtHandoff  uint64
diff --git a/stats/collector_outcome_test.go b/stats/collector_outcome_test.go
index 202c34a..a701114 100644
--- a/stats/collector_outcome_test.go
+++ b/stats/collector_outcome_test.go
@@ -22,14 +22,14 @@ func TestOutcomesAccumulatePerKey(t *testing.T) {
 	for i := 0; i < 3; i++ {
 		c.RecordOutcome(read, stats.OutcomeCommitted)
 	}
-	c.RecordOutcome(read, stats.OutcomeFailed)
+	c.RecordOutcome(read, stats.OutcomeReverted)
 	c.RecordOutcome(rmw, stats.OutcomeCommitted)
 
 	got := c.GetOperationStats()
 	require.Equal(t, uint64(3), got[read].Committed, "repeat calls must add, not overwrite")
-	require.Equal(t, uint64(1), got[read].Failed)
+	require.Equal(t, uint64(1), got[read].Reverted)
 	require.Equal(t, uint64(1), got[rmw].Committed, "one operation's outcomes reached another's count")
-	require.Zero(t, got[rmw].Failed)
+	require.Zero(t, got[rmw].Reverted)
 }
 
 // outcomeReaders pairs each state with the count it must reach. Keeping them in
@@ -41,7 +41,7 @@ var outcomeReaders = []struct {
 	read    func(stats.OperationStats) uint64
 }{
 	{"committed", stats.OutcomeCommitted, func(s stats.OperationStats) uint64 { return s.Committed }},
-	{"failed", stats.OutcomeFailed, func(s stats.OperationStats) uint64 { return s.Failed }},
+	{"reverted", stats.OutcomeReverted, func(s stats.OperationStats) uint64 { return s.Reverted }},
 	{"expired", stats.OutcomeExpired, func(s stats.OperationStats) uint64 { return s.Expired }},
 	{"dropped_at_cap", stats.OutcomeDroppedAtCap, func(s stats.OperationStats) uint64 { return s.DroppedAtCap }},
 	{"dropped_at_handoff", stats.OutcomeDroppedAtHandoff, func(s stats.OperationStats) uint64 { return s.DroppedAtHandoff }},
@@ -109,7 +109,7 @@ func TestAnUnknownOutcomeStaysVisible(t *testing.T) {
 // deliberate edit here rather than a side effect elsewhere.
 func TestOutcomeNamesAreStable(t *testing.T) {
 	require.Equal(t, "committed", stats.OutcomeCommitted.String())
-	require.Equal(t, "failed", stats.OutcomeFailed.String())
+	require.Equal(t, "reverted", stats.OutcomeReverted.String())
 	require.Equal(t, "expired", stats.OutcomeExpired.String())
 	require.Equal(t, "dropped_at_cap", stats.OutcomeDroppedAtCap.String())
 	require.Equal(t, "dropped_at_handoff", stats.OutcomeDroppedAtHandoff.String())
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 691c194..25caee7 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -34,7 +34,7 @@ func TestAllFailedReceiptsYieldZeroCommitted(t *testing.T) {
 
 	got := tr.collector.GetOperationStats()[key]
 	require.Zero(t, got.Committed, "a failed transaction counted as committed")
-	require.Equal(t, uint64(5), got.Failed)
+	require.Equal(t, uint64(5), got.Reverted)
 	require.Equal(t, uint64(5), tr.Summary().Included,
 		"they were included; inclusion and execution are different questions")
 }
@@ -62,7 +62,7 @@ func TestReceiptsSeparateCommittedFromFailed(t *testing.T) {
 
 	got := tr.collector.GetOperationStats()[key]
 	require.Equal(t, uint64(2), got.Committed)
-	require.Equal(t, uint64(2), got.Failed)
+	require.Equal(t, uint64(2), got.Reverted)
 	require.Zero(t, got.Unrecorded, "every matched transaction reached a real state")
 }
 
@@ -88,8 +88,8 @@ func TestOutcomesCarryTheOperation(t *testing.T) {
 
 	stats := tr.collector.GetOperationStats()
 	require.Equal(t, uint64(1), stats[OperationKey{Scenario: "storagerw", Operation: "read"}].Committed)
-	require.Equal(t, uint64(1), stats[OperationKey{Scenario: "storagerw", Operation: "write"}].Failed)
-	require.Zero(t, stats[OperationKey{Scenario: "storagerw", Operation: "read"}].Failed,
+	require.Equal(t, uint64(1), stats[OperationKey{Scenario: "storagerw", Operation: "write"}].Reverted)
+	require.Zero(t, stats[OperationKey{Scenario: "storagerw", Operation: "read"}].Reverted,
 		"one operation's outcome reached another's count")
 }
 
@@ -216,7 +216,7 @@ func TestReceiptWithoutStatusIsNotAFailure(t *testing.T) {
 	got := tr.collector.GetOperationStats()[key]
 	require.Equal(t, uint64(1), got.StatusUnavailable,
 		"an unreadable status was reported as a chain failure")
-	require.Zero(t, got.Failed)
+	require.Zero(t, got.Reverted)
 }
 
 // TestOutcomesPartitionEveryRegisteredTx is the guard stats/doc.go promises for
@@ -249,7 +249,7 @@ func TestOutcomesPartitionEveryRegisteredTx(t *testing.T) {
 	tr.matchBlock(context.Background(), 5, time.Unix(1002, 0))
 
 	got := tr.collector.GetOperationStats()[key]
-	sum := got.Committed + got.Failed + got.Expired + got.DroppedAtCap +
+	sum := got.Committed + got.Reverted + got.Expired + got.DroppedAtCap +
 		got.DroppedAtHandoff + got.StatusUnavailable + got.Unrecorded
 	inflight := tr.Summary().InflightAtShutdown
 	require.Equal(t, uint64(6), sum+inflight,
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index fc62255..43bab10 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -40,6 +40,11 @@ type blockReceipt struct {
 // receiptSource yields one block's receipts by number. Consumer-side interface
 // so tests can drive matching without a live chain. One call per block,
 // whatever the block's transaction count.
+//
+// One call is what this process issues. It is not what the call costs the node
+// that answers it, which grows with the square of the block's transaction
+// count. Point the tracker at a node that takes no send load, and see
+// config.ReceiptEndpoint.
 type receiptSource interface {
 	BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error)
 }
@@ -347,7 +352,7 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 			case r.Status == ethtypes.ReceiptStatusSuccessful:
 				outcome = OutcomeCommitted
 			default:
-				outcome = OutcomeFailed
+				outcome = OutcomeReverted
 			}
 			resolved = append(resolved, resolvedOutcome{
 				outcome:  outcome,
diff --git a/stats/outcome.go b/stats/outcome.go
index 0631fd7..c45a483 100644
--- a/stats/outcome.go
+++ b/stats/outcome.go
@@ -6,7 +6,7 @@ package stats
 // the inclusion_outcome metric and to Collector.RecordOutcome. The run report
 // does not read the collector's counts yet.
 //
-// Two distinctions carry the point of the type. Committed and Failed separate a
+// Two distinctions carry the point of the type. Committed and Reverted separate a
 // transaction that did what the workload asked from one that burned its gas
 // doing nothing, which an inclusion count cannot tell apart. And
 // StatusUnavailable separates "the run did not see" from "the chain did not take
@@ -16,7 +16,7 @@ package stats
 // Six states, and a residual. Every accepted transaction reaches exactly one of
 // them, or is still in the registry when the run ends:
 //
-//	accepted = committed + failed + status_unavailable
+//	accepted = committed + reverted + status_unavailable
 //	         + expired + dropped_at_cap + dropped_at_handoff
 //	         + inflight_at_shutdown
 type Outcome uint8
@@ -37,14 +37,16 @@ const (
 	// OutcomeCommitted is a receipt reporting a successful status.
 	OutcomeCommitted
 
-	// OutcomeFailed is a receipt reporting a failed status.
+	// OutcomeReverted is a receipt reporting a failed status. Named for what the
+	// EVM calls it, and named apart from RunSummary.Failed, which is a send that
+	// returned an error rather than a transaction the chain took and ran.
 	//
 	// A receipt carries one status bit. An explicit revert, an out-of-gas and an
 	// invalid opcode all arrive here. Separating them needs a trace call for
 	// every transaction, and this tracker reads status once per block however
 	// many transactions that block carries. So the state names what the run
 	// observed, not a cause it cannot see.
-	OutcomeFailed
+	OutcomeReverted
 
 	// OutcomeExpired is a transaction no receipt named within reapAfter.
 	//
@@ -93,7 +95,7 @@ const (
 var outcomeNames = [outcomeCount]string{
 	outcomeUnset:             "unrecorded",
 	OutcomeCommitted:         "committed",
-	OutcomeFailed:            "failed",
+	OutcomeReverted:          "reverted",
 	OutcomeExpired:           "expired",
 	OutcomeDroppedAtCap:      "dropped_at_cap",
 	OutcomeDroppedAtHandoff:  "dropped_at_handoff",

From 0047f6b65ac61e5e6f7288ac3ec6a3049b7c5695 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Wed, 26 Aug 2026 17:10:03 -0700
Subject: [PATCH 04/17] fix(stats): an idle block is not a block the run could
 not read

Round 2 found that the previous commit's own fix was worse than the bug it
removed. Three lenses reached it separately and one measured it: 80 of 100
consecutive Sei mainnet blocks return an empty receipts array.

Treating that as a hole marked almost every block. Because the marker only ever
grows and a transaction inherits it for its whole time in flight, one idle block
in a 30-second window converted the entire registry, and expired stopped being
reachable at all. The case the tool exists to detect makes it worse rather than
better: a chain that stopped accepting work produces nothing but idle blocks, so
the run would have reported "I could not see" for the one run where "the chain
took nothing" is the answer.

The premise behind that branch was wrong. sei-chain already separates the three
answers on the wire. A block it cannot see returns null, which arrives as
ethereum.NotFound. Pruned receipts return an error. An empty array means the
block carried no EVM transaction, and it is the truthful answer. So the branch
is gone rather than made conditional, and an array of nothing but nulls, which
is a node answering nothing at all, becomes an error where it is read.

The registry counters now split the way the outcomes do. They reached the
operator through the closing log line while the outcome ledger reached the same
operator through the metric, so one transaction was expired on the surface read
first and status_unavailable on the surface read second.

The preflight failed on the one case a Sei node never produces and passed on
every case it does. A node serving no EVM HTTP refuses the connection, which
classified as other and let the run start blind, and that is the case the doc
comment named first. A gateway that filters methods answers with an HTTP status
carrying the JSON-RPC code in a body the decoder never reads. Both refuse the
run now. Classification leads with the typed checks that hold across servers,
and the two substring tests that cannot be typed are ordered so pruning is
tested before availability, because Sei's two messages differ by one word.

The abort message named --receipt-endpoint. There is no such flag: the setting
is receiptEndpoint in the profile. The one error allowed to end a run told the
operator to use a control that does not exist, and a test now fails on the flag
spelling.

Register takes the context its caller already holds. The comment saying the
caller had none was false; the signature declined it.

A second registration of a hash already in flight overwrote the first, so two
accepted transactions shared one terminal state. It is counted now.

The partition guard covered three of seven states and passed with the whole reap
report loop deleted. It exercises every reachable state, asserts each leg, and
checks that the registry counters and the outcome ledger describe the same
transactions.

sender/doc.go is the file stats/doc.go nominates as owning the conservation
identity, and it still stated it with the retired word and with registered on
the left where dropped_at_cap sits on the right. The two shorter restatements
elsewhere are replaced by a pointer to it.

HasStatus detects a post-state-root receipt and no other shape. go-ethereum
makes both fields optional, so a receipt carrying neither is indistinguishable
from a failure after decoding, and the comment says so rather than implying a
guarantee.

Also: narrow becomes a plain function, matching every other helper in the
package; the cost claim names seid rather than the method, because upstream
go-ethereum is linear and the quadratic belongs to the implementation being
measured; the nolint directive naming a linter this repo does not run is gone
while its explanation stays; the fallback to the load endpoint warns; and the
README documents receiptEndpoint.

Guards proven by breaking what they cover: the idle block, the split counters,
the duplicate registration, the unreachable endpoint, and the all-null array.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 README.md                       |  15 +-
 config/config.go                |  19 ++-
 main.go                         |  14 +-
 sender/doc.go                   |  15 +-
 sender/sharded_sender.go        |   2 +-
 stats/doc.go                    |   2 +-
 stats/inclusion_outcome_test.go | 223 +++++++++++++++++++--------
 stats/inclusion_tracker.go      | 261 ++++++++++++++++++++++++--------
 stats/inclusion_tracker_test.go |  18 +--
 stats/metrics.go                |   2 +-
 stats/run_summary.go            |  15 +-
 11 files changed, 420 insertions(+), 166 deletions(-)

diff --git a/README.md b/README.md
index e762f7b..7d3dc46 100644
--- a/README.md
+++ b/README.md
@@ -21,6 +21,7 @@ Edit `my-config.json`:
 ```json
 {
   "endpoints": ["http://localhost:8545"],
+  "receiptEndpoint": "http://localhost:8546",
   "chainId": 713714,
   "scenarios": [
     {"name": "EVMTransfer", "weight": 100}
@@ -38,6 +39,18 @@ Edit `my-config.json`:
 }
 ```
 
+`endpoints` take the load. `receiptEndpoint` is the node the inclusion tracker
+reads receipts from, and it should be a node taking no send load. On seid, a
+receipts read costs the serving node work that grows with the square of the
+block's transaction count, so a node doing both degrades, and the tracker
+reports that degradation as a chain result. Leave it out and the tracker reads
+from `endpoints[0]`, which is fine for a local chain and not for a load run.
+
+Time `eth_getBlockReceipts` against the chain you are testing, on a block as
+wide as the run will produce, before you turn `trackReceipts` on. A read slower
+than the block interval makes the tracker blind, and a blind tracker reports
+`status_unavailable` rather than a number.
+
 ### 3. Run
 
 ```bash
@@ -54,7 +67,7 @@ Edit `my-config.json`:
 | `--buffer-size, -b` | 1000 | Sender queue size |
 | `--dry-run` | false | Simulate without sending |
 | `--debug` | false | Log each transaction |
-| `--track-receipts` | false | Enable the block-indexed tx→inclusion tracker (stamps InclusionTime; reports included/expired/inflight-at-shutdown) |
+| `--track-receipts` | false | Read each block's receipts and report what the chain did with every transaction: committed, reverted, expired, status-unavailable, dropped-at-cap, or still in flight. Set `receiptEndpoint` in the config with it |
 | `--inclusion-reap-after` | 30s | How long an un-included tx waits before reaping as expired (tune to expected inclusion time on congested chains) |
 | `--track-blocks` | false | Track block statistics |
 | `--track-user-latency` | false | Track user latency metrics |
diff --git a/config/config.go b/config/config.go
index 50b124a..e6121b2 100644
--- a/config/config.go
+++ b/config/config.go
@@ -36,17 +36,16 @@ type LoadConfig struct {
 	// ReceiptEndpoint is the node the inclusion tracker reads receipts from.
 	// Empty uses Endpoints[0].
 	//
-	// Name a node that takes no send load. eth_getBlockReceipts costs the node
-	// answering it work that grows with the square of the block's transaction
-	// count, because it resolves each receipt's index by walking the whole
-	// block. A node that both takes the send load and answers that read
-	// degrades, and the tracker reports the degradation as a chain result.
+	// Name a node that takes no send load. On seid, eth_getBlockReceipts costs
+	// the node answering it work that grows with the square of the block's
+	// transaction count. A node that both takes the send load and answers that
+	// read degrades, and the tracker reports the degradation as a chain result.
 	//
-	// This bounds what the tracking node can keep up with. Measure
-	// eth_getBlockReceipts against the target chain at the run's block width
-	// before enabling --track-receipts: a read that outruns the block interval
-	// makes the tracker blind, and a blind tracker reports status_unavailable
-	// rather than a number.
+	// This bounds what the tracking node can keep up with. Time
+	// eth_getBlockReceipts against the target chain, on a block as wide as the
+	// run will produce, before enabling trackReceipts. A read slower than the
+	// chain's block interval makes the tracker blind, and a blind tracker
+	// reports status_unavailable rather than a number.
 	ReceiptEndpoint string         `json:"receiptEndpoint,omitempty"`
 	Accounts        *AccountConfig `json:"accounts,omitempty"`
 	Scenarios       []Scenario     `json:"scenarios,omitempty"`
diff --git a/main.go b/main.go
index 924d9ec..99f749e 100644
--- a/main.go
+++ b/main.go
@@ -300,6 +300,10 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 			receiptEndpoint := cfg.ReceiptEndpoint
 			if receiptEndpoint == "" {
 				receiptEndpoint = cfg.Endpoints[0]
+				log.Printf("⚠️  Reading receipts from the load endpoint %s. "+
+					"Set receiptEndpoint to a node taking no send load: a receipts "+
+					"read costs the serving node work that grows with the block's "+
+					"transaction count.", receiptEndpoint)
 			}
 			s.SpawnBgNamed("inclusion tracker", func() error {
 				return inclusionTracker.Run(ctx, cfg.Endpoints[0], receiptEndpoint)
@@ -407,10 +411,16 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 		summary.InclusionTracked = true
 		summary.Included = incl.Included
 		summary.Expired = incl.Expired
+		summary.StatusUnavailable = incl.StatusUnavailable
 		summary.DroppedAtCap = incl.DroppedAtCap
 		summary.InflightAtShutdown = incl.InflightAtShutdown
-		log.Printf("📦 Inclusion: included=%d expired=%d dropped_at_cap=%d inflight_at_shutdown=%d",
-			incl.Included, incl.Expired, incl.DroppedAtCap, incl.InflightAtShutdown)
+		log.Printf("📦 Inclusion: included=%d expired=%d status_unavailable=%d dropped_at_cap=%d inflight_at_shutdown=%d",
+			incl.Included, incl.Expired, incl.StatusUnavailable, incl.DroppedAtCap,
+			incl.InflightAtShutdown)
+		if incl.StatusUnavailable > 0 {
+			log.Printf("⚠️  %d txs were in flight while a receipt read failed. "+
+				"The run cannot say whether the chain took them.", incl.StatusUnavailable)
+		}
 	}
 	collector.EmitRunSummary(ctx, summary)
 	if d := cfg.Settings.PostSummaryFlushDelay.ToDuration(); d > 0 {
diff --git a/sender/doc.go b/sender/doc.go
index 9d03c60..c1ba21e 100644
--- a/sender/doc.go
+++ b/sender/doc.go
@@ -94,18 +94,21 @@
 // receipts once (one request per block, whatever the block's tx count), and
 // stamps InclusionTime on every matched in-flight tx with the block's
 // header-ARRIVAL time. A receipt carries the execution status, so a matched tx
-// resolves to committed or failed rather than to one state covering both.
+// resolves to committed or reverted rather than to one state covering both.
 // Un-included txs are reaped after reapAfter. inclusion_latency (arrival minus
 // IntendedSendTime) is an open-loop-only measure; in closed-loop IntendedSendTime
 // is enqueue time, so the latency sample is omitted (counts are tracked in both).
 //
 // Conservation. Over [stats.Outcome]'s terminal states,
 //
-//	registered == committed + failed + status_unavailable
-//	            + expired + dropped_at_cap + inflight_at_shutdown
+//	accepted == committed + reverted + status_unavailable
+//	          + expired + dropped_at_cap + inflight_at_shutdown
 //
-// and registered ⊆ succeeded (only successful sends are registered).
-// dropped_at_handoff joins the identity with the hand-off channel. The inclusion
+// and accepted ⊆ succeeded (only a successful send is accepted). The left side
+// is accepted rather than registered, because a tx refused at the cap is
+// counted by dropped_at_cap and never entered the registry.
+// dropped_at_handoff joins the identity with the hand-off channel; nothing
+// produces it yet. The inclusion
 // denominator is succeeded (txs_accepted), never a minted "registered" series;
 // dropped_at_cap txs are excluded from it. inflight_at_shutdown is read only
 // after both the senders and the tracker have joined.
@@ -114,7 +117,7 @@
 // counted (block_gaps) but never backfilled, so its txs reap as expired —
 // an undercount of inclusions, never a miscount. (2) Reorgs use
 // first-observation-wins (stamp + delete). This fixes an execution status as
-// well as a time: a tx that failed on an orphaned block and committed on the
+// well as a time: a tx that reverted on an orphaned block and committed on the
 // canonical one keeps the first answer. Sei's finality makes the window small,
 // and there is no canonical reconciliation. (3) The tracker reads receipts from
 // receiptEndpoint when a run names one, and from Endpoints[0] otherwise. A
diff --git a/sender/sharded_sender.go b/sender/sharded_sender.go
index ca798ef..92214ac 100644
--- a/sender/sharded_sender.go
+++ b/sender/sharded_sender.go
@@ -124,7 +124,7 @@ func (ss *ShardedSender) Run(ctx context.Context) error {
 
 				// Queue for inclusion check.
 				if inclusion, ok := ss.inclusion.Get(); ok {
-					inclusion.Register(tx)
+					inclusion.Register(ctx, tx)
 				}
 				ss.queue.PopSent(addr)
 				return nil
diff --git a/stats/doc.go b/stats/doc.go
index 57412f2..1c66463 100644
--- a/stats/doc.go
+++ b/stats/doc.go
@@ -80,6 +80,6 @@
 //
 // sender/doc.go owns the conservation identity and states it there.
 // TestInclusion_Conservation asserts the registry identity, and
-// TestOutcomesPartitionEveryRegisteredTx asserts that Outcome's terminal states
+// TestOutcomesPartitionEveryAcceptedTx asserts that Outcome's terminal states
 // partition every registered transaction.
 package stats
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 25caee7..ed9a0d5 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -7,13 +7,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/sei-protocol/sei-load/types"
+
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/stretchr/testify/require"
 )
 
-// TestAllFailedReceiptsYieldZeroCommitted fails when a failed transaction and a
+// TestAllRevertedReceiptsYieldZeroCommitted fails when a reverted transaction and a
 // committed one land in the same terminal state.
-func TestAllFailedReceiptsYieldZeroCommitted(t *testing.T) {
+func TestAllRevertedReceiptsYieldZeroCommitted(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
 	key := OperationKey{Scenario: "storagerw", Operation: "write"}
@@ -22,7 +24,7 @@ func TestAllFailedReceiptsYieldZeroCommitted(t *testing.T) {
 	for i := uint64(1); i <= 5; i++ {
 		tx := loadTx(i, time.Unix(1000, 0))
 		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-		tr.Register(tx)
+		tr.Register(context.Background(), tx)
 		receipts = append(receipts, blockReceipt{
 			Hash:      tx.EthTx.Hash(),
 			Status:    ethtypes.ReceiptStatusFailed,
@@ -39,9 +41,9 @@ func TestAllFailedReceiptsYieldZeroCommitted(t *testing.T) {
 		"they were included; inclusion and execution are different questions")
 }
 
-// TestReceiptsSeparateCommittedFromFailed covers the mixed block, which is the
+// TestReceiptsSeparateCommittedFromReverted covers the mixed block, which is the
 // shape a real run produces. A block carrying both must split them.
-func TestReceiptsSeparateCommittedFromFailed(t *testing.T) {
+func TestReceiptsSeparateCommittedFromReverted(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
 	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
@@ -50,7 +52,7 @@ func TestReceiptsSeparateCommittedFromFailed(t *testing.T) {
 	for i := uint64(1); i <= 4; i++ {
 		tx := loadTx(i, time.Unix(1000, 0))
 		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-		tr.Register(tx)
+		tr.Register(context.Background(), tx)
 		status := ethtypes.ReceiptStatusSuccessful
 		if i%2 == 0 {
 			status = ethtypes.ReceiptStatusFailed
@@ -77,8 +79,8 @@ func TestOutcomesCarryTheOperation(t *testing.T) {
 	read.Scenario.Name, read.Scenario.Operation = "storagerw", "read"
 	write := loadTx(2, time.Unix(1000, 0))
 	write.Scenario.Name, write.Scenario.Operation = "storagerw", "write"
-	tr.Register(read)
-	tr.Register(write)
+	tr.Register(context.Background(), read)
+	tr.Register(context.Background(), write)
 
 	src.SetReceipts(3,
 		blockReceipt{Hash: read.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true},
@@ -108,7 +110,7 @@ func TestRequestsPerBlockDoNotTrackVolume(t *testing.T) {
 			var receipts []blockReceipt
 			for i := 1; i <= volume; i++ {
 				tx := loadTx(uint64(i), time.Unix(1000, 0))
-				tr.Register(tx)
+				tr.Register(context.Background(), tx)
 				receipts = append(receipts, blockReceipt{
 					Hash:      tx.EthTx.Hash(),
 					Status:    ethtypes.ReceiptStatusSuccessful,
@@ -135,7 +137,7 @@ func TestReapedTransactionsReachTheCollector(t *testing.T) {
 
 	tx := loadTx(1, time.Unix(1000, 0))
 	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-	tr.Register(tx)
+	tr.Register(context.Background(), tx)
 	time.Sleep(time.Millisecond)
 	tr.reap(context.Background())
 
@@ -146,35 +148,76 @@ func TestReapedTransactionsReachTheCollector(t *testing.T) {
 
 // TestUnreadableBlockIsNotAChainVerdict fails when a block the run could not
 // read is reported as a chain that left the transaction out. Expired is a claim
-// about the chain; a fetch that returned nothing supports no such claim.
+// about the chain; a read that failed supports no such claim.
 func TestUnreadableBlockIsNotAChainVerdict(t *testing.T) {
-	cases := []struct {
-		name  string
-		drive func(*MockBlockSource)
-	}{
-		{"fetch_error", func(s *MockBlockSource) { s.SetFetchErr(errors.New("connection refused")) }},
-		{"empty_array", func(s *MockBlockSource) { s.SetReceipts(9) }},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			src := NewMockBlockSource()
-			tr := newTestTracker(t, time.Nanosecond, 100, src)
-			key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
 
-			tx := loadTx(1, time.Unix(1000, 0))
-			tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-			tr.Register(tx)
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), tx)
 
-			tc.drive(src)
-			tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
-			tr.reap(context.Background())
+	src.SetFetchErr(errors.New("connection refused"))
+	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+	tr.reap(context.Background())
 
-			got := tr.collector.GetOperationStats()[key]
-			require.Equal(t, uint64(1), got.StatusUnavailable,
-				"a block the run never read was reported as a chain result")
-			require.Zero(t, got.Expired)
-		})
-	}
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a block the run never read was reported as a chain result")
+	require.Zero(t, got.Expired)
+}
+
+// TestAnIdleBlockIsNotAHole fails when a block carrying no EVM transaction is
+// counted as one the run could not read.
+//
+// It is the most common answer on a live chain: 80 of 100 consecutive Sei
+// mainnet blocks return an empty array. Counting those as holes makes expired
+// unreachable, and a chain that stopped accepting work produces nothing but
+// idle blocks, which is the one run where expired is the answer.
+func TestAnIdleBlockIsNotAHole(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), tx)
+
+	src.SetReceipts(9) // the node holds the block and it carried nothing
+	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+	tr.reap(context.Background())
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Expired,
+		"an idle block was read as a hole, so the chain's verdict was thrown away")
+	require.Zero(t, got.StatusUnavailable)
+}
+
+// TestDuplicateRegistrationReachesATerminalState fails when two accepted
+// transactions sharing one hash report one terminal state between them. The
+// registry holds one slot per hash, and a resend after a send timeout produces
+// exactly that collision.
+func TestDuplicateRegistrationReachesATerminalState(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), tx)
+	tr.Register(context.Background(), tx)
+
+	src.SetReceipts(3, blockReceipt{
+		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
+	})
+	tr.matchBlock(context.Background(), 3, time.Unix(1002, 0))
+
+	got := tr.collector.GetOperationStats()[key]
+	sum := got.Committed + got.Reverted + got.Expired + got.DroppedAtCap +
+		got.DroppedAtHandoff + got.StatusUnavailable + got.Unrecorded
+	require.Equal(t, uint64(2), sum+tr.Summary().InflightAtShutdown,
+		"two were accepted and %d reached a terminal state", sum)
 }
 
 // TestTransactionsRegisteredAfterTheHoleStillExpire fails when one unreadable
@@ -190,7 +233,7 @@ func TestTransactionsRegisteredAfterTheHoleStillExpire(t *testing.T) {
 
 	tx := loadTx(2, time.Unix(1000, 0))
 	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-	tr.Register(tx)
+	tr.Register(context.Background(), tx)
 	tr.reap(context.Background())
 
 	got := tr.collector.GetOperationStats()[key]
@@ -209,7 +252,7 @@ func TestReceiptWithoutStatusIsNotAFailure(t *testing.T) {
 
 	tx := loadTx(1, time.Unix(1000, 0))
 	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-	tr.Register(tx)
+	tr.Register(context.Background(), tx)
 	src.SetReceipts(4, blockReceipt{Hash: tx.EthTx.Hash(), HasStatus: false})
 	tr.matchBlock(context.Background(), 4, time.Unix(1002, 0))
 
@@ -219,43 +262,78 @@ func TestReceiptWithoutStatusIsNotAFailure(t *testing.T) {
 	require.Zero(t, got.Reverted)
 }
 
-// TestOutcomesPartitionEveryRegisteredTx is the guard stats/doc.go promises for
-// the seven-term identity. It fails when a registered transaction reaches no
-// terminal state, or reaches two.
-func TestOutcomesPartitionEveryRegisteredTx(t *testing.T) {
+// TestOutcomesPartitionEveryAcceptedTx is the guard stats/doc.go promises for
+// the identity sender/doc.go states. It fails when an accepted transaction
+// reaches no terminal state, or reaches two.
+//
+// Accepted rather than registered: a transaction refused at the cap never
+// entered the registry, and the identity counts it all the same.
+//
+// Every reachable terminal state has a leg here. Leaving the reap out was worth
+// a mutation test on its own, because dropping the whole reap report loop left
+// a partition guard that still passed.
+func TestOutcomesPartitionEveryAcceptedTx(t *testing.T) {
+	ctx := context.Background()
 	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Minute, 4, src)
+	// The cap admits four; the last two of six are refused at it.
+	tr := newTestTracker(t, 30*time.Millisecond, 4, src)
 	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
 
-	// Four land in a block, split committed and failed. Two more are refused at
-	// the cap. One is left in flight and then reaped.
-	var receipts []blockReceipt
-	for i := uint64(1); i <= 4; i++ {
-		tx := loadTx(i, time.Unix(1000, 0))
+	register := func(nonce uint64) *types.LoadTx {
+		tx := loadTx(nonce, time.Unix(1000, 0))
 		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-		tr.Register(tx)
+		tr.Register(ctx, tx)
+		return tx
+	}
+
+	// Two commit and one reverts in block 5. The fourth is admitted and never
+	// named by a receipt, so it reaps as expired.
+	var receipts []blockReceipt
+	for i := uint64(1); i <= 3; i++ {
 		status := ethtypes.ReceiptStatusSuccessful
-		if i > 2 {
+		if i == 3 {
 			status = ethtypes.ReceiptStatusFailed
 		}
-		receipts = append(receipts, blockReceipt{Hash: tx.EthTx.Hash(), Status: status, HasStatus: true})
-	}
-	for i := uint64(5); i <= 6; i++ {
-		tx := loadTx(i, time.Unix(1000, 0))
-		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-		tr.Register(tx)
+		receipts = append(receipts, blockReceipt{
+			Hash: register(i).EthTx.Hash(), Status: status, HasStatus: true,
+		})
 	}
-	src.SetReceipts(5, receipts[:3]...)
-	tr.matchBlock(context.Background(), 5, time.Unix(1002, 0))
+	register(4)
+	// Two refused at the cap.
+	register(5)
+	register(6)
+
+	src.SetReceipts(5, receipts...)
+	tr.matchBlock(ctx, 5, time.Unix(1002, 0))
+
+	// A read that fails puts the run in a state it cannot attribute, so the
+	// fourth reaps as status_unavailable rather than expired.
+	src.SetFetchErr(errors.New("connection refused"))
+	tr.matchBlock(ctx, 6, time.Unix(1003, 0))
+	time.Sleep(40 * time.Millisecond)
+	tr.reap(ctx)
 
 	got := tr.collector.GetOperationStats()[key]
 	sum := got.Committed + got.Reverted + got.Expired + got.DroppedAtCap +
 		got.DroppedAtHandoff + got.StatusUnavailable + got.Unrecorded
-	inflight := tr.Summary().InflightAtShutdown
-	require.Equal(t, uint64(6), sum+inflight,
-		"six registered, %d reached a terminal state and %d are still in flight",
-		sum, inflight)
+	summary := tr.Summary()
+	require.Equal(t, uint64(6), sum+summary.InflightAtShutdown,
+		"six accepted, %d reached a terminal state and %d are still in flight",
+		sum, summary.InflightAtShutdown)
+
+	// Each leg is reachable, so a later change cannot satisfy the sum by
+	// collapsing two states into one.
+	require.Equal(t, uint64(2), got.Committed)
+	require.Equal(t, uint64(1), got.Reverted)
+	require.Equal(t, uint64(2), got.DroppedAtCap)
+	require.Equal(t, uint64(1), got.StatusUnavailable)
 	require.Zero(t, got.Unrecorded, "an outcome no caller may report was recorded")
+
+	// The registry counters and the outcome ledger describe the same
+	// transactions, so an operator reading the closing log line and the metric
+	// cannot see one transaction counted under two names.
+	require.Equal(t, got.Expired, summary.Expired)
+	require.Equal(t, got.StatusUnavailable, summary.StatusUnavailable)
 }
 
 // TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer fails when a run against
@@ -269,6 +347,8 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 		wantErr bool
 	}{
 		{"method_absent", errors.New("the method eth_getBlockReceipts does not exist/is not available"), true},
+		{"not_listening", errors.New("dial tcp 10.0.0.1:8545: connect: connection refused"), true},
+		{"no_such_host", errors.New("dial tcp: lookup rpc-0: no such host"), true},
 		{"node_busy", errors.New("context deadline exceeded"), false},
 		{"node_behind", errors.New("not found"), false},
 	}
@@ -282,7 +362,10 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 				require.NoError(t, err, "a busy endpoint failed the run instead of being retried")
 				return
 			}
-			require.ErrorContains(t, err, "does not serve eth_getBlockReceipts")
+			require.ErrorContains(t, err, "receiptEndpoint",
+				"the error names no setting the operator can actually change")
+			require.NotContains(t, err.Error(), "--receipt-endpoint",
+				"the error names a flag that does not exist")
 			require.ErrorContains(t, err, "http://node:8545", "the error does not name the endpoint")
 		})
 	}
@@ -293,11 +376,23 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 // and loses every result it gathered.
 func TestNilReceiptDoesNotEndTheRun(t *testing.T) {
 	hash := loadTx(1, time.Unix(1000, 0)).EthTx.Hash()
-	src := ethReceiptSource{}
-	got := src.narrow([]*ethtypes.Receipt{
+	got, err := narrowReceipts([]*ethtypes.Receipt{
 		nil,
 		{TxHash: hash, Status: ethtypes.ReceiptStatusSuccessful},
 		nil,
 	})
+	require.NoError(t, err)
 	require.Equal(t, []blockReceipt{{Hash: hash, Status: 1, HasStatus: true}}, got)
+
+	// Every element null is a node answering nothing, not a block holding
+	// nothing. It has to reach the caller as an error or it reads as an idle
+	// block and the run silently loses those transactions.
+	_, err = narrowReceipts([]*ethtypes.Receipt{nil, nil})
+	require.ErrorContains(t, err, "null")
+
+	// An empty array is a block that carried no EVM transaction. It is the
+	// answer for most blocks on an idle chain, and it is not an error.
+	got, err = narrowReceipts(nil)
+	require.NoError(t, err)
+	require.Empty(t, got)
 }
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 43bab10..c0188c3 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -1,13 +1,18 @@
 package stats
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"log"
+	"net"
+	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum"
 	"github.com/ethereum/go-ethereum/common"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -31,6 +36,12 @@ import (
 // HasStatus is false for a receipt that carries a post-state root instead of a
 // status, which is a receipt whose execution result cannot be read. Status is
 // meaningless when it is false.
+//
+// It detects that one shape and no other. go-ethereum makes both the status and
+// the root optional when it decodes, so a receipt carrying neither arrives here
+// indistinguishable from one reporting a failure, and this run reads it as a
+// failure. Sei always writes a status, so the undetectable shape does not come
+// from the chain this tracks.
 type blockReceipt struct {
 	Hash      common.Hash
 	Status    uint64
@@ -42,8 +53,11 @@ type blockReceipt struct {
 // whatever the block's transaction count.
 //
 // One call is what this process issues. It is not what the call costs the node
-// that answers it, which grows with the square of the block's transaction
-// count. Point the tracker at a node that takes no send load, and see
+// that answers it. seid resolves each receipt's index by walking the whole
+// block and recovering every sender again, so its work grows with the square of
+// the block's transaction count. Upstream go-ethereum does not do this; the
+// cost belongs to the implementation being measured, not to the method. Point
+// the tracker at a node that takes no send load, and see
 // config.ReceiptEndpoint.
 type receiptSource interface {
 	BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error)
@@ -60,15 +74,16 @@ func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockR
 	if err != nil {
 		return nil, err
 	}
-	return s.narrow(receipts), nil
+	return narrowReceipts(receipts)
 }
 
-// narrow keeps the two fields the tracker reads and drops the rest.
+// narrowReceipts keeps what the tracker reads and drops the rest.
 //
 // An endpoint decides what it puts in this array. A nil element would panic the
 // head loop, which ends the run and loses every result it gathered, so drop it
-// rather than trust the shape.
-func (ethReceiptSource) narrow(receipts []*ethtypes.Receipt) []blockReceipt {
+// rather than trust the shape. An array that held only nil elements is a node
+// answering nothing at all, which is an error rather than an empty block.
+func narrowReceipts(receipts []*ethtypes.Receipt) ([]blockReceipt, error) {
 	out := make([]blockReceipt, 0, len(receipts))
 	for _, r := range receipts {
 		if r == nil {
@@ -80,24 +95,40 @@ func (ethReceiptSource) narrow(receipts []*ethtypes.Receipt) []blockReceipt {
 			HasStatus: len(r.PostState) == 0,
 		})
 	}
-	return out
+	if len(receipts) > 0 && len(out) == 0 {
+		return nil, fmt.Errorf("every one of %d receipts was null", len(receipts))
+	}
+	return out, nil
 }
 
 type entry struct {
 	tx           *types.LoadTx
 	registeredAt time.Time
-	// blindFetches is inclusionState.blindFetches at registration. A reap
-	// compares it against the current count: a higher count means a receipt
-	// fetch failed while this tx was in flight, so the run cannot say the chain
-	// left it out. See reap.
-	blindFetches uint64
+	// blindFetchesAtRegistration is inclusionState.blindFetches at the moment
+	// this tx was registered. A reap compares it against the count now: a
+	// higher count means a receipt fetch failed while this tx was in flight, so
+	// the run cannot say the chain left it out. See reap.
+	blindFetchesAtRegistration uint64
 }
 
 type inclusionState struct {
-	// blindFetches counts receipt fetches that returned nothing usable. It only
-	// ever grows, so an entry's terminal state follows from comparing this
-	// against the value it recorded at registration.
-	blindFetches       uint64
+	// blindFetches counts receipt reads that failed. It only ever grows, so an
+	// entry's terminal state follows from comparing this against the value it
+	// recorded at registration.
+	//
+	// The count carries no height, so it marks a tx in flight across any failed
+	// read rather than a tx in the block that read failed on. The tracker does
+	// not know which registered txs a block it never read was holding, and
+	// claiming to would be the invention this state exists to avoid.
+	blindFetches uint64
+	// statusUnavailable counts reaped txs that spanned a failed read, plus
+	// duplicate registrations. It is the part the run cannot attribute to the
+	// chain.
+	statusUnavailable uint64
+	// duplicates counts registrations of a hash already in flight. The registry
+	// holds one slot per hash, so the second one has no place to go and the run
+	// can say nothing about it.
+	duplicates         uint64
 	inflight           map[common.Hash]*entry
 	included           uint64
 	expired            uint64
@@ -105,9 +136,12 @@ type inclusionState struct {
 	inflightAtShutdown uint64
 }
 
-// InclusionTracker matches arriving blocks against in-flight txs to stamp
-// InclusionTime. Conservation: registered == included + expired +
-// inflight_at_shutdown, and registered ⊆ succeeded (see sender/doc.go).
+// InclusionTracker matches arriving blocks against in-flight txs, stamps
+// InclusionTime, and resolves each one to a terminal [Outcome].
+//
+// sender/doc.go states the conservation identity these outcomes satisfy. It is
+// not restated here: it was, in a shorter form, and the two drifted apart when
+// the states grew.
 type InclusionTracker struct {
 	seiChainID  string
 	reapAfter   time.Duration
@@ -175,22 +209,29 @@ func newInclusionTrackerWithSource(t *InclusionTracker, source receiptSource) *I
 // Register hands ownership of tx's InclusionTime to the tracker. Caller must
 // invoke it only for successful sends, at send-completion, so
 // registered ⊆ succeeded holds. At cap the tx is dropped and counted.
-func (t *InclusionTracker) Register(tx *types.LoadTx) {
+func (t *InclusionTracker) Register(ctx context.Context, tx *types.LoadTx) {
 	hash := tx.EthTx.Hash()
-	var droppedAtCap bool
+	var outcome Outcome
 	for s := range t.state.Lock() {
 		// Cap check and insert share one critical section: race-free admission.
 		if len(s.inflight) >= t.maxInflight {
 			s.droppedAtCap++
-			droppedAtCap = true
+			outcome = OutcomeDroppedAtCap
+			break
+		}
+		// One map slot per hash, so a second registration of the same hash
+		// would overwrite the first and the run would report one terminal state
+		// for two accepted transactions. A resend after a send timeout produces
+		// exactly that. Count the duplicate rather than lose it.
+		if _, dup := s.inflight[hash]; dup {
+			s.duplicates++
+			outcome = OutcomeStatusUnavailable
 			break
 		}
-		s.inflight[hash] = &entry{tx: tx, registeredAt: time.Now(), blindFetches: s.blindFetches}
+		s.inflight[hash] = &entry{tx: tx, registeredAt: time.Now(), blindFetchesAtRegistration: s.blindFetches}
 	}
-	if droppedAtCap {
-		// The sender calls this from the send path and carries no context here,
-		// so this one report has none to pass.
-		t.report(context.Background(), OutcomeDroppedAtCap, tx.Scenario)
+	if outcome != outcomeUnset {
+		t.report(ctx, outcome, tx.Scenario)
 	}
 }
 
@@ -278,29 +319,42 @@ func (t *InclusionTracker) processHead(ctx context.Context, num uint64, arrival
 	return num
 }
 
-// preflight proves the receipt endpoint answers before the run starts. A node in
-// validator or seed mode serves no EVM HTTP at all, and a node may deny the
-// method by name. Without this the run completes, reports every transaction
+// preflight proves the endpoint can answer a receipts read before the tracker
+// starts matching. Without it the run completes, reports every transaction
 // un-included, and exits zero, which reads as a chain that accepted nothing.
+//
+// It asks for height 0, which proves the endpoint speaks the method and nothing
+// about what state it holds: Sei answers genesis from a constant, ahead of its
+// watermark and its receipt store. That is the point. A probe of the head would
+// fail on a node that is merely a block behind, which is ordinary and not a
+// reason to refuse a run.
+//
+// Two failures refuse the run, and they are the two that stay broken for its
+// whole length. A node that does not answer at all is one: validator and seed
+// modes serve no EVM HTTP, so a run pointed at either finds nothing listening.
+// A node that answers and refuses the method is the other. Anything else is a
+// node busy or behind, which the run reports as it finds.
 func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error {
 	probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
-	head, err := t.source.BlockReceipts(probeCtx, 0)
-	// Height 0 is the genesis block: every node that serves the method answers
-	// it, and no node needs to hold recent state to do so. An empty list is a
-	// pass, because genesis carries no EVM transactions.
-	_ = head
+	_, err := t.source.BlockReceipts(probeCtx, 0)
 	if err == nil {
 		return nil
 	}
-	if reason := fetchFailureReason(err); reason == "method_unavailable" {
+	switch fetchFailureReason(err) {
+	case reasonMethodUnavailable:
 		return fmt.Errorf(
-			"inclusion tracker: %s does not serve eth_getBlockReceipts (%w). "+
-				"Point --receipt-endpoint at a node in fullNode or archive mode",
+			"inclusion tracker: %s answers, but not eth_getBlockReceipts (%w). "+
+				"Set receiptEndpoint in the profile to a node in fullNode or "+
+				"archive mode, which are the modes that serve EVM HTTP",
+			endpoint, err)
+	case reasonUnreachable:
+		return fmt.Errorf(
+			"inclusion tracker: %s is not serving EVM JSON-RPC (%w). "+
+				"Set receiptEndpoint in the profile to a node in fullNode or "+
+				"archive mode; validator and seed modes serve no EVM HTTP",
 			endpoint, err)
 	}
-	// Any other error is the endpoint being busy or behind, not the endpoint
-	// being wrong. The run proceeds and reports what it sees.
 	log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
 	return nil
 }
@@ -313,21 +367,19 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 	receipts, err := t.source.BlockReceipts(fetchCtx, num)
 	cancel()
 	if err != nil {
-		// No retry: a retry piles RPC onto an endpoint already failing. The
-		// block goes unmatched, and blindFetches records that this run has a
-		// hole, so a tx in flight across it reaps as status_unavailable rather
-		// than as a verdict about the chain.
+		// The block goes unmatched, and blindFetches records that this run has
+		// a hole, so a tx in flight across it reaps as status_unavailable
+		// rather than as a verdict about the chain.
+		//
+		// An empty slice is not this case. A node that holds a block carrying
+		// no EVM transaction answers it truthfully with an empty array, and a
+		// node that holds nothing answers null, which arrives here as
+		// ethereum.NotFound. Reading an empty array as a hole would mark every
+		// idle block, and a chain that stopped accepting work produces nothing
+		// but idle blocks, which is the one run where expired is the answer.
 		t.recordBlindFetch(ctx, num, fetchFailureReason(err), err)
 		return
 	}
-	if len(receipts) == 0 {
-		// An empty array and an error are the same answer to this tracker and
-		// arrive differently: a node that holds the block but has lost its
-		// receipt bodies returns an empty array with no error. Untreated it
-		// reaps a whole block of transactions as expired, silently.
-		t.recordBlindFetch(ctx, num, "empty", nil)
-		return
-	}
 	matched := make([]inclusionSample, 0, len(receipts))
 	resolved := make([]resolvedOutcome, 0, len(receipts))
 	for s := range t.state.Lock() {
@@ -441,15 +493,23 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 				continue
 			}
 			delete(s.inflight, h)
-			s.expired++
 			// A fetch failed while this tx was in flight, so the block that
 			// would have carried it was never read. The chain may well have
 			// included it. Expired would report a chain problem where the truth
 			// is a measurement problem, which is what Outcome's two states are
 			// for.
+			//
+			// The registry counters split the same way the outcomes do. They
+			// reach the operator through InclusionSummary and the closing log
+			// line, so letting them stay merged would print one transaction as
+			// expired on the surface an operator reads first and as
+			// status_unavailable on the one they read second.
 			outcome := OutcomeExpired
-			if s.blindFetches > e.blindFetches {
+			if s.blindFetches > e.blindFetchesAtRegistration {
 				outcome = OutcomeStatusUnavailable
+				s.statusUnavailable++
+			} else {
+				s.expired++
 			}
 			expired = append(expired, resolvedOutcome{outcome: outcome, scenario: e.tx.Scenario})
 		}
@@ -478,29 +538,97 @@ func (t *InclusionTracker) recordBlindFetch(ctx context.Context, num uint64, rea
 		attribute.String("reason", reason)))
 }
 
-// fetchFailureReason buckets a receipt-fetch error so an operator reads the
-// cause off a dashboard instead of the pod log. The strings are label values:
-// keep them few, and keep them stable.
+// Reasons a receipt read failed. They are metric label values and preflight
+// branches on two of them, so one constant owns each string rather than a
+// literal at every site.
+const (
+	reasonMethodUnavailable = "method_unavailable"
+	reasonUnreachable       = "unreachable"
+	reasonPruned            = "pruned"
+	reasonNotFound          = "not_found"
+	reasonTimeout           = "timeout"
+	reasonOther             = "other"
+)
+
+// methodNotFoundCode is JSON-RPC 2.0's "Method not found". Every server returns
+// it under that code whatever prose it puts beside it, so the code is what this
+// matches on.
+const methodNotFoundCode = -32601
+
+// fetchFailureReason buckets a receipt-read error so an operator reads the cause
+// off a dashboard instead of the pod log. The strings are label values: keep
+// them few, and keep them stable.
+//
+// The typed checks come first because they hold across servers. The substring
+// checks below them do not, and they are here because no typed error exists for
+// what they catch. Sei writes "receipts have been pruned; earliest available is
+// N" for one and "not yet available" for the other, which differ by one word, so
+// pruning is tested before anything matching on availability.
 func fetchFailureReason(err error) string {
+	var rpcErr rpc.Error
+	if errors.As(err, &rpcErr) && rpcErr.ErrorCode() == methodNotFoundCode {
+		return reasonMethodUnavailable
+	}
+	// A gateway that filters methods answers with an HTTP status, and the
+	// JSON-RPC code sits in the body where the decoder never looks. Reading the
+	// body is what separates "this node will never serve the method" from "this
+	// node is rate-limiting me", which are different operator actions.
+	var httpErr rpc.HTTPError
+	if errors.As(err, &httpErr) {
+		if bytes.Contains(httpErr.Body, []byte(strconv.Itoa(methodNotFoundCode))) {
+			return reasonMethodUnavailable
+		}
+		if httpErr.StatusCode == http.StatusNotFound {
+			return reasonUnreachable
+		}
+		return reasonOther
+	}
+	if errors.Is(err, ethereum.NotFound) {
+		return reasonNotFound
+	}
+	if errors.Is(err, context.DeadlineExceeded) {
+		return reasonTimeout
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) && netErr.Timeout() {
+		return reasonTimeout
+	}
+	// A node serving no EVM HTTP refuses the connection or resolves to nothing,
+	// and a URL pointing somewhere else answers with prose rather than JSON.
+	var opErr *net.OpError
+	var dnsErr *net.DNSError
+	if errors.As(err, &opErr) || errors.As(err, &dnsErr) {
+		return reasonUnreachable
+	}
 	switch msg := strings.ToLower(err.Error()); {
-	case strings.Contains(msg, "does not exist") || strings.Contains(msg, "not available"):
-		return "method_unavailable"
 	case strings.Contains(msg, "pruned"):
-		return "pruned"
+		return reasonPruned
+	case strings.Contains(msg, "does not exist") || strings.Contains(msg, "not available"):
+		return reasonMethodUnavailable
+	case strings.Contains(msg, "connection refused") ||
+		strings.Contains(msg, "no such host") ||
+		strings.Contains(msg, "looking for beginning of value"):
+		return reasonUnreachable
 	case strings.Contains(msg, "not found"):
-		return "not_found"
-	case errors.Is(err, context.DeadlineExceeded) || strings.Contains(msg, "deadline exceeded"):
-		return "timeout"
+		return reasonNotFound
+	case strings.Contains(msg, "deadline exceeded"):
+		return reasonTimeout
 	default:
-		return "other"
+		return reasonOther
 	}
 }
 
 // InclusionSummary is the conservation tally. Read only after both sender and
 // the tracker have joined, so inflightAtShutdown is final.
 type InclusionSummary struct {
-	Included           uint64
-	Expired            uint64
+	Included uint64
+	// Expired is a tx the run read every block for and never saw. It is a claim
+	// about the chain.
+	Expired uint64
+	// StatusUnavailable is a tx that was in flight while a receipt read failed.
+	// It is a claim about the run, and the two are counted apart so the closing
+	// log line and the outcome ledger cannot disagree about one transaction.
+	StatusUnavailable  uint64
 	DroppedAtCap       uint64
 	InflightAtShutdown uint64
 }
@@ -512,6 +640,7 @@ func (t *InclusionTracker) Summary() InclusionSummary {
 		return InclusionSummary{
 			Included:           s.included,
 			Expired:            s.expired,
+			StatusUnavailable:  s.statusUnavailable + s.duplicates,
 			DroppedAtCap:       s.droppedAtCap,
 			InflightAtShutdown: s.inflightAtShutdown,
 		}
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index 0b19c41..660dfa6 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -120,7 +120,7 @@ func TestInclusion_MatchStamps(t *testing.T) {
 
 	intended := time.Unix(1000, 0)
 	tx := loadTx(1, intended)
-	tr.Register(tx)
+	tr.Register(context.Background(), tx)
 	require.Equal(t, 1, inflightLen(t, tr))
 
 	arrival := time.Unix(1002, 0)
@@ -143,7 +143,7 @@ func TestInclusion_ClosedLoopCountsNoLatency(t *testing.T) {
 	require.False(t, tr.openLoop, "tracker built closed-loop: latency sample is gated")
 
 	tx := loadTx(1, time.Unix(1000, 0))
-	tr.Register(tx)
+	tr.Register(context.Background(), tx)
 	arrival := time.Unix(1002, 0)
 	src.SetBlock(5, tx.EthTx.Hash())
 	tr.matchBlock(context.Background(), 5, arrival)
@@ -157,7 +157,7 @@ func TestInclusion_ReapExpires(t *testing.T) {
 	tr := newTestTracker(t, 10*time.Millisecond, 100, NewMockBlockSource())
 
 	tx := loadTx(1, time.Now())
-	tr.Register(tx)
+	tr.Register(context.Background(), tx)
 	require.Equal(t, 1, inflightLen(t, tr))
 
 	time.Sleep(20 * time.Millisecond)
@@ -176,7 +176,7 @@ func TestInclusion_ReapVsLateInclusion(t *testing.T) {
 		src := NewMockBlockSource()
 		tr := newTestTracker(t, time.Nanosecond, 100, src)
 		tx := loadTx(1, time.Unix(1000, 0))
-		tr.Register(tx)
+		tr.Register(context.Background(), tx)
 		time.Sleep(time.Millisecond)
 		tr.reap(context.Background()) // wins: expired
 		src.SetBlock(5, tx.EthTx.Hash())
@@ -190,7 +190,7 @@ func TestInclusion_ReapVsLateInclusion(t *testing.T) {
 		src := NewMockBlockSource()
 		tr := newTestTracker(t, time.Nanosecond, 100, src)
 		tx := loadTx(1, time.Unix(1000, 0))
-		tr.Register(tx)
+		tr.Register(context.Background(), tx)
 		src.SetBlock(5, tx.EthTx.Hash())
 		tr.matchBlock(context.Background(), 5, time.Unix(1002, 0)) // wins: included
 		time.Sleep(time.Millisecond)
@@ -208,7 +208,7 @@ func TestInclusion_BoundedCap(t *testing.T) {
 	tr := newTestTracker(t, time.Minute, cap, NewMockBlockSource())
 
 	for i := range uint64(10) {
-		tr.Register(loadTx(i, time.Now()))
+		tr.Register(context.Background(), loadTx(i, time.Now()))
 		require.LessOrEqual(t, inflightLen(t, tr), cap, "map never exceeds cap")
 	}
 	s := tr.Summary()
@@ -221,7 +221,7 @@ func TestInclusion_BoundedCap(t *testing.T) {
 func TestInclusion_NonPositiveCapFallsBack(t *testing.T) {
 	tr := newTestTracker(t, time.Minute, 0, NewMockBlockSource())
 	for i := range uint64(5) {
-		tr.Register(loadTx(i, time.Now()))
+		tr.Register(context.Background(), loadTx(i, time.Now()))
 	}
 	require.Equal(t, 5, inflightLen(t, tr), "registrations are admitted, not all dropped")
 	require.Equal(t, uint64(0), tr.Summary().DroppedAtCap)
@@ -269,7 +269,7 @@ func TestInclusion_Conservation(t *testing.T) {
 			txs := make([]*types.LoadTx, tc.attempts)
 			for i := range txs {
 				txs[i] = loadTx(uint64(i), time.Unix(1000, 0))
-				tr.Register(txs[i])
+				tr.Register(context.Background(), txs[i])
 			}
 			for i := 0; i < tc.matched; i++ {
 				src.SetBlock(uint64(i), txs[i].EthTx.Hash())
@@ -330,7 +330,7 @@ func TestInclusion_ConcurrentRaceSafe(t *testing.T) {
 	go func() {
 		defer wg.Done()
 		for i := range txs {
-			tr.Register(txs[i])
+			tr.Register(context.Background(), txs[i])
 		}
 	}()
 	go func() {
diff --git a/stats/metrics.go b/stats/metrics.go
index 452d208..eb6ef34 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -78,7 +78,7 @@ var (
 
 	inclusionBlockFetchErrors = must(meter.Int64Counter(
 		"block_fetch_errors",
-		metric.WithDescription("Receipt reads that returned nothing usable, by reason; the block goes unmatched and its txs reach status_unavailable (no retry)"),
+		metric.WithDescription("Receipt reads that failed, by reason; that block goes unmatched and no retry follows, so a tx in flight across it reaches status_unavailable"),
 		metric.WithUnit("{blocks}")))
 
 	// Run-summary: the only inclusion tally with no live series, since it is the
diff --git a/stats/run_summary.go b/stats/run_summary.go
index 9b46633..7a94f6e 100644
--- a/stats/run_summary.go
+++ b/stats/run_summary.go
@@ -26,16 +26,21 @@ type RunSummary struct {
 	// scheduled = dropped + succeeded + failed) is auditable from the run summary.
 	Failed uint64
 
-	// Inclusion-stage tally (see sender/doc.go). The conservation identity is
-	// registered == Included + Expired + InflightAtShutdown, with
-	// registered ⊆ succeeded. InclusionTracked disambiguates a not-tracked run
-	// (all zero, flag false) from a tracked run with no inclusions yet.
+	// Inclusion-stage tally. sender/doc.go states the conservation identity these
+	// satisfy; do not restate it here, because it has already drifted once.
+	// InclusionTracked disambiguates a not-tracked run (all zero, flag false)
+	// from a tracked run with no inclusions yet.
 	// TODO(PLT-467): owns run-summary schema versioning for these fields.
 	InclusionTracked bool
 	// Included is the count of txs the tracker observed on-chain (stamped).
 	Included uint64
-	// Expired is the count of registered txs reaped un-included after reapAfter.
+	// Expired is the count of registered txs the run read every block for and
+	// never saw. It is a claim about the chain.
 	Expired uint64
+	// StatusUnavailable is the count of registered txs that were in flight while
+	// a receipt read failed. It is a claim about the run, and it is counted
+	// apart from Expired so the two cannot be read as one number.
+	StatusUnavailable uint64
 	// DroppedAtCap is the count of successful sends rejected at the in-flight cap;
 	// excluded from the inclusion denominator (they were never registered).
 	DroppedAtCap uint64

From 42efb65dc7efcfb15df8d80246b622f9032c380a Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Wed, 26 Aug 2026 17:12:16 -0700
Subject: [PATCH 05/17] fix(stats): re-read a height the receipt node has not
 reached

Shipping receiptEndpoint is what makes this reachable, so it belongs with it.
Two reviewers named the same sequence: heads arrive from the load node, the
receipt node has not committed that height yet, and it answers null. That
arrives as ethereum.NotFound, which counted as a hole with no retry, so a
receipt node trailing by one block produced a hole every block and expired
became unreachable again. The defect this PR just removed, reintroduced through
the topology the PR recommends.

A height the node has not reached is now re-read on the next head. One re-read
is the bound: a node still behind a block interval later is behind rather than
busy, and that is a hole worth counting.

This is TOT-020 pulled forward from phase 3c, for the same reason the unreadable
block attribution came forward from 3b. Leaving it out means merging a change
whose recommended configuration breaks it.

Guards proven by breaking what they cover: a lagging node written off on first
sight, and a node behind forever retried forever.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_outcome_test.go | 60 +++++++++++++++++++++++++++++++++
 stats/inclusion_tracker.go      | 52 +++++++++++++++++++++++++++-
 2 files changed, 111 insertions(+), 1 deletion(-)

diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index ed9a0d5..da31f1a 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/ethereum/go-ethereum"
 	"github.com/sei-protocol/sei-load/types"
 
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
@@ -396,3 +397,62 @@ func TestNilReceiptDoesNotEndTheRun(t *testing.T) {
 	require.NoError(t, err)
 	require.Empty(t, got)
 }
+
+// TestALaggingReceiptNodeIsRetriedNotCountedAsAHole fails when a height the
+// receipt node has not reached becomes a hole on first sight.
+//
+// It is the normal case once receiptEndpoint names a second node: the head
+// arrives from one node and the other has not committed that height yet.
+// Counting it would mark a hole per block, which is how expired became
+// unreachable before.
+func TestALaggingReceiptNodeIsRetriedNotCountedAsAHole(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	// Head 7 arrives before the receipt node holds it.
+	src.SetFetchErr(ethereum.NotFound)
+	last := tr.processHead(ctx, 7, time.Unix(1002, 0), 6)
+	require.Equal(t, uint64(7), last)
+
+	// By the next head it has caught up, and the re-read finds the transaction.
+	src.SetFetchErr(nil)
+	src.SetReceipts(7, blockReceipt{
+		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
+	})
+	tr.processHead(ctx, 8, time.Unix(1003, 0), 7)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Committed,
+		"a height the node had not reached was written off instead of re-read")
+	require.Zero(t, got.StatusUnavailable)
+}
+
+// TestAReceiptNodeThatStaysBehindBecomesAHole fails when a node that never
+// catches up is retried forever. One re-read is the bound: a node still behind
+// a block interval later is behind rather than busy.
+func TestAReceiptNodeThatStaysBehindBecomesAHole(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	src.SetFetchErr(ethereum.NotFound)
+	tr.processHead(ctx, 7, time.Unix(1002, 0), 6)
+	tr.processHead(ctx, 8, time.Unix(1003, 0), 7)
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a node that never caught up was never counted as a hole")
+	require.Zero(t, got.Expired)
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index c0188c3..923c6b6 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -128,7 +128,12 @@ type inclusionState struct {
 	// duplicates counts registrations of a hash already in flight. The registry
 	// holds one slot per hash, so the second one has no place to go and the run
 	// can say nothing about it.
-	duplicates         uint64
+	duplicates uint64
+	// pending holds heights the receipt node had not reached, waiting for one
+	// re-read on the next head. retried remembers which heights have already had
+	// that one, so a node that stays behind produces a hole rather than a loop.
+	pending            []uint64
+	retried            map[uint64]struct{}
 	inflight           map[common.Hash]*entry
 	included           uint64
 	expired            uint64
@@ -315,10 +320,50 @@ func (t *InclusionTracker) processHead(ctx context.Context, num uint64, arrival
 		inclusionBlockGaps.Add(ctx, int64(num-lastSeen-1), metric.WithAttributes(
 			attribute.String("chain_id", t.seiChainID)))
 	}
+	// A height the receipt node had not reached yet is retried on the next head
+	// rather than counted as a hole. That case is ordinary and it is the normal
+	// case once receiptEndpoint names a second node: the head arrives from one
+	// node and the other has not committed that height yet. Counting it would
+	// mark a hole per block and make expired unreachable.
+	for _, pending := range t.takePending() {
+		t.matchBlock(ctx, pending, arrival)
+	}
 	t.matchBlock(ctx, num, arrival)
 	return num
 }
 
+// takePending returns the heights waiting for a re-read and clears them.
+func (t *InclusionTracker) takePending() []uint64 {
+	for s := range t.state.Lock() {
+		if len(s.pending) == 0 {
+			return nil
+		}
+		out := s.pending
+		s.pending = nil
+		return out
+	}
+	panic("unreachable")
+}
+
+// deferHeight queues a height for one re-read on the next head. It reports
+// whether the height was queued: a height already retried once is not, because
+// a node that has not caught up in a block interval is behind rather than busy.
+func (t *InclusionTracker) deferHeight(num uint64) bool {
+	for s := range t.state.Lock() {
+		if s.retried == nil {
+			s.retried = make(map[uint64]struct{})
+		}
+		if _, seen := s.retried[num]; seen {
+			delete(s.retried, num)
+			return false
+		}
+		s.retried[num] = struct{}{}
+		s.pending = append(s.pending, num)
+		return true
+	}
+	panic("unreachable")
+}
+
 // preflight proves the endpoint can answer a receipts read before the tracker
 // starts matching. Without it the run completes, reports every transaction
 // un-included, and exits zero, which reads as a chain that accepted nothing.
@@ -367,6 +412,11 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 	receipts, err := t.source.BlockReceipts(fetchCtx, num)
 	cancel()
 	if err != nil {
+		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(num) {
+			// The node has not reached this height. Ordinary, and not a hole
+			// until a re-read says so.
+			return
+		}
 		// The block goes unmatched, and blindFetches records that this run has
 		// a hole, so a tx in flight across it reaps as status_unavailable
 		// rather than as a verdict about the chain.

From e7a92558e9042097d0eea42fe42ebe02754773e7 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 07:54:03 -0700
Subject: [PATCH 06/17] fix(stats): make the deferred re-read honest about
 time, memory, and shutdown

Round 3 reviewed the re-read that round 2's fix needed, and found four defects in
it. Both lenses reached the first two independently.

The re-read stamped the transaction with the arrival of the head that triggered
it, not the head the block actually arrived on. That arrival becomes the
inclusion latency sample, and in the two-node topology this tracker recommends
every height is deferred at least once, so the error landed on every sample
rather than a few. One block interval against a histogram whose first bucket is
half a second. The pending queue carries its own arrival now.

A height still waiting when the heads stopped was dropped, and its transactions
reaped as expired: a claim about the chain for a block nothing ever read. That is
the defect round 2 existed to remove, through a new path. The queue drains
whatever ends the head loop.

A skipped head was never counted either. sender/doc.go called that an undercount
rather than a miscount, which was true when the tracker only counted inclusions
and stopped being true when expired became a claim about the chain. Every height
in a gap is counted now.

The retry bookkeeping was a map that only deleted on the failure path, so a node
that caught up left an entry per block for the life of the run. One queue
carrying a try count replaces it, bounded in both directions, which also raises
the budget past one block: a node two behind used to produce no inclusion data at
all.

The hardened classifier refused a healthy run three ways. A connection reset is
what a busy node does to a caller and read as unreachable. A rate-limited
response whose request id happened to contain those six digits read as the method
being absent, because the body was searched without regard to the status. A bare
404 from an ingress mid-reconcile read as nothing listening. Refusing a healthy
run is worse than the blind run the refusal exists to prevent, so unreachable now
means a refused dial or a name that does not resolve, the body is read only under
a status that means refusal, and the preflight tries three times before it speaks
for the whole run. An endpoint that answers nothing at all across all three is
also a refusal, which is what a dropped route looks like.

The typed checks the last commit added had no test. Every one could be deleted
with the suite still green, because the tests drove error strings the substring
fallback caught anyway. They are covered now, by construction rather than by
text.

Two more from the same review.

The empty-array premise was wrong a second time, in the other direction.
sei-chain swallows a per-hash receipt lookup that comes back not-found and
compacts the slot out, so an empty array can also mean the block's transactions
existed and their receipts were gone. Restoring the branch is not the answer:
reading an empty array as a hole is what made expired unreachable on 80% of real
blocks. The head's gas comes from the consensus result rather than from any
receipt, so gas burned with no receipt returned is the one witness that the two
cases differ, and it is counted rather than acted on. Gas covers Cosmos
transactions too, so treating it as a hole would invent one on any chain carrying
non-EVM traffic.

Any null element in the array is an error now, not only an array of nothing but
nulls. The run cannot see what that element was going to say either way.

A re-read gets a shorter budget than a first read. Head processing is serial and
the node drops a subscription whose head buffer fills, which ends the run, so two
full-length reads in one head cost more than that affords. The head channel is
buffered for the same reason.

Guards proven by breaking what they cover: the arrival stamp, the queue drain on
shutdown, the skipped head, the duplicate leg on both surfaces, the queue
draining after a read, a reset read as unreachable, and a rate-limit status
allowed to speak for the method.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_outcome_test.go | 210 ++++++++++++++++++++++--
 stats/inclusion_tracker.go      | 283 ++++++++++++++++++++++++--------
 stats/inclusion_tracker_test.go |  23 ++-
 stats/metrics.go                |   7 +-
 4 files changed, 435 insertions(+), 88 deletions(-)

diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index da31f1a..b0fba98 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -3,11 +3,14 @@ package stats
 import (
 	"context"
 	"errors"
+	"net"
 	"strconv"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/sei-protocol/sei-load/types"
 
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
@@ -350,8 +353,9 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 		{"method_absent", errors.New("the method eth_getBlockReceipts does not exist/is not available"), true},
 		{"not_listening", errors.New("dial tcp 10.0.0.1:8545: connect: connection refused"), true},
 		{"no_such_host", errors.New("dial tcp: lookup rpc-0: no such host"), true},
-		{"node_busy", errors.New("context deadline exceeded"), false},
-		{"node_behind", errors.New("not found"), false},
+		{"answers_nothing", context.DeadlineExceeded, true},
+		{"node_behind", ethereum.NotFound, false},
+		{"connection_reset", errors.New("read tcp 10.0.0.1:8545: read: connection reset by peer"), false},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -377,20 +381,30 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 // and loses every result it gathered.
 func TestNilReceiptDoesNotEndTheRun(t *testing.T) {
 	hash := loadTx(1, time.Unix(1000, 0)).EthTx.Hash()
+
+	// Any null element is a loss, not only an array of nothing but nulls. The
+	// run cannot see what that element was going to say, so a transaction it
+	// would have named must not reap as a verdict about the chain.
+	for _, tc := range []struct {
+		name string
+		in   []*ethtypes.Receipt
+	}{
+		{"one_of_two", []*ethtypes.Receipt{nil, {TxHash: hash, Status: 1}}},
+		{"all", []*ethtypes.Receipt{nil, nil}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := narrowReceipts(tc.in)
+			require.ErrorContains(t, err, "null")
+		})
+	}
+
+	// A receipt the endpoint did send survives with its fields.
 	got, err := narrowReceipts([]*ethtypes.Receipt{
-		nil,
 		{TxHash: hash, Status: ethtypes.ReceiptStatusSuccessful},
-		nil,
 	})
 	require.NoError(t, err)
 	require.Equal(t, []blockReceipt{{Hash: hash, Status: 1, HasStatus: true}}, got)
 
-	// Every element null is a node answering nothing, not a block holding
-	// nothing. It has to reach the caller as an error or it reads as an idle
-	// block and the run silently loses those transactions.
-	_, err = narrowReceipts([]*ethtypes.Receipt{nil, nil})
-	require.ErrorContains(t, err, "null")
-
 	// An empty array is a block that carried no EVM transaction. It is the
 	// answer for most blocks on an idle chain, and it is not an error.
 	got, err = narrowReceipts(nil)
@@ -417,7 +431,7 @@ func TestALaggingReceiptNodeIsRetriedNotCountedAsAHole(t *testing.T) {
 
 	// Head 7 arrives before the receipt node holds it.
 	src.SetFetchErr(ethereum.NotFound)
-	last := tr.processHead(ctx, 7, time.Unix(1002, 0), 6)
+	last := tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
 	require.Equal(t, uint64(7), last)
 
 	// By the next head it has caught up, and the re-read finds the transaction.
@@ -425,7 +439,7 @@ func TestALaggingReceiptNodeIsRetriedNotCountedAsAHole(t *testing.T) {
 	src.SetReceipts(7, blockReceipt{
 		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
 	})
-	tr.processHead(ctx, 8, time.Unix(1003, 0), 7)
+	tr.processHead(ctx, 8, 0, time.Unix(1003, 0), 7)
 
 	got := tr.collector.GetOperationStats()[key]
 	require.Equal(t, uint64(1), got.Committed,
@@ -447,8 +461,10 @@ func TestAReceiptNodeThatStaysBehindBecomesAHole(t *testing.T) {
 	tr.Register(ctx, tx)
 
 	src.SetFetchErr(ethereum.NotFound)
-	tr.processHead(ctx, 7, time.Unix(1002, 0), 6)
-	tr.processHead(ctx, 8, time.Unix(1003, 0), 7)
+	// Every head gives the height one more read. Past the budget it is a hole.
+	for h := uint64(7); h <= 7+maxDeferredReads+1; h++ {
+		tr.processHead(ctx, h, 0, time.Unix(1002, 0), h-1)
+	}
 	tr.reap(ctx)
 
 	got := tr.collector.GetOperationStats()[key]
@@ -456,3 +472,169 @@ func TestAReceiptNodeThatStaysBehindBecomesAHole(t *testing.T) {
 		"a node that never caught up was never counted as a hole")
 	require.Zero(t, got.Expired)
 }
+
+// TestFetchFailureReasonReadsTypedErrors fails when the classifier's typed
+// checks are removed, which the substring fallback would otherwise hide.
+//
+// The buckets decide whether a run is refused, so a false refusal on a busy
+// node is worse than the blind run the refusal exists to prevent.
+func TestFetchFailureReasonReadsTypedErrors(t *testing.T) {
+	cases := []struct {
+		name string
+		err  error
+		want string
+	}{
+		{"rpc_method_not_found", rpc.HTTPError{
+			StatusCode: 403, Status: "403 Forbidden",
+			Body: []byte(`{"error":{"code":-32601,"message":"not whitelisted"}}`),
+		}, reasonMethodUnavailable},
+		{"rate_limited_body_holding_the_digits", rpc.HTTPError{
+			StatusCode: 429, Status: "429 Too Many Requests",
+			Body: []byte(`{"error":{"code":-32005},"id":"req-32601-a"}`),
+		}, reasonOther},
+		{"ingress_404_mid_reconcile", rpc.HTTPError{
+			StatusCode: 404, Status: "404 Not Found", Body: []byte("404 page not found"),
+		}, reasonOther},
+		{"bad_gateway", rpc.HTTPError{
+			StatusCode: 502, Status: "502 Bad Gateway",
+		}, reasonOther},
+		{"not_found_sentinel", ethereum.NotFound, reasonNotFound},
+		{"deadline", context.DeadlineExceeded, reasonTimeout},
+		{"dial_refused", &net.OpError{
+			Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED,
+		}, reasonUnreachable},
+		{"name_does_not_resolve", &net.DNSError{
+			Err: "no such host", Name: "rpc-0", IsNotFound: true,
+		}, reasonUnreachable},
+		{"reset_by_a_busy_node", &net.OpError{
+			Op: "read", Net: "tcp", Err: syscall.ECONNRESET,
+		}, reasonOther},
+		{"pruned_before_availability", errors.New(
+			"receipts have been pruned; earliest available is 100"), reasonPruned},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			require.Equal(t, tc.want, fetchFailureReason(tc.err))
+		})
+	}
+}
+
+// TestADeferredHeightKeepsItsOwnArrival fails when a re-read stamps the
+// transaction with the arrival of the head that triggered it.
+//
+// Arrival becomes the inclusion latency sample. In the two-node topology this
+// tracker recommends, every height is deferred at least once, so the error
+// would land on every sample rather than a few.
+func TestADeferredHeightKeepsItsOwnArrival(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.IntendedSendTime = time.Unix(1000, 0)
+	tr.Register(ctx, tx)
+
+	blockSeven := time.Unix(1002, 0)
+	blockEight := blockSeven.Add(400 * time.Millisecond)
+
+	src.SetFetchErr(ethereum.NotFound)
+	tr.processHead(ctx, 7, 0, blockSeven, 6)
+
+	src.SetFetchErr(nil)
+	src.SetReceipts(7, blockReceipt{
+		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
+	})
+	tr.processHead(ctx, 8, 0, blockEight, 7)
+
+	require.Equal(t, blockSeven, tx.InclusionTime,
+		"the transaction was stamped with a later block's arrival, so every "+
+			"latency sample in this topology is inflated by one block")
+}
+
+// TestAPendingHeightIsCountedWhenTheHeadsStop fails when a height still waiting
+// to be read is dropped instead of counted. Nothing else will read it, and
+// leaving it lets a transaction the chain may well have included reap as a
+// verdict about the chain.
+func TestAPendingHeightIsCountedWhenTheHeadsStop(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	src.SetFetchErr(ethereum.NotFound)
+	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
+	// The head stream ends here, as it does at shutdown or on a stalled chain.
+	tr.flushDeferred(ctx)
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a height nothing ever read was reported as a chain result")
+	require.Zero(t, got.Expired)
+}
+
+// TestASkippedHeadIsCountedAsAHole fails when a height the run never saw a head
+// for goes uncounted. Its transactions would reap as expired, which is a claim
+// about the chain for a block nothing read.
+func TestASkippedHeadIsCountedAsAHole(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	last := tr.processHead(ctx, 10, 0, time.Unix(1002, 0), 0)
+	tr.processHead(ctx, 13, 0, time.Unix(1003, 0), last) // 11 and 12 missed
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a transaction that may have been in a skipped block was blamed on the chain")
+	require.Zero(t, got.Expired)
+}
+
+// TestTheDuplicateLegReachesBothLedgers fails when a duplicate registration is
+// counted on one operator-facing surface and not the other.
+func TestTheDuplicateLegReachesBothLedgers(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+	tr.Register(ctx, tx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable)
+	require.Equal(t, got.StatusUnavailable, tr.Summary().StatusUnavailable,
+		"the closing log line and the metric disagree about the duplicate")
+}
+
+// TestTheDeferredQueueDrains fails when a height stays queued after it is read,
+// which would make every later head re-read every height the run ever deferred.
+func TestTheDeferredQueueDrains(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	src.FailTimes(1, ethereum.NotFound)
+	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
+	tr.processHead(ctx, 8, 0, time.Unix(1003, 0), 7)
+
+	for s := range tr.state.Lock() {
+		require.Empty(t, s.pending, "a height stayed queued after it was read")
+	}
+	before := src.FetchCount()
+	tr.processHead(ctx, 9, 0, time.Unix(1004, 0), 8)
+	require.Equal(t, int64(1), src.FetchCount()-before,
+		"a later head re-read a height that was already done")
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 923c6b6..011bb34 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
@@ -81,26 +82,57 @@ func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockR
 //
 // An endpoint decides what it puts in this array. A nil element would panic the
 // head loop, which ends the run and loses every result it gathered, so drop it
-// rather than trust the shape. An array that held only nil elements is a node
-// answering nothing at all, which is an error rather than an empty block.
+// rather than trust the shape.
+//
+// Any nil element is an error, not just an array of nothing but nils. The run
+// cannot see what that element was going to say, and a transaction it would have
+// named reaps as a verdict about the chain instead. That is the same loss the
+// all-nil case was guarded against, in a smaller share.
 func narrowReceipts(receipts []*ethtypes.Receipt) ([]blockReceipt, error) {
 	out := make([]blockReceipt, 0, len(receipts))
+	var nulls int
 	for _, r := range receipts {
 		if r == nil {
+			nulls++
 			continue
 		}
 		out = append(out, blockReceipt{
-			Hash:      r.TxHash,
-			Status:    r.Status,
-			HasStatus: len(r.PostState) == 0,
+			Hash:   r.TxHash,
+			Status: r.Status,
+			// A root alone means no status. A root beside a status means the
+			// status is there and is the more specific of the two, so it wins:
+			// reading it as unavailable would report a whole run as unreadable
+			// against any endpoint that sends both.
+			HasStatus: len(r.PostState) == 0 || r.Status != 0,
 		})
 	}
-	if len(receipts) > 0 && len(out) == 0 {
-		return nil, fmt.Errorf("every one of %d receipts was null", len(receipts))
+	if nulls > 0 {
+		return nil, fmt.Errorf("%d of %d receipts were null", nulls, len(receipts))
 	}
 	return out, nil
 }
 
+// deferredRead is a height the receipt node had not reached, waiting to be read
+// again.
+//
+// It carries the arrival of its own head. The re-read happens on a later head,
+// and arrival becomes the inclusion latency sample, so passing the later head's
+// arrival would add one head-to-head interval to every sample. In the two-node
+// topology this tracker recommends, every height is deferred at least once, so
+// that error would land on every sample rather than a few.
+type deferredRead struct {
+	num     uint64
+	gasUsed uint64
+	arrival time.Time
+	tries   int
+}
+
+// maxDeferredReads bounds both how many heights wait to be read again and how
+// many times each is tried. A node one block behind needs one try. A node
+// further behind needs the depth, and past this bound it is behind rather than
+// busy, which is a hole worth counting.
+const maxDeferredReads = 4
+
 type entry struct {
 	tx           *types.LoadTx
 	registeredAt time.Time
@@ -129,11 +161,11 @@ type inclusionState struct {
 	// holds one slot per hash, so the second one has no place to go and the run
 	// can say nothing about it.
 	duplicates uint64
-	// pending holds heights the receipt node had not reached, waiting for one
-	// re-read on the next head. retried remembers which heights have already had
-	// that one, so a node that stays behind produces a hole rather than a loop.
-	pending            []uint64
-	retried            map[uint64]struct{}
+	// pending holds heights the receipt node had not reached, waiting to be read
+	// again. It is bounded by maxDeferredReads entries, and each entry is
+	// dropped once it is read or once it runs out of tries, so a node that stays
+	// behind produces holes rather than a growing queue.
+	pending            []deferredRead
 	inflight           map[common.Hash]*entry
 	included           uint64
 	expired            uint64
@@ -262,9 +294,10 @@ func (t *InclusionTracker) meterOutcome(ctx context.Context, outcome Outcome, sc
 // receipts from receiptEndpoint. Pass the same string for both to run against
 // one node.
 //
-// The tracker only ever reads the height it just received as a head. It never
-// backfills, so the serving node's receipt retention does not bound it. A change
-// that reaches further back does.
+// The tracker reads the height it just received as a head, and re-reads a height
+// the serving node had not reached yet. It reaches back no further than
+// maxDeferredReads heights, so the serving node's receipt retention does not
+// bound it. A change that reaches further back does.
 func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoint string) error {
 	wsEndpoint := utils.GetWSEndpoint(headEndpoint)
 	if t.source == nil {
@@ -283,7 +316,9 @@ func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoin
 		if err != nil {
 			return fmt.Errorf("inclusion tracker: connect WebSocket %s: %w", wsEndpoint, err)
 		}
-		headers := make(chan *ethtypes.Header)
+		// Buffered: head processing is serial and a slow read must not cost a
+		// head. The node drops a subscription whose buffer fills.
+		headers := make(chan *ethtypes.Header, 32)
 		sub, err := client.SubscribeNewHead(ctx, headers)
 		if err != nil {
 			return fmt.Errorf("inclusion tracker: subscribe new heads: %w", err)
@@ -299,12 +334,17 @@ func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoin
 		s.Spawn(func() error { return t.reapLoop(ctx) })
 
 		var lastSeen uint64 // 0 = unset; first head seeds it (no backfill).
+		// Whatever ends the head loop, a height still waiting to be read is
+		// counted before the run reports. Nothing else will read it, and
+		// leaving it lets a tx the chain may well have included reap as a
+		// verdict about the chain.
+		defer t.flushDeferred(ctx)
 		for ctx.Err() == nil {
 			header, err := utils.Recv(ctx, headers)
 			if err != nil {
 				return err
 			}
-			lastSeen = t.processHead(ctx, header.Number.Uint64(), time.Now(), lastSeen)
+			lastSeen = t.processHead(ctx, header.Number.Uint64(), header.GasUsed, time.Now(), lastSeen)
 		}
 		return ctx.Err()
 	})
@@ -312,53 +352,69 @@ func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoin
 
 // processHead handles one arriving head: counts any gap (no backfill), matches
 // the block, and returns the new lastSeen. lastSeen==0 seeds on the first head.
-func (t *InclusionTracker) processHead(ctx context.Context, num uint64, arrival time.Time, lastSeen uint64) uint64 {
+func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64, arrival time.Time, lastSeen uint64) uint64 {
+	// The re-read runs before the early return. A repeated or out-of-order head
+	// is no reason to leave a height waiting: nothing else drains the queue, and
+	// a height left in it reaps as a chain verdict for a block never read.
+	t.rereadDeferred(ctx)
 	if lastSeen != 0 && num <= lastSeen {
 		return lastSeen // duplicate or out-of-order head: no re-fetch, no spurious gap.
 	}
 	if lastSeen != 0 && num > lastSeen+1 {
-		inclusionBlockGaps.Add(ctx, int64(num-lastSeen-1), metric.WithAttributes(
+		inclusionBlockGaps.Add(ctx, int64(num-lastSeen-1), metric.WithAttributes( //nolint:gosec
 			attribute.String("chain_id", t.seiChainID)))
+		// A height nothing read is a hole, however it went missing. Leaving the
+		// gap uncounted let a tx in one of those blocks reap as expired, which
+		// is a claim about the chain the run has no grounds for.
+		for missed := lastSeen + 1; missed < num; missed++ {
+			t.recordBlindFetch(ctx, missed, reasonNotFound, nil)
+		}
 	}
-	// A height the receipt node had not reached yet is retried on the next head
-	// rather than counted as a hole. That case is ordinary and it is the normal
-	// case once receiptEndpoint names a second node: the head arrives from one
-	// node and the other has not committed that height yet. Counting it would
-	// mark a hole per block and make expired unreachable.
-	for _, pending := range t.takePending() {
-		t.matchBlock(ctx, pending, arrival)
-	}
-	t.matchBlock(ctx, num, arrival)
+	t.matchBlockAttempt(ctx, num, gasUsed, arrival, 0)
 	return num
 }
 
-// takePending returns the heights waiting for a re-read and clears them.
-func (t *InclusionTracker) takePending() []uint64 {
+// rereadDeferred reads every height the receipt node had not reached, using that
+// height's own arrival. A height out of tries becomes a hole here, because
+// nothing downstream would notice it was never read.
+func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
+	var due []deferredRead
 	for s := range t.state.Lock() {
-		if len(s.pending) == 0 {
-			return nil
+		due, s.pending = s.pending, nil
+	}
+	for _, d := range due {
+		if d.tries >= maxDeferredReads {
+			t.recordBlindFetch(ctx, d.num, reasonNotFound, nil)
+			continue
 		}
-		out := s.pending
-		s.pending = nil
-		return out
+		t.matchBlockAttempt(ctx, d.num, d.gasUsed, d.arrival, d.tries)
 	}
-	panic("unreachable")
 }
 
-// deferHeight queues a height for one re-read on the next head. It reports
-// whether the height was queued: a height already retried once is not, because
-// a node that has not caught up in a block interval is behind rather than busy.
-func (t *InclusionTracker) deferHeight(num uint64) bool {
+// flushDeferred counts every height still waiting as a hole. The head stream has
+// stopped, so nothing will read them, and leaving them would let a transaction
+// the chain may well have included reap as a verdict about the chain.
+func (t *InclusionTracker) flushDeferred(ctx context.Context) {
+	var due []deferredRead
 	for s := range t.state.Lock() {
-		if s.retried == nil {
-			s.retried = make(map[uint64]struct{})
-		}
-		if _, seen := s.retried[num]; seen {
-			delete(s.retried, num)
+		due, s.pending = s.pending, nil
+	}
+	for _, d := range due {
+		t.recordBlindFetch(ctx, d.num, reasonNotFound, nil)
+	}
+}
+
+// deferHeight queues a height to be read again, and reports whether it took it.
+// It refuses once the queue is full, so a node far behind produces holes instead
+// of a queue that grows with the run.
+func (t *InclusionTracker) deferHeight(num, gasUsed uint64, arrival time.Time, tries int) bool {
+	for s := range t.state.Lock() {
+		if len(s.pending) >= maxDeferredReads {
 			return false
 		}
-		s.retried[num] = struct{}{}
-		s.pending = append(s.pending, num)
+		s.pending = append(s.pending, deferredRead{
+			num: num, gasUsed: gasUsed, arrival: arrival, tries: tries + 1,
+		})
 		return true
 	}
 	panic("unreachable")
@@ -380,13 +436,30 @@ func (t *InclusionTracker) deferHeight(num uint64) bool {
 // A node that answers and refuses the method is the other. Anything else is a
 // node busy or behind, which the run reports as it finds.
 func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error {
-	probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-	_, err := t.source.BlockReceipts(probeCtx, 0)
-	if err == nil {
-		return nil
+	// Several attempts, because one verdict stands for the whole run. A single
+	// reset or timeout from a node that is merely busy must not end a run before
+	// it starts.
+	var err error
+	var reason string
+	for attempt := range preflightAttempts {
+		if attempt > 0 {
+			if _, waitErr := utils.Recv(ctx, time.After(preflightBackoff)); waitErr != nil {
+				return nil
+			}
+		}
+		probeCtx, cancel := context.WithTimeout(ctx, preflightTimeout)
+		_, err = t.source.BlockReceipts(probeCtx, 0)
+		cancel()
+		if err == nil {
+			return nil
+		}
+		reason = fetchFailureReason(err)
+		if reason == reasonOther {
+			// Busy, reset, or something this run cannot name. Report and go.
+			break
+		}
 	}
-	switch fetchFailureReason(err) {
+	switch reason {
 	case reasonMethodUnavailable:
 		return fmt.Errorf(
 			"inclusion tracker: %s answers, but not eth_getBlockReceipts (%w). "+
@@ -399,6 +472,15 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 				"Set receiptEndpoint in the profile to a node in fullNode or "+
 				"archive mode; validator and seed modes serve no EVM HTTP",
 			endpoint, err)
+	case reasonTimeout:
+		// Nothing answered, every attempt. A dropped route and a firewall both
+		// look like this, and a node that cannot serve one genesis read in this
+		// many tries cannot serve a run.
+		return fmt.Errorf(
+			"inclusion tracker: %s did not answer %d receipt reads (%w). "+
+				"Check that receiptEndpoint names a reachable node and that "+
+				"nothing between here and it is dropping the connection",
+			endpoint, preflightAttempts, err)
 	}
 	log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
 	return nil
@@ -407,12 +489,28 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 // matchBlock fetches block num once and stamps every in-flight tx it includes
 // with the header-arrival time.
 func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival time.Time) {
+	t.matchBlockAttempt(ctx, num, 0, arrival, 0)
+}
+
+// matchBlockAttempt is matchBlock, carrying how many times this height has
+// already been read so a re-read does not restart the budget.
+func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed uint64, arrival time.Time, tries int) {
 	// Explicit per-iteration cancel (not deferred-in-loop): bound the fetch.
-	fetchCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	//
+	// A re-read gets a shorter budget than a first read. Head processing is
+	// serial and the node buffers a bounded number of heads before it drops the
+	// subscription, which ends the whole run, so two full-length reads in one
+	// head is more than that budget affords. A re-read is best-effort by
+	// construction: it becomes a hole rather than blocking on a node that hangs.
+	budget := firstReadTimeout
+	if tries > 0 {
+		budget = rereadTimeout
+	}
+	fetchCtx, cancel := context.WithTimeout(ctx, budget)
 	receipts, err := t.source.BlockReceipts(fetchCtx, num)
 	cancel()
 	if err != nil {
-		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(num) {
+		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(num, gasUsed, arrival, tries) {
 			// The node has not reached this height. Ordinary, and not a hole
 			// until a re-read says so.
 			return
@@ -430,6 +528,22 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 		t.recordBlindFetch(ctx, num, fetchFailureReason(err), err)
 		return
 	}
+	if len(receipts) == 0 && gasUsed > 0 {
+		// An empty array usually means the block carried no EVM transaction,
+		// and it is the answer for most blocks on an idle chain. It has one
+		// other cause: sei-chain drops a receipt its store cannot find and
+		// compacts it out of the array, so a block whose receipts went missing
+		// answers the same way.
+		//
+		// The head's gas comes from the consensus result rather than from any
+		// receipt, so gas burned with no receipt returned is the one witness
+		// that the two cases differ. It is counted and not acted on. Gas covers
+		// Cosmos transactions too, so treating it as a hole would invent one on
+		// any chain carrying non-EVM traffic, and over-reporting holes is what
+		// made expired unreachable once already.
+		inclusionEmptyWithGas.Add(ctx, 1, metric.WithAttributes(
+			attribute.String("chain_id", t.seiChainID)))
+	}
 	matched := make([]inclusionSample, 0, len(receipts))
 	resolved := make([]resolvedOutcome, 0, len(receipts))
 	for s := range t.state.Lock() {
@@ -448,6 +562,9 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival t
 			outcome := OutcomeStatusUnavailable
 			switch {
 			case !r.HasStatus:
+				// It was in a block, so it is included. What it did there is
+				// unreadable, so the registry counts it where the ledger does.
+				s.statusUnavailable++
 				// A receipt with a post-state root instead of a status says the
 				// tx executed and does not say how it ended. Reading that as a
 				// failure would invent a chain result.
@@ -600,6 +717,21 @@ const (
 	reasonOther             = "other"
 )
 
+// Read budgets. Head processing is serial, and the node drops a subscription
+// whose head buffer fills, which ends the run, so the total spent in one head
+// matters more than any single read.
+const (
+	firstReadTimeout = 10 * time.Second
+	rereadTimeout    = 2 * time.Second
+)
+
+// How hard the preflight tries before it speaks for the whole run.
+const (
+	preflightAttempts = 3
+	preflightTimeout  = 5 * time.Second
+	preflightBackoff  = 500 * time.Millisecond
+)
+
 // methodNotFoundCode is JSON-RPC 2.0's "Method not found". Every server returns
 // it under that code whatever prose it puts beside it, so the code is what this
 // matches on.
@@ -620,16 +752,21 @@ func fetchFailureReason(err error) string {
 		return reasonMethodUnavailable
 	}
 	// A gateway that filters methods answers with an HTTP status, and the
-	// JSON-RPC code sits in the body where the decoder never looks. Reading the
-	// body is what separates "this node will never serve the method" from "this
-	// node is rate-limiting me", which are different operator actions.
+	// JSON-RPC code sits in the body where the decoder never looks.
+	//
+	// The status gates the body read. A rate-limited response carries a body
+	// too, and a request id inside it can hold these six digits by coincidence,
+	// which would refuse a run against a node that is merely busy. Only a
+	// status that means "I will not serve this" is allowed to speak for the
+	// method.
 	var httpErr rpc.HTTPError
 	if errors.As(err, &httpErr) {
-		if bytes.Contains(httpErr.Body, []byte(strconv.Itoa(methodNotFoundCode))) {
-			return reasonMethodUnavailable
-		}
-		if httpErr.StatusCode == http.StatusNotFound {
-			return reasonUnreachable
+		switch httpErr.StatusCode {
+		case http.StatusOK, http.StatusBadRequest,
+			http.StatusForbidden, http.StatusMethodNotAllowed:
+			if bytes.Contains(httpErr.Body, []byte(strconv.Itoa(methodNotFoundCode))) {
+				return reasonMethodUnavailable
+			}
 		}
 		return reasonOther
 	}
@@ -643,11 +780,20 @@ func fetchFailureReason(err error) string {
 	if errors.As(err, &netErr) && netErr.Timeout() {
 		return reasonTimeout
 	}
-	// A node serving no EVM HTTP refuses the connection or resolves to nothing,
-	// and a URL pointing somewhere else answers with prose rather than JSON.
-	var opErr *net.OpError
+	// Unreachable means the endpoint is not there, and it is the only network
+	// bucket allowed to refuse a run. A refused dial and a name that does not
+	// resolve stay broken for the run's length.
+	//
+	// A connection reset does not. It is what a healthy node under load does to
+	// a caller, and reading it as unreachable would kill a run for being busy,
+	// which is worse than the blind run this bucket exists to prevent. It falls
+	// through to other.
 	var dnsErr *net.DNSError
-	if errors.As(err, &opErr) || errors.As(err, &dnsErr) {
+	if errors.As(err, &dnsErr) {
+		return reasonUnreachable
+	}
+	var opErr *net.OpError
+	if errors.As(err, &opErr) && opErr.Op == "dial" && errors.Is(err, syscall.ECONNREFUSED) {
 		return reasonUnreachable
 	}
 	switch msg := strings.ToLower(err.Error()); {
@@ -656,8 +802,7 @@ func fetchFailureReason(err error) string {
 	case strings.Contains(msg, "does not exist") || strings.Contains(msg, "not available"):
 		return reasonMethodUnavailable
 	case strings.Contains(msg, "connection refused") ||
-		strings.Contains(msg, "no such host") ||
-		strings.Contains(msg, "looking for beginning of value"):
+		strings.Contains(msg, "no such host"):
 		return reasonUnreachable
 	case strings.Contains(msg, "not found"):
 		return reasonNotFound
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index 660dfa6..ac1643e 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -25,6 +25,7 @@ type MockBlockSource struct {
 	blocks   map[uint64][]blockReceipt
 	fetches  atomic.Int64
 	fetchErr error
+	failures int
 }
 
 func NewMockBlockSource() *MockBlockSource {
@@ -52,6 +53,17 @@ func (m *MockBlockSource) SetFetchErr(err error) *MockBlockSource {
 	m.mu.Lock()
 	defer m.mu.Unlock()
 	m.fetchErr = err
+	m.failures = -1 // every fetch
+	return m
+}
+
+// FailTimes makes the next n fetches fail and the rest succeed, so a test can
+// drive a node that recovers rather than one that stays broken.
+func (m *MockBlockSource) FailTimes(n int, err error) *MockBlockSource {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.fetchErr = err
+	m.failures = n
 	return m
 }
 
@@ -59,7 +71,10 @@ func (m *MockBlockSource) BlockReceipts(_ context.Context, n uint64) ([]blockRec
 	m.fetches.Add(1)
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	if m.fetchErr != nil {
+	if m.fetchErr != nil && m.failures != 0 {
+		if m.failures > 0 {
+			m.failures--
+		}
 		return nil, m.fetchErr
 	}
 	return m.blocks[n], nil
@@ -305,9 +320,9 @@ func TestInclusion_GapNoBackfill(t *testing.T) {
 	tr := newTestTracker(t, time.Minute, 100, src)
 	ctx := context.Background()
 
-	last := tr.processHead(ctx, 10, time.Now(), 0) // seeds, no gap
-	last = tr.processHead(ctx, 11, time.Now(), last)
-	last = tr.processHead(ctx, 15, time.Now(), last) // gap 12,13,14
+	last := tr.processHead(ctx, 10, 0, time.Now(), 0) // seeds, no gap
+	last = tr.processHead(ctx, 11, 0, time.Now(), last)
+	last = tr.processHead(ctx, 15, 0, time.Now(), last) // gap 12,13,14
 	_ = last
 
 	require.Equal(t, int64(3), src.FetchCount(),
diff --git a/stats/metrics.go b/stats/metrics.go
index eb6ef34..2939589 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -76,9 +76,14 @@ var (
 		metric.WithDescription("Missed block heights observed by the inclusion tracker (no backfill)"),
 		metric.WithUnit("{blocks}")))
 
+	inclusionEmptyWithGas = must(meter.Int64Counter(
+		"block_empty_with_gas",
+		metric.WithDescription("Blocks that returned no receipts while the head reported gas burned. Usually non-EVM traffic; a sustained rate means receipts are going missing"),
+		metric.WithUnit("{blocks}")))
+
 	inclusionBlockFetchErrors = must(meter.Int64Counter(
 		"block_fetch_errors",
-		metric.WithDescription("Receipt reads that failed, by reason; that block goes unmatched and no retry follows, so a tx in flight across it reaches status_unavailable"),
+		metric.WithDescription("Receipt reads that failed, by reason, after the re-reads a height gets; that block goes unmatched, so a tx in flight across it reaches status_unavailable"),
 		metric.WithUnit("{blocks}")))
 
 	// Run-summary: the only inclusion tally with no live series, since it is the

From 6be231d958ac9526239261efa41f03e59ddeb98f Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 08:01:31 -0700
Subject: [PATCH 07/17] fix(stats): a partly unreadable read keeps what arrived

Found by reading my own round-3 fix rather than by review. Treating any null
element as a failed read threw away the receipts the endpoint did send, so a
transaction that committed and whose receipt arrived got nothing and later reaped
as unattributable. That trades a known outcome for an unknown one.

narrowReceipts reports how many elements were null instead of refusing the
array, and receiptSource says so in its signature, because a read that partly
succeeded is a real answer and the interface should be able to express it. The
block is a hole for the transactions the missing part would have named and not
for the ones it named.

Guards proven by breaking what they cover, in both directions: discarding what
arrived, and hiding the loss.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_outcome_test.go | 69 ++++++++++++++++++++++-----------
 stats/inclusion_tracker.go      | 41 ++++++++++++--------
 stats/inclusion_tracker_test.go | 16 ++++++--
 3 files changed, 83 insertions(+), 43 deletions(-)

diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index b0fba98..46f1883 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -379,39 +379,62 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 // TestNilReceiptDoesNotEndTheRun fails when an endpoint that returns a null
 // array element panics the head loop. Nothing recovers there, so the run dies
 // and loses every result it gathered.
+//
+// A null element is reported rather than refused. The run cannot see what that
+// element was going to say, so the block is a hole for the transactions it did
+// not name. It is not a hole for the ones it did: discarding a receipt the
+// endpoint actually sent would throw away a known outcome to describe an
+// unknown one.
 func TestNilReceiptDoesNotEndTheRun(t *testing.T) {
 	hash := loadTx(1, time.Unix(1000, 0)).EthTx.Hash()
 
-	// Any null element is a loss, not only an array of nothing but nulls. The
-	// run cannot see what that element was going to say, so a transaction it
-	// would have named must not reap as a verdict about the chain.
-	for _, tc := range []struct {
-		name string
-		in   []*ethtypes.Receipt
-	}{
-		{"one_of_two", []*ethtypes.Receipt{nil, {TxHash: hash, Status: 1}}},
-		{"all", []*ethtypes.Receipt{nil, nil}},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			_, err := narrowReceipts(tc.in)
-			require.ErrorContains(t, err, "null")
-		})
-	}
-
-	// A receipt the endpoint did send survives with its fields.
-	got, err := narrowReceipts([]*ethtypes.Receipt{
+	got, nulls := narrowReceipts([]*ethtypes.Receipt{
+		nil,
 		{TxHash: hash, Status: ethtypes.ReceiptStatusSuccessful},
+		nil,
 	})
-	require.NoError(t, err)
-	require.Equal(t, []blockReceipt{{Hash: hash, Status: 1, HasStatus: true}}, got)
+	require.Equal(t, 2, nulls, "the nulls were not reported")
+	require.Equal(t, []blockReceipt{{Hash: hash, Status: 1, HasStatus: true}}, got,
+		"a receipt the endpoint did send was discarded")
 
 	// An empty array is a block that carried no EVM transaction. It is the
-	// answer for most blocks on an idle chain, and it is not an error.
-	got, err = narrowReceipts(nil)
-	require.NoError(t, err)
+	// answer for most blocks on an idle chain, and it is not a loss.
+	got, nulls = narrowReceipts(nil)
+	require.Zero(t, nulls)
 	require.Empty(t, got)
 }
 
+// TestAPartlyNullReadMatchesWhatArrivedAndCountsTheRest fails when a read that
+// was partly unreadable either throws away the part that arrived or hides the
+// part that did not.
+func TestAPartlyNullReadMatchesWhatArrivedAndCountsTheRest(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	named := loadTx(1, time.Unix(1000, 0))
+	unnamed := loadTx(2, time.Unix(1000, 0))
+	for _, tx := range []*types.LoadTx{named, unnamed} {
+		tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+		tr.Register(ctx, tx)
+	}
+
+	// The endpoint sent one receipt and one null.
+	src.SetNulls(1).SetReceipts(4, blockReceipt{
+		Hash: named.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
+	})
+	tr.matchBlock(ctx, 4, time.Unix(1002, 0))
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Committed,
+		"a receipt that arrived was discarded because another was null")
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"the transaction the null would have named was blamed on the chain")
+	require.Zero(t, got.Expired)
+}
+
 // TestALaggingReceiptNodeIsRetriedNotCountedAsAHole fails when a height the
 // receipt node has not reached becomes a hole on first sight.
 //
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 011bb34..8e81a29 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -61,21 +61,25 @@ type blockReceipt struct {
 // the tracker at a node that takes no send load, and see
 // config.ReceiptEndpoint.
 type receiptSource interface {
-	BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error)
+	// BlockReceipts returns the block's receipts, and how many the endpoint
+	// sent as null. A null carries no hash, so those transactions are a hole
+	// even though the read succeeded.
+	BlockReceipts(ctx context.Context, n uint64) (receipts []blockReceipt, nulls int, err error)
 }
 
 // ethReceiptSource is the production receiptSource backed by an ethclient.
 type ethReceiptSource struct{ client *ethclient.Client }
 
-func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, error) {
+func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, int, error) {
 	//nolint:gosec // A block height never approaches MaxInt64, where the
 	// conversion would land on rpc.BlockNumber's negative sentinels.
 	number := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(n))
 	receipts, err := s.client.BlockReceipts(ctx, number)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
-	return narrowReceipts(receipts)
+	out, nulls := narrowReceipts(receipts)
+	return out, nulls, nil
 }
 
 // narrowReceipts keeps what the tracker reads and drops the rest.
@@ -84,13 +88,13 @@ func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockR
 // head loop, which ends the run and loses every result it gathered, so drop it
 // rather than trust the shape.
 //
-// Any nil element is an error, not just an array of nothing but nils. The run
-// cannot see what that element was going to say, and a transaction it would have
-// named reaps as a verdict about the chain instead. That is the same loss the
-// all-nil case was guarded against, in a smaller share.
-func narrowReceipts(receipts []*ethtypes.Receipt) ([]blockReceipt, error) {
-	out := make([]blockReceipt, 0, len(receipts))
-	var nulls int
+// It reports how many elements were nil rather than refusing the whole array.
+// The run cannot see what a nil element was going to say, so the block is a hole
+// for the transactions it did not name. It is not a hole for the ones it did:
+// discarding a receipt the endpoint actually sent would throw away a known
+// outcome to describe an unknown one.
+func narrowReceipts(receipts []*ethtypes.Receipt) (out []blockReceipt, nulls int) {
+	out = make([]blockReceipt, 0, len(receipts))
 	for _, r := range receipts {
 		if r == nil {
 			nulls++
@@ -106,10 +110,7 @@ func narrowReceipts(receipts []*ethtypes.Receipt) ([]blockReceipt, error) {
 			HasStatus: len(r.PostState) == 0 || r.Status != 0,
 		})
 	}
-	if nulls > 0 {
-		return nil, fmt.Errorf("%d of %d receipts were null", nulls, len(receipts))
-	}
-	return out, nil
+	return out, nulls
 }
 
 // deferredRead is a height the receipt node had not reached, waiting to be read
@@ -448,7 +449,7 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 			}
 		}
 		probeCtx, cancel := context.WithTimeout(ctx, preflightTimeout)
-		_, err = t.source.BlockReceipts(probeCtx, 0)
+		_, _, err = t.source.BlockReceipts(probeCtx, 0)
 		cancel()
 		if err == nil {
 			return nil
@@ -507,7 +508,7 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 		budget = rereadTimeout
 	}
 	fetchCtx, cancel := context.WithTimeout(ctx, budget)
-	receipts, err := t.source.BlockReceipts(fetchCtx, num)
+	receipts, nulls, err := t.source.BlockReceipts(fetchCtx, num)
 	cancel()
 	if err != nil {
 		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(num, gasUsed, arrival, tries) {
@@ -528,6 +529,12 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 		t.recordBlindFetch(ctx, num, fetchFailureReason(err), err)
 		return
 	}
+	if nulls > 0 {
+		// The read succeeded and part of it is unreadable. Match what arrived,
+		// and record the hole so a transaction the missing part would have named
+		// is not blamed on the chain.
+		t.recordBlindFetch(ctx, num, reasonNotFound, nil)
+	}
 	if len(receipts) == 0 && gasUsed > 0 {
 		// An empty array usually means the block carried no EVM transaction,
 		// and it is the answer for most blocks on an idle chain. It has one
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index ac1643e..0ee7047 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -26,6 +26,7 @@ type MockBlockSource struct {
 	fetches  atomic.Int64
 	fetchErr error
 	failures int
+	nulls    int
 }
 
 func NewMockBlockSource() *MockBlockSource {
@@ -57,6 +58,15 @@ func (m *MockBlockSource) SetFetchErr(err error) *MockBlockSource {
 	return m
 }
 
+// SetNulls makes every later read report n receipts the endpoint sent as null,
+// so a test can drive a read that partly succeeded.
+func (m *MockBlockSource) SetNulls(n int) *MockBlockSource {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.nulls = n
+	return m
+}
+
 // FailTimes makes the next n fetches fail and the rest succeed, so a test can
 // drive a node that recovers rather than one that stays broken.
 func (m *MockBlockSource) FailTimes(n int, err error) *MockBlockSource {
@@ -67,7 +77,7 @@ func (m *MockBlockSource) FailTimes(n int, err error) *MockBlockSource {
 	return m
 }
 
-func (m *MockBlockSource) BlockReceipts(_ context.Context, n uint64) ([]blockReceipt, error) {
+func (m *MockBlockSource) BlockReceipts(_ context.Context, n uint64) ([]blockReceipt, int, error) {
 	m.fetches.Add(1)
 	m.mu.Lock()
 	defer m.mu.Unlock()
@@ -75,9 +85,9 @@ func (m *MockBlockSource) BlockReceipts(_ context.Context, n uint64) ([]blockRec
 		if m.failures > 0 {
 			m.failures--
 		}
-		return nil, m.fetchErr
+		return nil, 0, m.fetchErr
 	}
-	return m.blocks[n], nil
+	return m.blocks[n], m.nulls, nil
 }
 
 func (m *MockBlockSource) FetchCount() int64 { return m.fetches.Load() }

From 70f4a1a183125729bc8b532631a7432bb449d7a4 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 08:39:41 -0700
Subject: [PATCH 08/17] fix(stats): a refused run must not exit zero

The worst defect in this series, and it was mine from the last commit. The
preflight's refusal wrapped the probe's own error with %w, and main treats a
context.Canceled or a context.DeadlineExceeded as the run ending normally. So a
run refused for an endpoint that never answered exited zero: a Complete run that
carried no load, which the nightly harness then reports as a chain that accepted
work and included none of it. The exact false accusation this whole tracker
exists to remove, reintroduced through the path meant to prevent it.

Refusal now carries its own sentinel, and a test asserts that no refusal reads as
the run's context ending.

The rest of what round 4 found, in the order it matters.

The two read budgets were backwards. In the two-node topology this tracker
recommends, the first read of a height returns not-found cheaply and the re-read
is the one that carries the receipts, so the short budget landed on the only read
that mattered. Measured against pacific-1, two of five idle-block reads already
exceeded three seconds. One budget for both.

The deferral bound counted heads, which made the failure a cliff: at four heads
of lag every block read, at five every block became a hole and expired went
unreachable for the whole run. It is a duration now, which is the quantity that
actually matters, and the wait is recorded as a histogram so the drift is visible
before it is crossed rather than after.

The preflight kept only the last attempt's verdict, so two timeouts could erase
an earlier answer that proved the endpoint was there, and refuse the run on
evidence contradicting its own message. A refusal now needs every attempt to
agree on one cause.

Any DNS error refused the run. A resolver answering SERVFAIL is temporary, and
only a name that does not exist is permanent.

An endpoint answering prose rather than JSON stopped being refused when I
narrowed the classifier last round. That is the common operator typo: the metrics
port, the Cosmos RPC port, an ingress default backend. It is as durable a failure
as a refused dial, and it refuses again.

The summary's terms overlapped. A receipt whose status could not be read counted
in both included and status_unavailable, so adding up the closing log line gave
more than the run accepted. included now means the readable ones, and the terms
are disjoint.

A gap logged one line per height, so a fifty-height gap pushed the run's own
summary out of the fifty-line log tail that is the only diagnostic a failed
nightly carries. One record per gap, which changes nothing about attribution
because the reap only asks whether the count rose.

The shutdown sweep marked holes it could not justify. The head loop and the reap
loop end on the same signal, so no reap follows it and those transactions are
already counted as in flight at shutdown. Marking a hole put a failure on the
series that answers "was this run blind?" at the end of every healthy run.

Four causes shared one reason label. A node behind the head, a head never seen, a
height out of budget, and a receipt the node could not produce now have their own,
because the operator's next move differs for each.

One reviewer finding I did not take. It measured that Sei's block gas is
EVM-only, concluded block_empty_with_gas is clean signal, and asked me to act on
it. That measurement was of eth_getBlockByNumber, which sums receipt.GasUsed and
would be circular here. The newHeads header this code reads sums every
transaction's consensus result, Cosmos included, so the counter stays observed
rather than acted on. The comment now names which header, and names sei-chain's
own TODO to change it, because the ambiguity misled a careful reader.

Guards proven by breaking what they cover: the refusal's sentinel, the non-JSON
refusal, agreement across attempts, a resolver blip, the disjoint summary terms,
one record per gap, and a healthy shutdown marking no hole.

Method note: two mutations in this round reported as surviving when they were
really vet failures, and one survived because the fake fell through to success.
The battery checks vet and drives every attempt now.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 sender/doc.go                   |  18 ++-
 stats/inclusion_outcome_test.go | 139 +++++++++++++++--
 stats/inclusion_tracker.go      | 257 +++++++++++++++++++++++---------
 stats/inclusion_tracker_test.go |  15 ++
 stats/metrics.go                |  12 +-
 5 files changed, 350 insertions(+), 91 deletions(-)

diff --git a/sender/doc.go b/sender/doc.go
index c1ba21e..06b1086 100644
--- a/sender/doc.go
+++ b/sender/doc.go
@@ -114,8 +114,10 @@
 // after both the senders and the tracker have joined.
 //
 // Accepted boundaries. (1) WS gaps degrade conservatively: a missed head is
-// counted (block_gaps) but never backfilled, so its txs reap as expired —
-// an undercount of inclusions, never a miscount. (2) Reorgs use
+// counted (block_gaps) and never backfilled. The run never read those blocks,
+// so a tx in flight across the gap reaches status_unavailable rather than
+// expired: expired is a claim about the chain, and the run has no grounds for
+// one about a block it did not read. (2) Reorgs use
 // first-observation-wins (stamp + delete). This fixes an execution status as
 // well as a time: a tx that reverted on an orphaned block and committed on the
 // canonical one keeps the first answer. Sei's finality makes the window small,
@@ -124,11 +126,13 @@
 // receipts read costs the serving node work that grows with the block's tx
 // count, so a run at high TPS should name a second node. (4) InclusionTime is
 // the header-arrival wall clock, not fetch completion and not header.Time.
-// (5) A receipt read that returns nothing usable — an error, or an empty array
-// from a node that lost the bodies — is counted by reason
-// (block_fetch_errors) and not retried. Every tx in flight across it reaches
-// status_unavailable rather than expired, because the run cannot tell a chain
-// that left it out from a block it never read. (6) A tx registered after its
+// (5) A height the serving node has not reached is read again until a budget
+// runs out, because a receipt node trailing the head node is ordinary. Any other
+// read failure is counted by reason (block_fetch_errors) and not retried, since
+// a retry piles requests onto an endpoint already failing. Either way a tx in
+// flight across an unread height reaches status_unavailable rather than expired.
+// An empty array is not an unread height: a block carrying no EVM transaction
+// answers that way, and most blocks on an idle chain do. (6) A tx registered after its
 // including block was already scanned is missed and reaps as expired — bounded
 // by the microsecond register window versus block time, a rare conservative
 // undercount, the same direction as a WS gap.
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 46f1883..01b80d3 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -574,14 +574,18 @@ func TestADeferredHeightKeepsItsOwnArrival(t *testing.T) {
 			"latency sample in this topology is inflated by one block")
 }
 
-// TestAPendingHeightIsCountedWhenTheHeadsStop fails when a height still waiting
-// to be read is dropped instead of counted. Nothing else will read it, and
-// leaving it lets a transaction the chain may well have included reap as a
-// verdict about the chain.
-func TestAPendingHeightIsCountedWhenTheHeadsStop(t *testing.T) {
+// TestAPendingHeightAtShutdownIsNotAChainVerdict fails when a height still
+// waiting to be read at shutdown is turned into a failure.
+//
+// The head loop and the reap loop end on the same signal, so no reap follows the
+// shutdown sweep and those transactions are counted as in flight at shutdown,
+// which is already not a claim about the chain. Marking a hole would put a
+// failure on the series that answers "was this run blind?" at the end of every
+// healthy run in the topology this tracker recommends.
+func TestAPendingHeightAtShutdownIsNotAChainVerdict(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	tr := newTestTracker(t, time.Minute, 100, src)
 	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
 
 	tx := loadTx(1, time.Unix(1000, 0))
@@ -590,14 +594,21 @@ func TestAPendingHeightIsCountedWhenTheHeadsStop(t *testing.T) {
 
 	src.SetFetchErr(ethereum.NotFound)
 	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
-	// The head stream ends here, as it does at shutdown or on a stalled chain.
 	tr.flushDeferred(ctx)
-	tr.reap(ctx)
 
+	for s := range tr.state.Lock() {
+		require.Zero(t, s.blindFetches,
+			"a healthy shutdown marked a hole, so the series that answers "+
+				"'was this run blind?' is non-zero on every healthy run")
+	}
+	require.Equal(t, uint64(1), tr.Summary().InflightAtShutdown,
+		"the transaction was lost rather than counted as in flight")
+
+	// And a reap after that shutdown, were one to happen, still must not call it
+	// a chain verdict.
+	tr.reap(ctx)
 	got := tr.collector.GetOperationStats()[key]
-	require.Equal(t, uint64(1), got.StatusUnavailable,
-		"a height nothing ever read was reported as a chain result")
-	require.Zero(t, got.Expired)
+	require.Zero(t, got.Expired, "a pending height became a chain verdict")
 }
 
 // TestASkippedHeadIsCountedAsAHole fails when a height the run never saw a head
@@ -661,3 +672,109 @@ func TestTheDeferredQueueDrains(t *testing.T) {
 	require.Equal(t, int64(1), src.FetchCount()-before,
 		"a later head re-read a height that was already done")
 }
+
+// TestARefusedRunDoesNotLookLikeAFinishedOne fails when the preflight's refusal
+// wraps a context sentinel.
+//
+// main treats a context.Canceled or a context.DeadlineExceeded as the run ending
+// normally and exits zero. A refusal that wraps one is therefore a Complete run
+// that carried no load, which anything downstream reads as a chain that included
+// nothing: the false accusation this whole tracker exists to remove.
+func TestARefusedRunDoesNotLookLikeAFinishedOne(t *testing.T) {
+	for _, tc := range []struct {
+		name string
+		err  error
+	}{
+		{"answers_nothing", context.DeadlineExceeded},
+		{"not_listening", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}},
+		{"not_json", errors.New("invalid character '<' looking for beginning of value")},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			src := NewMockBlockSource().SetFetchErr(tc.err)
+			tr := newTestTracker(t, time.Minute, 100, src)
+
+			err := tr.preflight(context.Background(), "http://node:8545")
+			require.ErrorIs(t, err, ErrEndpointUnusable, "the refusal is not identifiable")
+			require.False(t,
+				errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded),
+				"the refusal reads as the run's own context ending, so the process exits zero")
+		})
+	}
+}
+
+// TestPreflightNeedsEveryAttemptToAgree fails when one attempt's verdict speaks
+// for the run. An endpoint that answers at all, even with an error, is there.
+func TestPreflightNeedsEveryAttemptToAgree(t *testing.T) {
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	// The first attempt times out; the second says the node is merely behind,
+	// which proves it answers and speaks the method.
+	src.FailTimes(1, context.DeadlineExceeded)
+	require.NoError(t, tr.preflight(context.Background(), "http://node:8545"))
+
+	// Two attempts, two different permanent-looking causes. That is not one
+	// permanent cause, so the run proceeds and reports what it finds.
+	// Three, one per attempt, so the loop cannot fall through to a success and
+	// reach the same verdict by another route.
+	refused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}
+	mixed := NewMockBlockSource().SetErrSequence(
+		context.DeadlineExceeded, refused, refused,
+	)
+	trMixed := newTestTracker(t, time.Minute, 100, mixed)
+	require.NoError(t, trMixed.preflight(context.Background(), "http://node:8545"),
+		"two different causes were treated as one settled verdict")
+
+	// Every attempt the same permanent cause is a refusal.
+	src.SetFetchErr(context.DeadlineExceeded)
+	require.ErrorIs(t, tr.preflight(context.Background(), "http://node:8545"),
+		ErrEndpointUnusable)
+}
+
+// TestATemporaryResolverFailureDoesNotRefuseTheRun fails when a resolver blip
+// ends a run. Only a name that does not exist is permanent.
+func TestATemporaryResolverFailureDoesNotRefuseTheRun(t *testing.T) {
+	servfail := &net.DNSError{Err: "server misbehaving", Name: "rpc-0", IsTemporary: true}
+	require.Equal(t, reasonOther, fetchFailureReason(servfail))
+
+	missing := &net.DNSError{Err: "no such host", Name: "rpc-0", IsNotFound: true}
+	require.Equal(t, reasonUnreachable, fetchFailureReason(missing))
+}
+
+// TestTheSummaryTermsAreDisjoint fails when one transaction lands in two terms
+// of the closing log line, which invites an operator to add them up and get more
+// than the run accepted.
+func TestTheSummaryTermsAreDisjoint(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tr.Register(ctx, tx)
+	// A receipt carrying a post-state root instead of a status: it was in a
+	// block, and what it did there cannot be read.
+	src.SetReceipts(4, blockReceipt{Hash: tx.EthTx.Hash(), HasStatus: false})
+	tr.matchBlock(ctx, 4, time.Unix(1002, 0))
+
+	s := tr.Summary()
+	total := s.Included + s.Expired + s.StatusUnavailable + s.DroppedAtCap + s.InflightAtShutdown
+	require.Equal(t, uint64(1), total,
+		"one accepted transaction was counted %d times across the summary's terms", total)
+}
+
+// TestAGapIsRecordedOnce fails when a missed-head range logs and counts per
+// height. A long gap would then push the run's own summary out of any bounded log
+// tail, which is the only diagnostic a failed run carries.
+func TestAGapIsRecordedOnce(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	last := tr.processHead(ctx, 10, 0, time.Unix(1002, 0), 0)
+	tr.processHead(ctx, 60, 0, time.Unix(1003, 0), last) // 49 heights missed
+
+	for s := range tr.state.Lock() {
+		require.Equal(t, uint64(1), s.blindFetches,
+			"a 49-height gap was recorded %d times", s.blindFetches)
+	}
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 8e81a29..2694bf0 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -125,14 +125,24 @@ type deferredRead struct {
 	num     uint64
 	gasUsed uint64
 	arrival time.Time
-	tries   int
+	// deferredAt is when the run first found the node had not reached this
+	// height. The wait is measured from here rather than from arrival, so a
+	// slow first read does not eat the budget.
+	deferredAt time.Time
+	tries      int
 }
 
-// maxDeferredReads bounds both how many heights wait to be read again and how
-// many times each is tried. A node one block behind needs one try. A node
-// further behind needs the depth, and past this bound it is behind rather than
-// busy, which is a hole worth counting.
-const maxDeferredReads = 4
+// How long a height waits to be read, and how many entries can wait.
+//
+// The wait is a duration rather than a count of heads. A count of heads means a
+// different tolerance on every chain, and it makes the failure a cliff: at one
+// head under the bound every block reads, at one head over it every block is a
+// hole. A duration is the quantity that actually matters, which is how far the
+// receipt node trails the head node.
+const (
+	deferredReadBudget = 5 * time.Second
+	maxDeferredReads   = 64
+)
 
 type entry struct {
 	tx           *types.LoadTx
@@ -317,8 +327,10 @@ func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoin
 		if err != nil {
 			return fmt.Errorf("inclusion tracker: connect WebSocket %s: %w", wsEndpoint, err)
 		}
-		// Buffered: head processing is serial and a slow read must not cost a
-		// head. The node drops a subscription whose buffer fills.
+		// Buffered: head processing is serial, and a read that takes its whole
+		// budget must not cost a head. go-ethereum buffers far more than this
+		// on its own side and fails the subscription rather than dropping
+		// quietly, so this is headroom for the handoff, not the real bound.
 		headers := make(chan *ethtypes.Header, 32)
 		sub, err := client.SubscribeNewHead(ctx, headers)
 		if err != nil {
@@ -367,11 +379,15 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64,
 		// A height nothing read is a hole, however it went missing. Leaving the
 		// gap uncounted let a tx in one of those blocks reap as expired, which
 		// is a claim about the chain the run has no grounds for.
-		for missed := lastSeen + 1; missed < num; missed++ {
-			t.recordBlindFetch(ctx, missed, reasonNotFound, nil)
-		}
-	}
-	t.matchBlockAttempt(ctx, num, gasUsed, arrival, 0)
+		//
+		// One record for the whole gap. The reap only asks whether the count
+		// rose, so recording each height would change nothing about attribution
+		// and would emit a log line per height. A long gap would then push the
+		// run's own summary out of any bounded log tail, which is the only
+		// diagnostic a failed run carries.
+		t.recordBlindGap(ctx, lastSeen+1, num-1)
+	}
+	t.matchBlockAttempt(ctx, num, gasUsed, arrival, time.Time{}, 0)
 	return num
 }
 
@@ -384,37 +400,57 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 		due, s.pending = s.pending, nil
 	}
 	for _, d := range due {
-		if d.tries >= maxDeferredReads {
-			t.recordBlindFetch(ctx, d.num, reasonNotFound, nil)
+		if waited := time.Since(d.deferredAt); waited > deferredReadBudget {
+			// The node has had its budget. It is behind rather than busy, and
+			// the run has to say it could not read this height.
+			inclusionDeferredWait.Record(ctx, waited.Seconds(), metric.WithAttributes(
+				attribute.String("chain_id", t.seiChainID),
+				attribute.String("outcome", "abandoned")))
+			t.recordBlindFetch(ctx, d.num, reasonBehind, nil)
 			continue
 		}
-		t.matchBlockAttempt(ctx, d.num, d.gasUsed, d.arrival, d.tries)
+		t.matchBlockAttempt(ctx, d.num, d.gasUsed, d.arrival, d.deferredAt, d.tries)
 	}
 }
 
-// flushDeferred counts every height still waiting as a hole. The head stream has
-// stopped, so nothing will read them, and leaving them would let a transaction
-// the chain may well have included reap as a verdict about the chain.
+// flushDeferred records that the run ended with heights it never read.
+//
+// It does not mark a hole. The head loop only ends when the run's context is
+// done, and the reap loop ends on the same signal, so no reap follows this and
+// the transactions in those heights are counted as in flight at shutdown, which
+// is already not a verdict about the chain. Marking a hole here would put a
+// failure on the series that answers "was this run blind?" at the end of every
+// healthy run.
 func (t *InclusionTracker) flushDeferred(ctx context.Context) {
 	var due []deferredRead
 	for s := range t.state.Lock() {
 		due, s.pending = s.pending, nil
 	}
 	for _, d := range due {
-		t.recordBlindFetch(ctx, d.num, reasonNotFound, nil)
+		inclusionDeferredWait.Record(ctx, time.Since(d.deferredAt).Seconds(),
+			metric.WithAttributes(
+				attribute.String("chain_id", t.seiChainID),
+				attribute.String("outcome", "unread_at_shutdown")))
 	}
 }
 
 // deferHeight queues a height to be read again, and reports whether it took it.
 // It refuses once the queue is full, so a node far behind produces holes instead
 // of a queue that grows with the run.
-func (t *InclusionTracker) deferHeight(num, gasUsed uint64, arrival time.Time, tries int) bool {
+func (t *InclusionTracker) deferHeight(num, gasUsed uint64, arrival, deferredAt time.Time, tries int) bool {
 	for s := range t.state.Lock() {
 		if len(s.pending) >= maxDeferredReads {
 			return false
 		}
+		// A re-deferral keeps the moment the run first found the node behind, so
+		// the budget measures the node's drift rather than restarting per try.
+		first := deferredAt
+		if first.IsZero() {
+			first = time.Now()
+		}
 		s.pending = append(s.pending, deferredRead{
-			num: num, gasUsed: gasUsed, arrival: arrival, tries: tries + 1,
+			num: num, gasUsed: gasUsed, arrival: arrival,
+			deferredAt: first, tries: tries + 1,
 		})
 		return true
 	}
@@ -440,8 +476,11 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 	// Several attempts, because one verdict stands for the whole run. A single
 	// reset or timeout from a node that is merely busy must not end a run before
 	// it starts.
+	// A refusal has to hold across every attempt. One attempt that answered at
+	// all, even with an error, proves the endpoint is there and speaks the
+	// method, and that evidence must not be erased by a later timeout.
 	var err error
-	var reason string
+	refusing := ""
 	for attempt := range preflightAttempts {
 		if attempt > 0 {
 			if _, waitErr := utils.Recv(ctx, time.After(preflightBackoff)); waitErr != nil {
@@ -454,64 +493,80 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 		if err == nil {
 			return nil
 		}
-		reason = fetchFailureReason(err)
-		if reason == reasonOther {
-			// Busy, reset, or something this run cannot name. Report and go.
-			break
+		reason := fetchFailureReason(err)
+		if !refusesTheRun(reason) {
+			// The endpoint answered. Whatever it said, it is there.
+			log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
+			return nil
 		}
+		if refusing != "" && refusing != reason {
+			// Two different permanent-looking causes is not one permanent
+			// cause. Report and let the run say what it finds.
+			log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
+			return nil
+		}
+		refusing = reason
 	}
-	switch reason {
+	switch refusing {
 	case reasonMethodUnavailable:
 		return fmt.Errorf(
-			"inclusion tracker: %s answers, but not eth_getBlockReceipts (%w). "+
+			"inclusion tracker: %w: %s answers, but not eth_getBlockReceipts (%v). "+
 				"Set receiptEndpoint in the profile to a node in fullNode or "+
 				"archive mode, which are the modes that serve EVM HTTP",
-			endpoint, err)
+			ErrEndpointUnusable, endpoint, err)
 	case reasonUnreachable:
 		return fmt.Errorf(
-			"inclusion tracker: %s is not serving EVM JSON-RPC (%w). "+
+			"inclusion tracker: %w: %s is not serving EVM JSON-RPC (%v). "+
 				"Set receiptEndpoint in the profile to a node in fullNode or "+
 				"archive mode; validator and seed modes serve no EVM HTTP",
-			endpoint, err)
+			ErrEndpointUnusable, endpoint, err)
 	case reasonTimeout:
 		// Nothing answered, every attempt. A dropped route and a firewall both
 		// look like this, and a node that cannot serve one genesis read in this
-		// many tries cannot serve a run.
+		// many tries cannot serve a run. Genesis costs the node nothing.
 		return fmt.Errorf(
-			"inclusion tracker: %s did not answer %d receipt reads (%w). "+
+			"inclusion tracker: %w: %s did not answer %d receipt reads (%v). "+
 				"Check that receiptEndpoint names a reachable node and that "+
 				"nothing between here and it is dropping the connection",
-			endpoint, preflightAttempts, err)
+			ErrEndpointUnusable, endpoint, preflightAttempts, err)
+	case reasonNotJSON:
+		return fmt.Errorf(
+			"inclusion tracker: %w: %s answered, but not with JSON-RPC (%v). "+
+				"Check that receiptEndpoint names the EVM HTTP port rather than "+
+				"the metrics port, the Cosmos RPC port, or an ingress path",
+			ErrEndpointUnusable, endpoint, err)
 	}
 	log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
 	return nil
 }
 
+// refusesTheRun reports whether a reason describes an endpoint that will still
+// be wrong when the run ends. Anything else is a node busy or behind, and the
+// run reports what it finds rather than refusing to start.
+func refusesTheRun(reason string) bool {
+	switch reason {
+	case reasonMethodUnavailable, reasonUnreachable, reasonNotJSON, reasonTimeout:
+		return true
+	default:
+		return false
+	}
+}
+
 // matchBlock fetches block num once and stamps every in-flight tx it includes
 // with the header-arrival time.
 func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival time.Time) {
-	t.matchBlockAttempt(ctx, num, 0, arrival, 0)
+	t.matchBlockAttempt(ctx, num, 0, arrival, time.Time{}, 0)
 }
 
 // matchBlockAttempt is matchBlock, carrying how many times this height has
 // already been read so a re-read does not restart the budget.
-func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed uint64, arrival time.Time, tries int) {
+func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed uint64, arrival, deferredAt time.Time, tries int) {
 	// Explicit per-iteration cancel (not deferred-in-loop): bound the fetch.
-	//
-	// A re-read gets a shorter budget than a first read. Head processing is
-	// serial and the node buffers a bounded number of heads before it drops the
-	// subscription, which ends the whole run, so two full-length reads in one
-	// head is more than that budget affords. A re-read is best-effort by
-	// construction: it becomes a hole rather than blocking on a node that hangs.
-	budget := firstReadTimeout
-	if tries > 0 {
-		budget = rereadTimeout
-	}
-	fetchCtx, cancel := context.WithTimeout(ctx, budget)
+	fetchCtx, cancel := context.WithTimeout(ctx, readTimeout)
 	receipts, nulls, err := t.source.BlockReceipts(fetchCtx, num)
 	cancel()
 	if err != nil {
-		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(num, gasUsed, arrival, tries) {
+		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(num, gasUsed, arrival, deferredAt, tries) {
 			// The node has not reached this height. Ordinary, and not a hole
 			// until a re-read says so.
 			return
@@ -529,11 +584,20 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 		t.recordBlindFetch(ctx, num, fetchFailureReason(err), err)
 		return
 	}
+	if !deferredAt.IsZero() {
+		// The node had not reached this height and now has. The distribution of
+		// these waits is how far the receipt node trails the head node, which is
+		// the quantity that decides whether this topology works.
+		inclusionDeferredWait.Record(ctx, time.Since(deferredAt).Seconds(),
+			metric.WithAttributes(
+				attribute.String("chain_id", t.seiChainID),
+				attribute.String("outcome", "read")))
+	}
 	if nulls > 0 {
 		// The read succeeded and part of it is unreadable. Match what arrived,
 		// and record the hole so a transaction the missing part would have named
 		// is not blamed on the chain.
-		t.recordBlindFetch(ctx, num, reasonNotFound, nil)
+		t.recordBlindFetch(ctx, num, reasonNullReceipt, nil)
 	}
 	if len(receipts) == 0 && gasUsed > 0 {
 		// An empty array usually means the block carried no EVM transaction,
@@ -542,12 +606,19 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 		// compacts it out of the array, so a block whose receipts went missing
 		// answers the same way.
 		//
-		// The head's gas comes from the consensus result rather than from any
-		// receipt, so gas burned with no receipt returned is the one witness
-		// that the two cases differ. It is counted and not acted on. Gas covers
-		// Cosmos transactions too, so treating it as a hole would invent one on
-		// any chain carrying non-EVM traffic, and over-reporting holes is what
-		// made expired unreachable once already.
+		// This is the gas on the head from the newHeads subscription, which
+		// sei-chain sums over every transaction's consensus result rather than
+		// over receipts. That independence is the whole reason it can witness a
+		// missing receipt. Note which header: eth_getBlockByNumber sums
+		// receipt.GasUsed instead, which would be circular here, and
+		// sei-chain's evmrpc/subscribe.go carries a TODO to make newHeads exact
+		// the same way. If that lands, this counter stops being able to fire.
+		//
+		// Counted and not acted on. The gas covers non-EVM transactions and an
+		// EVM transaction that failed the ante handler, both of which burn gas
+		// and produce no receipt legitimately, so treating this as a hole would
+		// invent holes. Over-reporting holes is what made expired unreachable
+		// once already.
 		inclusionEmptyWithGas.Add(ctx, 1, metric.WithAttributes(
 			attribute.String("chain_id", t.seiChainID)))
 	}
@@ -563,22 +634,24 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 			// wins (delete-on-touch) — see reorg note in sender/doc.go.
 			e.tx.InclusionTime = arrival
 			delete(s.inflight, r.Hash)
-			s.included++
 			// A receipt carries one status bit, so the outcome names what the run
 			// observed. Every failure cause shares the failed status.
 			outcome := OutcomeStatusUnavailable
 			switch {
 			case !r.HasStatus:
-				// It was in a block, so it is included. What it did there is
-				// unreadable, so the registry counts it where the ledger does.
-				s.statusUnavailable++
 				// A receipt with a post-state root instead of a status says the
 				// tx executed and does not say how it ended. Reading that as a
 				// failure would invent a chain result.
+				//
+				// It counts here and not in included, so the summary's terms
+				// stay disjoint and an operator can add them up.
+				s.statusUnavailable++
 			case r.Status == ethtypes.ReceiptStatusSuccessful:
 				outcome = OutcomeCommitted
+				s.included++
 			default:
 				outcome = OutcomeReverted
+				s.included++
 			}
 			resolved = append(resolved, resolvedOutcome{
 				outcome:  outcome,
@@ -695,6 +768,19 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 	}
 }
 
+// recordBlindGap marks that the run never saw the heads from first to last, so
+// it never read those blocks.
+func (t *InclusionTracker) recordBlindGap(ctx context.Context, first, last uint64) {
+	for s := range t.state.Lock() {
+		s.blindFetches++
+	}
+	log.Printf("inclusion tracker: missed heads %d..%d (%d blocks), never read",
+		first, last, last-first+1)
+	inclusionBlockFetchErrors.Add(ctx, 1, metric.WithAttributes(
+		attribute.String("chain_id", t.seiChainID),
+		attribute.String("reason", reasonMissedHead)))
+}
+
 // recordBlindFetch marks that the run could not read one block's receipts. Every
 // tx in flight now reaps as status_unavailable rather than expired, because the
 // run cannot tell the two apart for a block it never read.
@@ -722,16 +808,40 @@ const (
 	reasonNotFound          = "not_found"
 	reasonTimeout           = "timeout"
 	reasonOther             = "other"
-)
 
-// Read budgets. Head processing is serial, and the node drops a subscription
-// whose head buffer fills, which ends the run, so the total spent in one head
-// matters more than any single read.
-const (
-	firstReadTimeout = 10 * time.Second
-	rereadTimeout    = 2 * time.Second
+	// These name why a height went unread rather than why a call failed. They
+	// share the counter and not the cause, because the operator's next move
+	// differs: a node behind the head is a topology problem, a head the run
+	// never saw is a subscription problem, and a receipt the node could not
+	// produce is a chain problem.
+	// reasonNotJSON is an endpoint that answered with something other than
+	// JSON-RPC: an ingress error page, a metrics port, the Cosmos RPC port. It
+	// stays broken for the run's length, so it refuses the run.
+	reasonNotJSON = "not_json"
+
+	reasonBehind      = "receipt_node_behind"
+	reasonMissedHead  = "missed_head"
+	reasonNullReceipt = "null_receipt"
 )
 
+// readTimeout bounds one receipt read.
+//
+// Every read gets the same budget. An earlier version gave a re-read less, on
+// the reasoning that two long reads in one head cost too much. That had it
+// backwards: in the two-node topology this tracker recommends, the first read of
+// a height returns not-found cheaply and the re-read is the one that carries the
+// receipts, so the short budget landed on the only read that matters.
+const readTimeout = 10 * time.Second
+
+// ErrEndpointUnusable means the receipt endpoint cannot serve this run, and the
+// run stops rather than reporting numbers it did not measure.
+//
+// It is a sentinel because main treats a context.Canceled or a
+// context.DeadlineExceeded as the run ending normally. Wrapping the probe's own
+// error would have made a refusal exit zero, which is a Complete run with no
+// load, read by anything downstream as a chain that included nothing.
+var ErrEndpointUnusable = errors.New("receipt endpoint unusable")
+
 // How hard the preflight tries before it speaks for the whole run.
 const (
 	preflightAttempts = 3
@@ -769,8 +879,7 @@ func fetchFailureReason(err error) string {
 	var httpErr rpc.HTTPError
 	if errors.As(err, &httpErr) {
 		switch httpErr.StatusCode {
-		case http.StatusOK, http.StatusBadRequest,
-			http.StatusForbidden, http.StatusMethodNotAllowed:
+		case http.StatusBadRequest, http.StatusForbidden, http.StatusMethodNotAllowed:
 			if bytes.Contains(httpErr.Body, []byte(strconv.Itoa(methodNotFoundCode))) {
 				return reasonMethodUnavailable
 			}
@@ -795,9 +904,14 @@ func fetchFailureReason(err error) string {
 	// a caller, and reading it as unreachable would kill a run for being busy,
 	// which is worse than the blind run this bucket exists to prevent. It falls
 	// through to other.
+	// Only a name that does not exist. A resolver answering SERVFAIL is
+	// temporary, and refusing a run for it would end a run over a blip in DNS.
 	var dnsErr *net.DNSError
 	if errors.As(err, &dnsErr) {
-		return reasonUnreachable
+		if dnsErr.IsNotFound {
+			return reasonUnreachable
+		}
+		return reasonOther
 	}
 	var opErr *net.OpError
 	if errors.As(err, &opErr) && opErr.Op == "dial" && errors.Is(err, syscall.ECONNREFUSED) {
@@ -811,6 +925,9 @@ func fetchFailureReason(err error) string {
 	case strings.Contains(msg, "connection refused") ||
 		strings.Contains(msg, "no such host"):
 		return reasonUnreachable
+	case strings.Contains(msg, "looking for beginning of value") ||
+		strings.Contains(msg, "invalid character"):
+		return reasonNotJSON
 	case strings.Contains(msg, "not found"):
 		return reasonNotFound
 	case strings.Contains(msg, "deadline exceeded"):
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index 0ee7047..99df156 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -27,6 +27,7 @@ type MockBlockSource struct {
 	fetchErr error
 	failures int
 	nulls    int
+	errSeq   []error
 }
 
 func NewMockBlockSource() *MockBlockSource {
@@ -67,6 +68,15 @@ func (m *MockBlockSource) SetNulls(n int) *MockBlockSource {
 	return m
 }
 
+// SetErrSequence makes each later read fail with the next error in turn, so a
+// test can drive an endpoint whose cause changes between attempts.
+func (m *MockBlockSource) SetErrSequence(errs ...error) *MockBlockSource {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.errSeq = errs
+	return m
+}
+
 // FailTimes makes the next n fetches fail and the rest succeed, so a test can
 // drive a node that recovers rather than one that stays broken.
 func (m *MockBlockSource) FailTimes(n int, err error) *MockBlockSource {
@@ -81,6 +91,11 @@ func (m *MockBlockSource) BlockReceipts(_ context.Context, n uint64) ([]blockRec
 	m.fetches.Add(1)
 	m.mu.Lock()
 	defer m.mu.Unlock()
+	if len(m.errSeq) > 0 {
+		err := m.errSeq[0]
+		m.errSeq = m.errSeq[1:]
+		return nil, 0, err
+	}
 	if m.fetchErr != nil && m.failures != 0 {
 		if m.failures > 0 {
 			m.failures--
diff --git a/stats/metrics.go b/stats/metrics.go
index 2939589..9fbff41 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -73,17 +73,23 @@ var (
 
 	inclusionBlockGaps = must(meter.Int64Counter(
 		"block_gaps",
-		metric.WithDescription("Missed block heights observed by the inclusion tracker (no backfill)"),
+		metric.WithDescription("Block heights the head subscription never delivered. Not backfilled; each gap is counted once as a hole so its txs are not blamed on the chain"),
 		metric.WithUnit("{blocks}")))
 
+	inclusionDeferredWait = must(meter.Float64Histogram(
+		"deferred_read_wait",
+		metric.WithDescription("How long a height waited for the receipt node to reach it, by outcome. A rising distribution is the receipt node falling behind the head node"),
+		metric.WithUnit("s"),
+		metric.WithExplicitBucketBoundaries(0.25, 0.5, 1, 2, 5, 10)))
+
 	inclusionEmptyWithGas = must(meter.Int64Counter(
 		"block_empty_with_gas",
-		metric.WithDescription("Blocks that returned no receipts while the head reported gas burned. Usually non-EVM traffic; a sustained rate means receipts are going missing"),
+		metric.WithDescription("Blocks that returned no receipts while the head reported gas burned. Causes: non-EVM traffic, an EVM tx that failed the ante handler and got no receipt, or receipts going missing from the store. Only the last is a defect, and this counter cannot tell them apart"),
 		metric.WithUnit("{blocks}")))
 
 	inclusionBlockFetchErrors = must(meter.Int64Counter(
 		"block_fetch_errors",
-		metric.WithDescription("Receipt reads that failed, by reason, after the re-reads a height gets; that block goes unmatched, so a tx in flight across it reaches status_unavailable"),
+		metric.WithDescription("Heights the run could not read, by reason; a tx in flight across one reaches status_unavailable rather than expired"),
 		metric.WithUnit("{blocks}")))
 
 	// Run-summary: the only inclusion tally with no live series, since it is the

From a7edf8043d216d2f9546335841efcc7d7cfa9d75 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 09:00:50 -0700
Subject: [PATCH 09/17] fix(stats): bound what one head spends re-reading

Found by reading my own last commit. Making the deferral bound a duration raised
how many heights can be waiting at once from four to about a dozen in steady
state, and the sweep gave each one a full read budget. Head processing is serial,
so one hanging node could spend minutes inside a single head while the chain
moved on.

The sweep shares one budget now. A height it does not reach stays queued for the
next head, which costs a head of delay rather than a hole, and the requeue puts
the oldest first so nothing is starved.

Guard proven by lifting the budget: the sweep then ran to the full length of
every queued read.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_outcome_test.go | 31 +++++++++++++++++++++++++++++++
 stats/inclusion_tracker.go      | 32 +++++++++++++++++++++++++++++---
 stats/inclusion_tracker_test.go | 20 ++++++++++++++++++++
 3 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 01b80d3..793917f 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -778,3 +778,34 @@ func TestAGapIsRecordedOnce(t *testing.T) {
 			"a 49-height gap was recorded %d times", s.blindFetches)
 	}
 }
+
+// TestOneHeadCannotSpendUnboundedTimeReReading fails when a sweep gives every
+// waiting height its own read budget.
+//
+// Head processing is serial. A dozen heights can be waiting at once, so a
+// per-read budget would let one hanging node spend minutes inside a single head
+// while the chain moves on.
+func TestOneHeadCannotSpendUnboundedTimeReReading(t *testing.T) {
+	ctx := context.Background()
+	src := NewSlowBlockSource(ethereum.NotFound, rereadSweepBudget/2)
+	tr := newInclusionTrackerWithSource(
+		NewInclusionTracker("test-chain", time.Minute, 100, true, NewCollector()), src)
+
+	// Queue several heights the node has not reached, without paying for a read
+	// per queueing.
+	now := time.Now()
+	for h := uint64(2); h <= 8; h++ {
+		require.True(t, tr.deferHeight(h, 0, now, now, 0))
+	}
+
+	start := time.Now()
+	tr.rereadDeferred(ctx)
+	spent := time.Since(start)
+
+	require.Less(t, spent, rereadSweepBudget*2,
+		"one head spent %s re-reading; the sweep budget is %s", spent, rereadSweepBudget)
+	for s := range tr.state.Lock() {
+		require.NotEmpty(t, s.pending,
+			"the heights the sweep ran out of time for were dropped rather than requeued")
+	}
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 2694bf0..21efe09 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -142,6 +142,10 @@ type deferredRead struct {
 const (
 	deferredReadBudget = 5 * time.Second
 	maxDeferredReads   = 64
+	// rereadSweepBudget bounds what one head spends re-reading, whatever is
+	// waiting. Head processing is serial, so this and one first read are what a
+	// head costs at worst.
+	rereadSweepBudget = 3 * time.Second
 )
 
 type entry struct {
@@ -391,15 +395,26 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64,
 	return num
 }
 
-// rereadDeferred reads every height the receipt node had not reached, using that
-// height's own arrival. A height out of tries becomes a hole here, because
+// rereadDeferred reads the heights the receipt node had not reached, using each
+// height's own arrival. A height out of budget becomes a hole here, because
 // nothing downstream would notice it was never read.
+//
+// Head processing is serial, so the whole sweep shares one budget rather than
+// each read carrying its own. A dozen heights can be waiting at once, and giving
+// each a full read budget would let one hanging node spend minutes inside a
+// single head. A height the sweep does not reach stays queued for the next one,
+// which costs a head of delay and not a hole.
 func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 	var due []deferredRead
 	for s := range t.state.Lock() {
 		due, s.pending = s.pending, nil
 	}
-	for _, d := range due {
+	sweepUntil := time.Now().Add(rereadSweepBudget)
+	for i, d := range due {
+		if time.Now().After(sweepUntil) {
+			t.requeue(due[i:])
+			return
+		}
 		if waited := time.Since(d.deferredAt); waited > deferredReadBudget {
 			// The node has had its budget. It is behind rather than busy, and
 			// the run has to say it could not read this height.
@@ -413,6 +428,17 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 	}
 }
 
+// requeue puts back the heights a sweep ran out of time for, ahead of anything
+// deferred since, so the oldest is tried first and cannot be starved.
+func (t *InclusionTracker) requeue(left []deferredRead) {
+	for s := range t.state.Lock() {
+		s.pending = append(left, s.pending...)
+		if len(s.pending) > maxDeferredReads {
+			s.pending = s.pending[:maxDeferredReads]
+		}
+	}
+}
+
 // flushDeferred records that the run ended with heights it never read.
 //
 // It does not mark a hole. The head loop only ends when the run's context is
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index 99df156..90cfdab 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -393,3 +393,23 @@ func TestInclusion_ConcurrentRaceSafe(t *testing.T) {
 	require.Equal(t, uint64(n), s.Included+s.Expired+s.InflightAtShutdown,
 		"conservation holds under concurrency")
 }
+
+// SlowBlockSource fails every read after a delay, so a test can measure what a
+// hanging endpoint costs a head.
+type SlowBlockSource struct {
+	err   error
+	delay time.Duration
+}
+
+func NewSlowBlockSource(err error, delay time.Duration) *SlowBlockSource {
+	return &SlowBlockSource{err: err, delay: delay}
+}
+
+func (s *SlowBlockSource) BlockReceipts(ctx context.Context, _ uint64) ([]blockReceipt, int, error) {
+	select {
+	case <-time.After(s.delay):
+		return nil, 0, s.err
+	case <-ctx.Done():
+		return nil, 0, ctx.Err()
+	}
+}

From aafa31e5d45efe02161f8d767979090ebb057a36 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 09:19:37 -0700
Subject: [PATCH 10/17] refactor(stats): retire what five rounds of fixes left
 behind

An idiom review of the file as it now stands, rather than of the last delta.
Five rounds of defect-driven fixes is exactly how a file accumulates
incoherence, and it had.

The retry count was dead. Round 5 replaced a count-based retirement with a
duration and left the field written, passed along, and never compared, plus four
comments still describing the mechanism it belonged to. An editor tuning
re-read depth would have changed a constant that moves nothing.

Removing it made a second fix obvious. matchBlockAttempt and deferHeight took
four and five positional parameters carrying most of deferredRead's fields, two
of them adjacent same-typed timestamps. Transposing arrival and deferredAt
compiles, and it would have measured the inclusion latency and the retry budget
from each other's instant. They take the struct now, so a first read passes no
hand-written zero values at all.

The take-and-clear critical section was duplicated verbatim at two sites that
must change together. takePending owns it again.

blindFetches counted heads that never arrived as well as reads that failed, so
neither its name nor its doc was true. It is blindHeights.

deferred_read_wait labelled its values outcome, which inclusion_outcome already
uses for a disjoint set. One label name meaning two things across two
instruments is the wire hazard Outcome's own type exists to prevent, and the
file states that rule thirty lines above where it broke it. The label is
disposition, and its three values have constants.

requeue dropped a height past the cap silently. deferHeight already holds the
queue at the cap so nothing reaches that branch, but a height vanishing from it
would leave blindHeights unmoved and let a transaction from that block reap as a
verdict about the chain. It records instead.

block_gaps described the arithmetic of a different counter: it adds one per
missed height, while the once-per-gap record lands on block_fetch_errors.

Comment discipline, with the line drawn where the reviewer drew it. A present
fact about the deployment shape stays, because it is the constraint that makes a
shorter re-read budget wrong. The argument with the version that had it
backwards goes, because that is the commit's job and not the code's. Same
treatment for the conservation identity's history and for the empty-array
incident. Two blocks that explain a non-obvious external API stay untouched.

Also: flushDeferred promised a flush and performed an abandon, so it is
recordUnreadAtShutdown, and the call site that still argued the behaviour it no
longer has is gone. refusesTheRun reads as though the reason refuses; it is
isPermanent. reasonNotJSON sat under a comment saying it was not a call failure.
The DNS rationale had been appended to the paragraph about connection resets.
Two nolint directives named a linter this repo does not enable, and their
reasoning survives as plain comments.

stats/doc.go gains the fourth sentinel, the queue's bounds under the lock it
lives behind, the lifecycle branch where a preflight refuses a run, and the two
tests that guard the queue. sender/doc.go gains the sweep bound and the shutdown
boundary.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 sender/doc.go                   |   4 +-
 stats/doc.go                    |  25 +++-
 stats/inclusion_outcome_test.go |  10 +-
 stats/inclusion_tracker.go      | 202 +++++++++++++++++---------------
 stats/inclusion_tracker_test.go |   6 +-
 stats/metrics.go                |   4 +-
 6 files changed, 145 insertions(+), 106 deletions(-)

diff --git a/sender/doc.go b/sender/doc.go
index 06b1086..28c5218 100644
--- a/sender/doc.go
+++ b/sender/doc.go
@@ -127,7 +127,9 @@
 // count, so a run at high TPS should name a second node. (4) InclusionTime is
 // the header-arrival wall clock, not fetch completion and not header.Time.
 // (5) A height the serving node has not reached is read again until a budget
-// runs out, because a receipt node trailing the head node is ordinary. Any other
+// runs out, because a receipt node trailing the head node is ordinary. One head
+// spends a bounded total on those re-reads, so a height the sweep does not reach
+// waits for the next head rather than becoming a hole. Any other
 // read failure is counted by reason (block_fetch_errors) and not retried, since
 // a retry piles requests onto an endpoint already failing. Either way a tx in
 // flight across an unread height reaches status_unavailable rather than expired.
diff --git a/stats/doc.go b/stats/doc.go
index 1c66463..550e094 100644
--- a/stats/doc.go
+++ b/stats/doc.go
@@ -19,7 +19,7 @@
 //
 // # Zero values and sentinels
 //
-// Three of them carry meaning, and each one exists because the obvious default
+// Four of them carry meaning, and each one exists because the obvious default
 // would lie.
 //
 //   - Outcome's zero value is outcomeUnset, not a real state. A transaction
@@ -29,6 +29,10 @@
 //     sender and the tracker have joined. Read earlier it is a race.
 //   - RunSummary.InclusionTracked separates a run with no tracker from a tracked
 //     run that saw no inclusions. Both would otherwise report zero.
+//   - ErrEndpointUnusable marks a refusal to start. It is deliberately not a
+//     context error: main reads a cancelled or expired run context as the run
+//     ending normally and exits zero, so a refusal wrapping one would report a
+//     run that carried no load as a run that finished.
 //
 // # Concurrency
 //
@@ -38,7 +42,10 @@
 //     RecordOutcome take it themselves; recordOperation is called with it
 //     already held.
 //   - TPSWindow.mu guards the rolling rate window.
-//   - InclusionTracker.state guards the in-flight registry.
+//   - InclusionTracker.state guards the in-flight registry and the queue of
+//     heights waiting to be read again. That queue holds at most
+//     maxDeferredReads entries, and an entry leaves it once the height is read
+//     or once deferredReadBudget runs out.
 //
 // The rule: report an outcome to the collector outside the tracker's registry
 // lock. The sender blocks on that lock at every send completion, so work held
@@ -55,8 +62,11 @@
 // point.
 //
 // InclusionTracker runs for the length of the run. Run dials the receipt
-// endpoint, proves it answers, subscribes to new heads, and then two goroutines
-// drive it: the head loop matches each arriving block once, and the reap loop
+// endpoint and proves it answers. That proof can refuse the run: an endpoint
+// that never answers, refuses the method, or replies with something other than
+// JSON-RPC ends the run with ErrEndpointUnusable rather than letting it report
+// numbers it did not measure. Otherwise Run subscribes to new heads, and then
+// two goroutines drive it: the head loop matches each arriving block once, and the reap loop
 // evicts transactions that outlived reapAfter. Both end when the run context
 // does. A transaction still in the registry at that point reached no terminal
 // state and is counted as inflight_at_shutdown, which is why the conservation
@@ -81,5 +91,10 @@
 // sender/doc.go owns the conservation identity and states it there.
 // TestInclusion_Conservation asserts the registry identity, and
 // TestOutcomesPartitionEveryAcceptedTx asserts that Outcome's terminal states
-// partition every registered transaction.
+// partition every accepted transaction.
+//
+// The deferred-read queue's bounds are guarded too:
+// TestOneHeadCannotSpendUnboundedTimeReReading holds one head's sweep inside
+// rereadSweepBudget, and TestAReceiptNodeThatStaysBehindBecomesAHole holds a
+// height inside deferredReadBudget.
 package stats
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 793917f..72b94ef 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -594,10 +594,10 @@ func TestAPendingHeightAtShutdownIsNotAChainVerdict(t *testing.T) {
 
 	src.SetFetchErr(ethereum.NotFound)
 	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
-	tr.flushDeferred(ctx)
+	tr.recordUnreadAtShutdown(ctx)
 
 	for s := range tr.state.Lock() {
-		require.Zero(t, s.blindFetches,
+		require.Zero(t, s.blindHeights,
 			"a healthy shutdown marked a hole, so the series that answers "+
 				"'was this run blind?' is non-zero on every healthy run")
 	}
@@ -774,8 +774,8 @@ func TestAGapIsRecordedOnce(t *testing.T) {
 	tr.processHead(ctx, 60, 0, time.Unix(1003, 0), last) // 49 heights missed
 
 	for s := range tr.state.Lock() {
-		require.Equal(t, uint64(1), s.blindFetches,
-			"a 49-height gap was recorded %d times", s.blindFetches)
+		require.Equal(t, uint64(1), s.blindHeights,
+			"a 49-height gap was recorded %d times", s.blindHeights)
 	}
 }
 
@@ -795,7 +795,7 @@ func TestOneHeadCannotSpendUnboundedTimeReReading(t *testing.T) {
 	// per queueing.
 	now := time.Now()
 	for h := uint64(2); h <= 8; h++ {
-		require.True(t, tr.deferHeight(h, 0, now, now, 0))
+		require.True(t, tr.deferHeight(deferredRead{num: h, arrival: now, deferredAt: now}))
 	}
 
 	start := time.Now()
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 21efe09..deed548 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -71,8 +71,8 @@ type receiptSource interface {
 type ethReceiptSource struct{ client *ethclient.Client }
 
 func (s ethReceiptSource) BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, int, error) {
-	//nolint:gosec // A block height never approaches MaxInt64, where the
-	// conversion would land on rpc.BlockNumber's negative sentinels.
+	// A block height never approaches MaxInt64, where the conversion would land
+	// on rpc.BlockNumber's negative sentinels.
 	number := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(n))
 	receipts, err := s.client.BlockReceipts(ctx, number)
 	if err != nil {
@@ -126,12 +126,22 @@ type deferredRead struct {
 	gasUsed uint64
 	arrival time.Time
 	// deferredAt is when the run first found the node had not reached this
-	// height. The wait is measured from here rather than from arrival, so a
-	// slow first read does not eat the budget.
+	// height, and zero on a first read. The wait is measured from here rather
+	// than from arrival, so a slow first read does not eat the budget and a
+	// re-read does not restart it.
 	deferredAt time.Time
-	tries      int
 }
 
+// What became of a height that waited to be read. These are label values on
+// deferred_read_wait. They are not Outcome values and the label is not called
+// outcome, because one label name meaning two disjoint sets across two
+// instruments is the same wire hazard Outcome's own type exists to prevent.
+const (
+	waitRead             = "read"
+	waitAbandoned        = "abandoned"
+	waitUnreadAtShutdown = "unread_at_shutdown"
+)
+
 // How long a height waits to be read, and how many entries can wait.
 //
 // The wait is a duration rather than a count of heads. A count of heads means a
@@ -151,23 +161,23 @@ const (
 type entry struct {
 	tx           *types.LoadTx
 	registeredAt time.Time
-	// blindFetchesAtRegistration is inclusionState.blindFetches at the moment
+	// blindHeightsAtRegistration is inclusionState.blindHeights at the moment
 	// this tx was registered. A reap compares it against the count now: a
-	// higher count means a receipt fetch failed while this tx was in flight, so
+	// higher count means a height went unread while this tx was in flight, so
 	// the run cannot say the chain left it out. See reap.
-	blindFetchesAtRegistration uint64
+	blindHeightsAtRegistration uint64
 }
 
 type inclusionState struct {
-	// blindFetches counts receipt reads that failed. It only ever grows, so an
-	// entry's terminal state follows from comparing this against the value it
-	// recorded at registration.
+	// blindHeights counts heights the run could not read, whether a read failed
+	// or a head never arrived. It only ever grows, so an entry's terminal state
+	// follows from comparing this against the value it recorded at registration.
 	//
-	// The count carries no height, so it marks a tx in flight across any failed
-	// read rather than a tx in the block that read failed on. The tracker does
+	// The count carries no height, so it marks a tx in flight across any unread
+	// height rather than a tx in the block that went unread. The tracker does
 	// not know which registered txs a block it never read was holding, and
 	// claiming to would be the invention this state exists to avoid.
-	blindFetches uint64
+	blindHeights uint64
 	// statusUnavailable counts reaped txs that spanned a failed read, plus
 	// duplicate registrations. It is the part the run cannot attribute to the
 	// chain.
@@ -177,8 +187,8 @@ type inclusionState struct {
 	// can say nothing about it.
 	duplicates uint64
 	// pending holds heights the receipt node had not reached, waiting to be read
-	// again. It is bounded by maxDeferredReads entries, and each entry is
-	// dropped once it is read or once it runs out of tries, so a node that stays
+	// again. It is bounded by maxDeferredReads entries, and an entry leaves once
+	// it is read or once its deferredReadBudget runs out, so a node that stays
 	// behind produces holes rather than a growing queue.
 	pending            []deferredRead
 	inflight           map[common.Hash]*entry
@@ -191,9 +201,8 @@ type inclusionState struct {
 // InclusionTracker matches arriving blocks against in-flight txs, stamps
 // InclusionTime, and resolves each one to a terminal [Outcome].
 //
-// sender/doc.go states the conservation identity these outcomes satisfy. It is
-// not restated here: it was, in a shorter form, and the two drifted apart when
-// the states grew.
+// sender/doc.go owns the conservation identity these outcomes satisfy, and it is
+// not restated here.
 type InclusionTracker struct {
 	seiChainID  string
 	reapAfter   time.Duration
@@ -280,7 +289,7 @@ func (t *InclusionTracker) Register(ctx context.Context, tx *types.LoadTx) {
 			outcome = OutcomeStatusUnavailable
 			break
 		}
-		s.inflight[hash] = &entry{tx: tx, registeredAt: time.Now(), blindFetchesAtRegistration: s.blindFetches}
+		s.inflight[hash] = &entry{tx: tx, registeredAt: time.Now(), blindHeightsAtRegistration: s.blindHeights}
 	}
 	if outcome != outcomeUnset {
 		t.report(ctx, outcome, tx.Scenario)
@@ -310,9 +319,10 @@ func (t *InclusionTracker) meterOutcome(ctx context.Context, outcome Outcome, sc
 // one node.
 //
 // The tracker reads the height it just received as a head, and re-reads a height
-// the serving node had not reached yet. It reaches back no further than
-// maxDeferredReads heights, so the serving node's receipt retention does not
-// bound it. A change that reaches further back does.
+// the serving node had not reached yet. A height is only re-read while it is
+// younger than deferredReadBudget, so the reach back is seconds and the serving
+// node's receipt retention does not bound it. A change that reaches further back
+// does.
 func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoint string) error {
 	wsEndpoint := utils.GetWSEndpoint(headEndpoint)
 	if t.source == nil {
@@ -351,11 +361,9 @@ func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoin
 		s.Spawn(func() error { return t.reapLoop(ctx) })
 
 		var lastSeen uint64 // 0 = unset; first head seeds it (no backfill).
-		// Whatever ends the head loop, a height still waiting to be read is
-		// counted before the run reports. Nothing else will read it, and
-		// leaving it lets a tx the chain may well have included reap as a
-		// verdict about the chain.
-		defer t.flushDeferred(ctx)
+		// The head loop can end with heights still queued; recordUnreadAtShutdown
+		// owns what happens to them.
+		defer t.recordUnreadAtShutdown(ctx)
 		for ctx.Err() == nil {
 			header, err := utils.Recv(ctx, headers)
 			if err != nil {
@@ -378,7 +386,7 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64,
 		return lastSeen // duplicate or out-of-order head: no re-fetch, no spurious gap.
 	}
 	if lastSeen != 0 && num > lastSeen+1 {
-		inclusionBlockGaps.Add(ctx, int64(num-lastSeen-1), metric.WithAttributes( //nolint:gosec
+		inclusionBlockGaps.Add(ctx, int64(num-lastSeen-1), metric.WithAttributes(
 			attribute.String("chain_id", t.seiChainID)))
 		// A height nothing read is a hole, however it went missing. Leaving the
 		// gap uncounted let a tx in one of those blocks reap as expired, which
@@ -391,7 +399,7 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64,
 		// diagnostic a failed run carries.
 		t.recordBlindGap(ctx, lastSeen+1, num-1)
 	}
-	t.matchBlockAttempt(ctx, num, gasUsed, arrival, time.Time{}, 0)
+	t.matchBlockAttempt(ctx, deferredRead{num: num, gasUsed: gasUsed, arrival: arrival})
 	return num
 }
 
@@ -405,14 +413,11 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64,
 // single head. A height the sweep does not reach stays queued for the next one,
 // which costs a head of delay and not a hole.
 func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
-	var due []deferredRead
-	for s := range t.state.Lock() {
-		due, s.pending = s.pending, nil
-	}
+	due := t.takePending()
 	sweepUntil := time.Now().Add(rereadSweepBudget)
 	for i, d := range due {
 		if time.Now().After(sweepUntil) {
-			t.requeue(due[i:])
+			t.requeue(ctx, due[i:])
 			return
 		}
 		if waited := time.Since(d.deferredAt); waited > deferredReadBudget {
@@ -420,26 +425,48 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 			// the run has to say it could not read this height.
 			inclusionDeferredWait.Record(ctx, waited.Seconds(), metric.WithAttributes(
 				attribute.String("chain_id", t.seiChainID),
-				attribute.String("outcome", "abandoned")))
+				attribute.String("disposition", waitAbandoned)))
 			t.recordBlindFetch(ctx, d.num, reasonBehind, nil)
 			continue
 		}
-		t.matchBlockAttempt(ctx, d.num, d.gasUsed, d.arrival, d.deferredAt, d.tries)
+		t.matchBlockAttempt(ctx, d)
+	}
+}
+
+// takePending removes every queued height and returns it. Both callers must take
+// and clear in one critical section, so one function owns that.
+func (t *InclusionTracker) takePending() []deferredRead {
+	for s := range t.state.Lock() {
+		due := s.pending
+		s.pending = nil
+		return due
 	}
+	panic("unreachable")
 }
 
 // requeue puts back the heights a sweep ran out of time for, ahead of anything
 // deferred since, so the oldest is tried first and cannot be starved.
-func (t *InclusionTracker) requeue(left []deferredRead) {
+//
+// A height past the cap is recorded rather than dropped. deferHeight already
+// holds the queue at the cap, so nothing reaches that branch today; a height
+// that vanished from it silently would leave blindHeights unmoved and let a tx
+// from that block reap as a verdict about the chain, which is the one thing this
+// file must never do quietly.
+func (t *InclusionTracker) requeue(ctx context.Context, left []deferredRead) {
+	var dropped []deferredRead
 	for s := range t.state.Lock() {
 		s.pending = append(left, s.pending...)
 		if len(s.pending) > maxDeferredReads {
+			dropped = s.pending[maxDeferredReads:]
 			s.pending = s.pending[:maxDeferredReads]
 		}
 	}
+	for _, d := range dropped {
+		t.recordBlindFetch(ctx, d.num, reasonBehind, nil)
+	}
 }
 
-// flushDeferred records that the run ended with heights it never read.
+// recordUnreadAtShutdown records that the run ended with heights it never read.
 //
 // It does not mark a hole. The head loop only ends when the run's context is
 // done, and the reap loop ends on the same signal, so no reap follows this and
@@ -447,37 +474,29 @@ func (t *InclusionTracker) requeue(left []deferredRead) {
 // is already not a verdict about the chain. Marking a hole here would put a
 // failure on the series that answers "was this run blind?" at the end of every
 // healthy run.
-func (t *InclusionTracker) flushDeferred(ctx context.Context) {
-	var due []deferredRead
-	for s := range t.state.Lock() {
-		due, s.pending = s.pending, nil
-	}
-	for _, d := range due {
+func (t *InclusionTracker) recordUnreadAtShutdown(ctx context.Context) {
+	for _, d := range t.takePending() {
 		inclusionDeferredWait.Record(ctx, time.Since(d.deferredAt).Seconds(),
 			metric.WithAttributes(
 				attribute.String("chain_id", t.seiChainID),
-				attribute.String("outcome", "unread_at_shutdown")))
+				attribute.String("disposition", waitUnreadAtShutdown)))
 	}
 }
 
 // deferHeight queues a height to be read again, and reports whether it took it.
 // It refuses once the queue is full, so a node far behind produces holes instead
 // of a queue that grows with the run.
-func (t *InclusionTracker) deferHeight(num, gasUsed uint64, arrival, deferredAt time.Time, tries int) bool {
+func (t *InclusionTracker) deferHeight(d deferredRead) bool {
 	for s := range t.state.Lock() {
 		if len(s.pending) >= maxDeferredReads {
 			return false
 		}
 		// A re-deferral keeps the moment the run first found the node behind, so
-		// the budget measures the node's drift rather than restarting per try.
-		first := deferredAt
-		if first.IsZero() {
-			first = time.Now()
+		// the budget measures the node's drift rather than restarting per read.
+		if d.deferredAt.IsZero() {
+			d.deferredAt = time.Now()
 		}
-		s.pending = append(s.pending, deferredRead{
-			num: num, gasUsed: gasUsed, arrival: arrival,
-			deferredAt: first, tries: tries + 1,
-		})
+		s.pending = append(s.pending, d)
 		return true
 	}
 	panic("unreachable")
@@ -506,7 +525,8 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 	// all, even with an error, proves the endpoint is there and speaks the
 	// method, and that evidence must not be erased by a later timeout.
 	var err error
-	refusing := ""
+	// The empty string means no verdict yet.
+	var refusing string
 	for attempt := range preflightAttempts {
 		if attempt > 0 {
 			if _, waitErr := utils.Recv(ctx, time.After(preflightBackoff)); waitErr != nil {
@@ -520,7 +540,7 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 			return nil
 		}
 		reason := fetchFailureReason(err)
-		if !refusesTheRun(reason) {
+		if !isPermanent(reason) {
 			// The endpoint answered. Whatever it said, it is there.
 			log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
 			return nil
@@ -566,10 +586,10 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 	return nil
 }
 
-// refusesTheRun reports whether a reason describes an endpoint that will still
+// isPermanent reports whether a reason describes an endpoint that will still
 // be wrong when the run ends. Anything else is a node busy or behind, and the
 // run reports what it finds rather than refusing to start.
-func refusesTheRun(reason string) bool {
+func isPermanent(reason string) bool {
 	switch reason {
 	case reasonMethodUnavailable, reasonUnreachable, reasonNotJSON, reasonTimeout:
 		return true
@@ -581,23 +601,25 @@ func refusesTheRun(reason string) bool {
 // matchBlock fetches block num once and stamps every in-flight tx it includes
 // with the header-arrival time.
 func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival time.Time) {
-	t.matchBlockAttempt(ctx, num, 0, arrival, time.Time{}, 0)
+	t.matchBlockAttempt(ctx, deferredRead{num: num, arrival: arrival})
 }
 
-// matchBlockAttempt is matchBlock, carrying how many times this height has
-// already been read so a re-read does not restart the budget.
-func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed uint64, arrival, deferredAt time.Time, tries int) {
+// matchBlockAttempt is matchBlock over a queued read. It takes the whole
+// deferredRead rather than its fields: arrival and deferredAt are both
+// timestamps, and transposing them as arguments would compile while measuring
+// the latency and the budget from each other's instant.
+func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, d deferredRead) {
 	// Explicit per-iteration cancel (not deferred-in-loop): bound the fetch.
 	fetchCtx, cancel := context.WithTimeout(ctx, readTimeout)
-	receipts, nulls, err := t.source.BlockReceipts(fetchCtx, num)
+	receipts, nulls, err := t.source.BlockReceipts(fetchCtx, d.num)
 	cancel()
 	if err != nil {
-		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(num, gasUsed, arrival, deferredAt, tries) {
+		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(d) {
 			// The node has not reached this height. Ordinary, and not a hole
 			// until a re-read says so.
 			return
 		}
-		// The block goes unmatched, and blindFetches records that this run has
+		// The block goes unmatched, and blindHeights records that this run has
 		// a hole, so a tx in flight across it reaps as status_unavailable
 		// rather than as a verdict about the chain.
 		//
@@ -607,25 +629,25 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 		// ethereum.NotFound. Reading an empty array as a hole would mark every
 		// idle block, and a chain that stopped accepting work produces nothing
 		// but idle blocks, which is the one run where expired is the answer.
-		t.recordBlindFetch(ctx, num, fetchFailureReason(err), err)
+		t.recordBlindFetch(ctx, d.num, fetchFailureReason(err), err)
 		return
 	}
-	if !deferredAt.IsZero() {
+	if !d.deferredAt.IsZero() {
 		// The node had not reached this height and now has. The distribution of
 		// these waits is how far the receipt node trails the head node, which is
 		// the quantity that decides whether this topology works.
-		inclusionDeferredWait.Record(ctx, time.Since(deferredAt).Seconds(),
+		inclusionDeferredWait.Record(ctx, time.Since(d.deferredAt).Seconds(),
 			metric.WithAttributes(
 				attribute.String("chain_id", t.seiChainID),
-				attribute.String("outcome", "read")))
+				attribute.String("disposition", waitRead)))
 	}
 	if nulls > 0 {
 		// The read succeeded and part of it is unreadable. Match what arrived,
 		// and record the hole so a transaction the missing part would have named
 		// is not blamed on the chain.
-		t.recordBlindFetch(ctx, num, reasonNullReceipt, nil)
+		t.recordBlindFetch(ctx, d.num, reasonNullReceipt, nil)
 	}
-	if len(receipts) == 0 && gasUsed > 0 {
+	if len(receipts) == 0 && d.gasUsed > 0 {
 		// An empty array usually means the block carried no EVM transaction,
 		// and it is the answer for most blocks on an idle chain. It has one
 		// other cause: sei-chain drops a receipt its store cannot find and
@@ -643,8 +665,7 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 		// Counted and not acted on. The gas covers non-EVM transactions and an
 		// EVM transaction that failed the ante handler, both of which burn gas
 		// and produce no receipt legitimately, so treating this as a hole would
-		// invent holes. Over-reporting holes is what made expired unreachable
-		// once already.
+		// invent holes, and holes reported too often make expired unreachable.
 		inclusionEmptyWithGas.Add(ctx, 1, metric.WithAttributes(
 			attribute.String("chain_id", t.seiChainID)))
 	}
@@ -658,7 +679,7 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 			}
 			// Single writer of InclusionTime, under the lock; first observation
 			// wins (delete-on-touch) — see reorg note in sender/doc.go.
-			e.tx.InclusionTime = arrival
+			e.tx.InclusionTime = d.arrival
 			delete(s.inflight, r.Hash)
 			// A receipt carries one status bit, so the outcome names what the run
 			// observed. Every failure cause shares the failed status.
@@ -690,7 +711,7 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, num, gasUsed u
 			// bogus epoch-based duration. See LoadTx contract.
 			if t.openLoop && !e.tx.IntendedSendTime.IsZero() {
 				matched = append(matched, inclusionSample{
-					latency:  arrival.Sub(e.tx.IntendedSendTime).Seconds(),
+					latency:  d.arrival.Sub(e.tx.IntendedSendTime).Seconds(),
 					scenario: e.tx.Scenario,
 				})
 			}
@@ -778,7 +799,7 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 			// expired on the surface an operator reads first and as
 			// status_unavailable on the one they read second.
 			outcome := OutcomeExpired
-			if s.blindFetches > e.blindFetchesAtRegistration {
+			if s.blindHeights > e.blindHeightsAtRegistration {
 				outcome = OutcomeStatusUnavailable
 				s.statusUnavailable++
 			} else {
@@ -798,7 +819,7 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 // it never read those blocks.
 func (t *InclusionTracker) recordBlindGap(ctx context.Context, first, last uint64) {
 	for s := range t.state.Lock() {
-		s.blindFetches++
+		s.blindHeights++
 	}
 	log.Printf("inclusion tracker: missed heads %d..%d (%d blocks), never read",
 		first, last, last-first+1)
@@ -812,7 +833,7 @@ func (t *InclusionTracker) recordBlindGap(ctx context.Context, first, last uint6
 // run cannot tell the two apart for a block it never read.
 func (t *InclusionTracker) recordBlindFetch(ctx context.Context, num uint64, reason string, err error) {
 	for s := range t.state.Lock() {
-		s.blindFetches++
+		s.blindHeights++
 	}
 	if err != nil {
 		log.Printf("inclusion tracker: fetch block %d (%s): %v", num, reason, err)
@@ -834,29 +855,25 @@ const (
 	reasonNotFound          = "not_found"
 	reasonTimeout           = "timeout"
 	reasonOther             = "other"
+	// reasonNotJSON is an endpoint that answered with something other than
+	// JSON-RPC: an ingress error page, a metrics port, the Cosmos RPC port. It
+	// stays broken for the run's length, so it refuses the run.
+	reasonNotJSON = "not_json"
 
 	// These name why a height went unread rather than why a call failed. They
 	// share the counter and not the cause, because the operator's next move
 	// differs: a node behind the head is a topology problem, a head the run
 	// never saw is a subscription problem, and a receipt the node could not
 	// produce is a chain problem.
-	// reasonNotJSON is an endpoint that answered with something other than
-	// JSON-RPC: an ingress error page, a metrics port, the Cosmos RPC port. It
-	// stays broken for the run's length, so it refuses the run.
-	reasonNotJSON = "not_json"
-
 	reasonBehind      = "receipt_node_behind"
 	reasonMissedHead  = "missed_head"
 	reasonNullReceipt = "null_receipt"
 )
 
-// readTimeout bounds one receipt read.
-//
-// Every read gets the same budget. An earlier version gave a re-read less, on
-// the reasoning that two long reads in one head cost too much. That had it
-// backwards: in the two-node topology this tracker recommends, the first read of
-// a height returns not-found cheaply and the re-read is the one that carries the
-// receipts, so the short budget landed on the only read that matters.
+// readTimeout bounds one receipt read. Every read gets the same budget: in the
+// two-node topology this tracker recommends, a height's first read returns
+// not-found cheaply and the re-read is the one that carries the receipts, so a
+// shorter re-read budget would land on the only read that matters.
 const readTimeout = 10 * time.Second
 
 // ErrEndpointUnusable means the receipt endpoint cannot serve this run, and the
@@ -930,7 +947,8 @@ func fetchFailureReason(err error) string {
 	// a caller, and reading it as unreachable would kill a run for being busy,
 	// which is worse than the blind run this bucket exists to prevent. It falls
 	// through to other.
-	// Only a name that does not exist. A resolver answering SERVFAIL is
+	//
+	// Only a name that does not exist counts. A resolver answering SERVFAIL is
 	// temporary, and refusing a run for it would end a run over a blip in DNS.
 	var dnsErr *net.DNSError
 	if errors.As(err, &dnsErr) {
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index 90cfdab..89be2bb 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -15,7 +15,11 @@ import (
 	"github.com/sei-protocol/sei-load/types"
 )
 
-// MockBlockSource is a deterministic receiptSource for tests. Setter style
+// MockBlockSource is a deterministic receiptSource for tests.
+//
+// Three knobs inject failure and they do not compose: a sequence set by
+// SetErrSequence wins while it has entries, and SetFetchErr and FailTimes share
+// one slot, so the later call replaces the earlier. Setter style
 // mirrors MockBlockStats: SetBlock seeds a block's receipts; fetches are counted.
 //
 // SetBlock takes hashes and marks each committed, which is what most tests want.
diff --git a/stats/metrics.go b/stats/metrics.go
index 9fbff41..3721ff4 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -73,12 +73,12 @@ var (
 
 	inclusionBlockGaps = must(meter.Int64Counter(
 		"block_gaps",
-		metric.WithDescription("Block heights the head subscription never delivered. Not backfilled; each gap is counted once as a hole so its txs are not blamed on the chain"),
+		metric.WithDescription("Block heights the head subscription never delivered, counted per height. Not backfilled; the gap is recorded once as a hole on block_fetch_errors{reason=missed_head}"),
 		metric.WithUnit("{blocks}")))
 
 	inclusionDeferredWait = must(meter.Float64Histogram(
 		"deferred_read_wait",
-		metric.WithDescription("How long a height waited for the receipt node to reach it, by outcome. A rising distribution is the receipt node falling behind the head node"),
+		metric.WithDescription("How long a height waited for the receipt node to reach it, by disposition. A rising distribution is the receipt node falling behind the head node"),
 		metric.WithUnit("s"),
 		metric.WithExplicitBucketBoundaries(0.25, 0.5, 1, 2, 5, 10)))
 

From ae36863524284c6a4064cba3fe1bf7cacd560d14 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 09:39:05 -0700
Subject: [PATCH 11/17] fix(stats): make the sweep budget an actual bound

The commit titled "bound what one head spends re-reading" did not. The check ran
before a read and never bounded the read in progress, so a read starting at
2.999s ran a further ten. Measured 12.8s against a stated 3s, and a worst case of
23s where the shape it replaced was 18s. The bound moved the wrong way in the
commit named for fixing it.

That is not only a cost. The head loop stamps arrival at dequeue, and that value
becomes the inclusion latency sample, so a sweep that overruns writes the
tracker's own backlog into the number the run exists to report.

A re-read now gets whatever is left of the sweep. Which exposed the next thing: a
read this process cut short is not evidence about the node, so that height goes
back in the queue rather than being called a hole. Blaming the serving node for a
deadline sei-load imposed on itself is the same error as blaming the chain for a
block the run never read.

requeue put the unreached tail in front of what the sweep had already re-deferred
and its comment claimed the opposite. A sweep walks oldest first, so what it
re-deferred is older; prepending served the newest first and let the oldest age
out against a budget they were never given a turn under.

The refusal reasons were two lists that had to agree: one deciding whether a
reason ends the run, one turning it into a message. Adding a reason to one and
not the other gave either a silent stall through the retry loop or a refusal
nothing could reach, with no signal from the compiler or a test. One table now.

block_fetch_errors counted a gap once while every other reason counted per
height, so an operator summing it undercounted by the length of every gap. The
watermark still rises once, because the reap only asks whether it rose.

Six guards were missing and one was worse than missing: deleting requeue outright
left the suite green while heights vanished, because the assertion only asked
whether the queue was non-empty and the heights the sweep did read had refilled
it. The whole duration-budget mechanism had no test at all, so it could be
disabled, set to five hundred hours, or restarted on every read without a
failure. Each now has a guard, proven by breaking what it covers.

Also: the latency histogram's count is no longer the included count, since a
matched receipt with an unreadable status still samples, and the comment said
otherwise.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_outcome_test.go | 132 +++++++++++++++++++++++++++++++-
 stats/inclusion_tracker.go      | 110 +++++++++++++++-----------
 stats/metrics.go                |   8 +-
 3 files changed, 200 insertions(+), 50 deletions(-)

diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 72b94ef..4ceb5d6 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -787,7 +787,11 @@ func TestAGapIsRecordedOnce(t *testing.T) {
 // while the chain moves on.
 func TestOneHeadCannotSpendUnboundedTimeReReading(t *testing.T) {
 	ctx := context.Background()
-	src := NewSlowBlockSource(ethereum.NotFound, rereadSweepBudget/2)
+	// Each read takes most of the whole sweep budget, so the second one starts
+	// with almost none left. A budget that only gates the next read rather than
+	// bounding the one in progress lets that second read run to its own full
+	// length, which is the overshoot this guards.
+	src := NewSlowBlockSource(ethereum.NotFound, rereadSweepBudget*9/10)
 	tr := newInclusionTrackerWithSource(
 		NewInclusionTracker("test-chain", time.Minute, 100, true, NewCollector()), src)
 
@@ -802,10 +806,130 @@ func TestOneHeadCannotSpendUnboundedTimeReReading(t *testing.T) {
 	tr.rereadDeferred(ctx)
 	spent := time.Since(start)
 
-	require.Less(t, spent, rereadSweepBudget*2,
+	require.Less(t, spent, rereadSweepBudget+time.Second,
 		"one head spent %s re-reading; the sweep budget is %s", spent, rereadSweepBudget)
+
+	// Every height is still accounted for. Asserting only that the queue is
+	// non-empty passes while heights vanish, because the ones the sweep did read
+	// re-defer themselves and fill it.
+	require.ElementsMatch(t, []uint64{2, 3, 4, 5, 6, 7, 8}, pendingHeights(tr),
+		"a height the sweep ran out of time for left the queue")
+	for s := range tr.state.Lock() {
+		require.Zero(t, s.blindHeights, "a height that is still queued was also called a hole")
+	}
+}
+
+// pendingHeights reads the queued heights, for a test that has to name them all.
+func pendingHeights(tr *InclusionTracker) []uint64 {
+	for s := range tr.state.Lock() {
+		out := make([]uint64, 0, len(s.pending))
+		for _, d := range s.pending {
+			out = append(out, d.num)
+		}
+		return out
+	}
+	panic("unreachable")
+}
+
+// TestTheOldestQueuedHeightIsReadFirst fails when a sweep puts the heights it
+// did not reach ahead of the ones it re-deferred. A sweep walks oldest first, so
+// what it re-deferred is older, and prepending would serve the newest first and
+// let the oldest age out against a budget it was never given a turn under.
+func TestTheOldestQueuedHeightIsReadFirst(t *testing.T) {
+	ctx := context.Background()
+	src := NewSlowBlockSource(ethereum.NotFound, rereadSweepBudget/2)
+	tr := newInclusionTrackerWithSource(
+		NewInclusionTracker("test-chain", time.Minute, 100, true, NewCollector()), src)
+
+	now := time.Now()
+	for h := uint64(2); h <= 8; h++ {
+		require.True(t, tr.deferHeight(deferredRead{num: h, arrival: now, deferredAt: now}))
+	}
+	tr.rereadDeferred(ctx)
+
+	got := pendingHeights(tr)
+	require.Equal(t, []uint64{2, 3, 4, 5, 6, 7, 8}, got,
+		"the queue came back out of age order as %v", got)
+}
+
+// TestAHeightOutOfBudgetBecomesAHole fails when the wait budget never retires a
+// height. The budget is the whole reason the bound is a duration rather than a
+// count of reads, and the queue cap must not be what fires instead.
+func TestAHeightOutOfBudgetBecomesAHole(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource().SetFetchErr(ethereum.NotFound)
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	// One height, so the queue cap cannot be what retires it.
+	old := time.Now().Add(-deferredReadBudget - time.Second)
+	require.True(t, tr.deferHeight(deferredRead{num: 7, arrival: old, deferredAt: old}))
+	tr.rereadDeferred(ctx)
+
+	require.Empty(t, pendingHeights(tr), "a height past its budget stayed queued")
 	for s := range tr.state.Lock() {
-		require.NotEmpty(t, s.pending,
-			"the heights the sweep ran out of time for were dropped rather than requeued")
+		require.Equal(t, uint64(1), s.blindHeights,
+			"a height past its budget left the queue without being called a hole")
 	}
 }
+
+// TestTheWaitBudgetDoesNotRestartOnEachRead fails when a re-read resets the
+// clock. The budget measures how far the receipt node trails the head node, and
+// restarting it per read would mean a node behind forever is never called
+// behind.
+func TestTheWaitBudgetDoesNotRestartOnEachRead(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource().SetFetchErr(ethereum.NotFound)
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	first := time.Now().Add(-deferredReadBudget / 2)
+	require.True(t, tr.deferHeight(deferredRead{num: 7, arrival: first, deferredAt: first}))
+	tr.rereadDeferred(ctx) // re-read fails, re-defers
+
+	queued := func() deferredRead {
+		for s := range tr.state.Lock() {
+			require.Len(t, s.pending, 1)
+			return s.pending[0]
+		}
+		panic("unreachable")
+	}()
+	require.Equal(t, first, queued.deferredAt,
+		"the wait restarted, so the height can never reach its budget")
+}
+
+// TestADuplicateHeadStillDrainsTheQueue fails when a repeated or out-of-order
+// head returns before the sweep. Nothing else drains the queue, so a height left
+// in it reaps as a chain verdict for a block never read.
+func TestADuplicateHeadStillDrainsTheQueue(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tr.Register(ctx, tx)
+
+	// Head 7 arrives before the receipt node holds it.
+	src.SetFetchErr(ethereum.NotFound)
+	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
+
+	// The same head again. The node has caught up by now.
+	src.SetFetchErr(nil)
+	src.SetReceipts(7, blockReceipt{
+		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
+	})
+	tr.processHead(ctx, 7, 0, time.Unix(1003, 0), 7)
+
+	require.Equal(t, uint64(1), tr.Summary().Included,
+		"a repeated head returned without draining the queue")
+}
+
+// TestAnAnswerAfterATimeoutDoesNotRefuseTheRun fails when one timed-out attempt
+// decides the run. A later attempt that answers at all proves the endpoint is
+// there, whatever it said.
+func TestAnAnswerAfterATimeoutDoesNotRefuseTheRun(t *testing.T) {
+	src := NewMockBlockSource().SetErrSequence(
+		context.DeadlineExceeded, ethereum.NotFound, ethereum.NotFound,
+	)
+	tr := newTestTracker(t, time.Minute, 100, src)
+	require.NoError(t, tr.preflight(context.Background(), "http://node:8545"),
+		"an endpoint that answered was refused because an earlier attempt timed out")
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index deed548..eaf68b4 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -399,7 +399,7 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64,
 		// diagnostic a failed run carries.
 		t.recordBlindGap(ctx, lastSeen+1, num-1)
 	}
-	t.matchBlockAttempt(ctx, deferredRead{num: num, gasUsed: gasUsed, arrival: arrival})
+	t.matchBlockAttempt(ctx, deferredRead{num: num, gasUsed: gasUsed, arrival: arrival}, readTimeout)
 	return num
 }
 
@@ -416,7 +416,11 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 	due := t.takePending()
 	sweepUntil := time.Now().Add(rereadSweepBudget)
 	for i, d := range due {
-		if time.Now().After(sweepUntil) {
+		// The deadline bounds the read, not just the decision to start one.
+		// Checking the clock before each read and then letting that read run its
+		// own full budget is not a bound: it is the budget plus one whole read.
+		left := time.Until(sweepUntil)
+		if left <= 0 {
 			t.requeue(ctx, due[i:])
 			return
 		}
@@ -429,7 +433,7 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 			t.recordBlindFetch(ctx, d.num, reasonBehind, nil)
 			continue
 		}
-		t.matchBlockAttempt(ctx, d)
+		t.matchBlockAttempt(ctx, d, left)
 	}
 }
 
@@ -444,8 +448,12 @@ func (t *InclusionTracker) takePending() []deferredRead {
 	panic("unreachable")
 }
 
-// requeue puts back the heights a sweep ran out of time for, ahead of anything
-// deferred since, so the oldest is tried first and cannot be starved.
+// requeue puts back the heights a sweep ran out of time for.
+//
+// They go behind what the sweep already re-deferred, not in front of it. A sweep
+// walks oldest first, so anything it re-deferred is older than anything it did
+// not reach, and prepending would put the newest heights at the head of the
+// queue and serve the oldest last.
 //
 // A height past the cap is recorded rather than dropped. deferHeight already
 // holds the queue at the cap, so nothing reaches that branch today; a height
@@ -455,7 +463,7 @@ func (t *InclusionTracker) takePending() []deferredRead {
 func (t *InclusionTracker) requeue(ctx context.Context, left []deferredRead) {
 	var dropped []deferredRead
 	for s := range t.state.Lock() {
-		s.pending = append(left, s.pending...)
+		s.pending = append(s.pending, left...)
 		if len(s.pending) > maxDeferredReads {
 			dropped = s.pending[maxDeferredReads:]
 			s.pending = s.pending[:maxDeferredReads]
@@ -540,7 +548,7 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 			return nil
 		}
 		reason := fetchFailureReason(err)
-		if !isPermanent(reason) {
+		if _, permanent := permanentReasons[reason]; !permanent {
 			// The endpoint answered. Whatever it said, it is there.
 			log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
 			return nil
@@ -553,67 +561,75 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 		}
 		refusing = reason
 	}
-	switch refusing {
-	case reasonMethodUnavailable:
+	if refuse, permanent := permanentReasons[refusing]; permanent {
+		return refuse(endpoint, preflightAttempts, err)
+	}
+	log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
+	return nil
+}
+
+// permanentReasons maps a reason that will still be wrong when the run ends to
+// the refusal it produces. One table, because a reason that refuses and a
+// refusal that names it are the same fact: two lists would let a reason be added
+// to one and forgotten in the other, giving either a silent stall through the
+// retry loop or a refusal nothing can reach.
+//
+// Anything absent here is a node busy or behind, and the run reports what it
+// finds rather than refusing to start.
+var permanentReasons = map[string]func(endpoint string, attempts int, err error) error{
+	reasonMethodUnavailable: func(endpoint string, _ int, err error) error {
 		return fmt.Errorf(
 			"inclusion tracker: %w: %s answers, but not eth_getBlockReceipts (%v). "+
 				"Set receiptEndpoint in the profile to a node in fullNode or "+
 				"archive mode, which are the modes that serve EVM HTTP",
 			ErrEndpointUnusable, endpoint, err)
-	case reasonUnreachable:
+	},
+	reasonUnreachable: func(endpoint string, _ int, err error) error {
 		return fmt.Errorf(
 			"inclusion tracker: %w: %s is not serving EVM JSON-RPC (%v). "+
 				"Set receiptEndpoint in the profile to a node in fullNode or "+
 				"archive mode; validator and seed modes serve no EVM HTTP",
 			ErrEndpointUnusable, endpoint, err)
-	case reasonTimeout:
-		// Nothing answered, every attempt. A dropped route and a firewall both
-		// look like this, and a node that cannot serve one genesis read in this
-		// many tries cannot serve a run. Genesis costs the node nothing.
-		return fmt.Errorf(
-			"inclusion tracker: %w: %s did not answer %d receipt reads (%v). "+
-				"Check that receiptEndpoint names a reachable node and that "+
-				"nothing between here and it is dropping the connection",
-			ErrEndpointUnusable, endpoint, preflightAttempts, err)
-	case reasonNotJSON:
+	},
+	reasonNotJSON: func(endpoint string, _ int, err error) error {
 		return fmt.Errorf(
 			"inclusion tracker: %w: %s answered, but not with JSON-RPC (%v). "+
 				"Check that receiptEndpoint names the EVM HTTP port rather than "+
 				"the metrics port, the Cosmos RPC port, or an ingress path",
 			ErrEndpointUnusable, endpoint, err)
-	}
-	log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
-	return nil
-}
-
-// isPermanent reports whether a reason describes an endpoint that will still
-// be wrong when the run ends. Anything else is a node busy or behind, and the
-// run reports what it finds rather than refusing to start.
-func isPermanent(reason string) bool {
-	switch reason {
-	case reasonMethodUnavailable, reasonUnreachable, reasonNotJSON, reasonTimeout:
-		return true
-	default:
-		return false
-	}
+	},
+	reasonTimeout: func(endpoint string, attempts int, err error) error {
+		return fmt.Errorf(
+			"inclusion tracker: %w: %s did not answer %d receipt reads (%v). "+
+				"Check that receiptEndpoint names a reachable node and that "+
+				"nothing between here and it is dropping the connection",
+			ErrEndpointUnusable, endpoint, attempts, err)
+	},
 }
 
 // matchBlock fetches block num once and stamps every in-flight tx it includes
 // with the header-arrival time.
 func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival time.Time) {
-	t.matchBlockAttempt(ctx, deferredRead{num: num, arrival: arrival})
+	t.matchBlockAttempt(ctx, deferredRead{num: num, arrival: arrival}, readTimeout)
 }
 
 // matchBlockAttempt is matchBlock over a queued read. It takes the whole
 // deferredRead rather than its fields: arrival and deferredAt are both
 // timestamps, and transposing them as arguments would compile while measuring
 // the latency and the budget from each other's instant.
-func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, d deferredRead) {
+func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, d deferredRead, budget time.Duration) {
 	// Explicit per-iteration cancel (not deferred-in-loop): bound the fetch.
-	fetchCtx, cancel := context.WithTimeout(ctx, readTimeout)
+	fetchCtx, cancel := context.WithTimeout(ctx, min(budget, readTimeout))
 	receipts, nulls, err := t.source.BlockReceipts(fetchCtx, d.num)
 	cancel()
 	if err != nil {
+		// A read this sweep cut short is not evidence about the node. The
+		// height goes back in the queue: calling it a hole would blame the
+		// serving node for a deadline this process imposed on itself.
+		cutShort := budget < readTimeout && errors.Is(err, context.DeadlineExceeded)
+		if cutShort && t.deferHeight(d) {
+			return
+		}
 		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(d) {
 			// The node has not reached this height. Ordinary, and not a hole
 			// until a re-read says so.
@@ -823,7 +839,11 @@ func (t *InclusionTracker) recordBlindGap(ctx context.Context, first, last uint6
 	}
 	log.Printf("inclusion tracker: missed heads %d..%d (%d blocks), never read",
 		first, last, last-first+1)
-	inclusionBlockFetchErrors.Add(ctx, 1, metric.WithAttributes(
+	// One height, one count, the same as every other reason on this counter.
+	// The watermark above rises once, because the reap only asks whether it
+	// rose, but a counter of heights that reported a gap as one would undercount
+	// by the length of every gap.
+	inclusionBlockFetchErrors.Add(ctx, int64(last-first+1), metric.WithAttributes(
 		attribute.String("chain_id", t.seiChainID),
 		attribute.String("reason", reasonMissedHead)))
 }
@@ -870,10 +890,14 @@ const (
 	reasonNullReceipt = "null_receipt"
 )
 
-// readTimeout bounds one receipt read. Every read gets the same budget: in the
-// two-node topology this tracker recommends, a height's first read returns
-// not-found cheaply and the re-read is the one that carries the receipts, so a
-// shorter re-read budget would land on the only read that matters.
+// readTimeout bounds one first read of a height. A re-read gets whatever is left
+// of the sweep instead, so the sweep's budget is a real ceiling on what one head
+// costs rather than a ceiling plus one more read.
+//
+// A first read is not shortened: in the two-node topology this tracker
+// recommends, a height's first read returns not-found cheaply and the re-read is
+// the one that carries the receipts, so a shorter budget on the wrong one of the
+// two is what a previous shape got wrong.
 const readTimeout = 10 * time.Second
 
 // ErrEndpointUnusable means the receipt endpoint cannot serve this run, and the
diff --git a/stats/metrics.go b/stats/metrics.go
index 3721ff4..8111508 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -57,9 +57,11 @@ var (
 		metric.WithUnit("{transactions}")))
 
 	// Inclusion tracker. inclusion_latency is open-loop only (closed-loop
-	// IntendedSendTime is enqueue time, not a schedule); its _count is the
-	// included count only there. Denominator for inclusion rate is the existing
-	// succeeded/txs_accepted series, never a new "registered" series.
+	// IntendedSendTime is enqueue time, not a schedule). Its _count is every tx
+	// a receipt named there, which is InclusionSummary.Included plus any whose
+	// status could not be read, so the two are not interchangeable. Denominator
+	// for inclusion rate is the existing succeeded/txs_accepted series, never a
+	// new "registered" series.
 	inclusionLatency = must(meter.Float64Histogram(
 		"inclusion_latency",
 		metric.WithDescription("Latency from intended send to observed on-chain inclusion in seconds"),

From 4e0179c411cb9e8d21ec3a575362378b5ed5a383 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 10:15:58 -0700
Subject: [PATCH 12/17] fix(stats): a run that is behind cannot say the chain
 left a tx out

The strongest finding of the review, and the one every earlier round walked
past. It was measured: 565 of 1000 transactions reported expired on a chain that
included every one of them, with every hole counter reading zero.

No read fails. No head is missed. No receipt is null. The tracker's head loop is
serial, so when reads take longer than the block interval a backlog builds, and
the block carrying a transaction is opened after the reap already evicted it.
Five rounds hardened every path where a read fails; this is the path where every
read succeeds and the tracker is late. sender/doc.go says expired is a claim
about the chain and the run has no grounds for one about a block it did not
read. At reap time the run had that block in hand and had not opened it.

The registry now knows the highest head taken off the wire and the highest whose
block has been read. A reap that finds them apart cannot say expired, because
the transaction may be sitting in a height the run is holding. A caught-up run
still says expired, which is the point of keeping the two states apart, and that
direction has its own test.

The same backlog corrupted the number this tool exists to report. Arrival was
stamped after the head came off the channel, so every latency sample carried the
queue. Measured at 8.2 seconds of error after 16 seconds of chain at 1.5x block
time, and 43 seconds at 3.75x, against a histogram whose top bucket is 120. The
stamp is taken where the head arrives now, by a step that exists to keep it
there, and head_lag reports the gap. That is the number that separates a chain
taking nothing from a run that could not keep up: both show un-included
transactions, and only this one says which.

A head stream that ends mid-run no longer fails the run. Any error from the
tracker cancelled the whole scope and exited non-zero, so a dropped WebSocket on
a read-only observer turned a good run red and killed the senders, the generator
and the report with it. That is the mirror of the refused-run-exits-zero defect
fixed earlier: this one is the false fail. Tracking stops, every later height is
unread so nothing after it reaps as a chain verdict, and the run finishes.

Also: the WebSocket client was never closed.

Guards proven by breaking what they cover: the backlog rule in both directions,
a dead head stream, a head never counted as received, a head counted only after
its block was read, and a stamp taken late.

The stamping needed the pump extracted to be testable at all. A guard that
drives processHead directly cannot see where Run takes its timestamp, and that
is the third time in this series a guard has named something it did not check.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_outcome_test.go | 113 +++++++++++++++++++++++++
 stats/inclusion_tracker.go      | 142 ++++++++++++++++++++++++++++++--
 stats/metrics.go                |   6 ++
 3 files changed, 252 insertions(+), 9 deletions(-)

diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 4ceb5d6..3d98ad2 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -3,6 +3,7 @@ package stats
 import (
 	"context"
 	"errors"
+	"math/big"
 	"net"
 	"strconv"
 	"syscall"
@@ -933,3 +934,115 @@ func TestAnAnswerAfterATimeoutDoesNotRefuseTheRun(t *testing.T) {
 	require.NoError(t, tr.preflight(context.Background(), "http://node:8545"),
 		"an endpoint that answered was refused because an earlier attempt timed out")
 }
+
+// TestABacklogIsNotAChainVerdict fails when a transaction is called expired
+// because the tracker had not got to its block yet.
+//
+// This is the case every other hole in this file misses. No read fails, no head
+// is missed, no receipt is null. The chain includes the transaction, the node
+// answers correctly, and the tracker is simply slower than the block rate, so
+// the reap evicts the transaction before the block carrying it is opened.
+// Reporting that as expired is a verdict about the chain drawn from a block the
+// run had in hand and had not read.
+func TestABacklogIsNotAChainVerdict(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	// Heads 7 and 8 have arrived. Only 7 has been read.
+	tr.noteHeadReceived(7)
+	tr.noteHeadReceived(8)
+	tr.noteHeadResolved(7)
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a transaction was blamed on the chain while its block sat unread")
+	require.Zero(t, got.Expired)
+}
+
+// TestACaughtUpTrackerStillReportsExpired fails when the backlog rule swallows
+// the chain signal. A run that has read every head it received and still never
+// saw the transaction is entitled to say the chain left it out, and that is the
+// whole point of keeping the two states apart.
+func TestACaughtUpTrackerStillReportsExpired(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	tr.noteHeadReceived(8)
+	tr.noteHeadResolved(8)
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Expired,
+		"a caught-up run could not say the chain left a transaction out")
+	require.Zero(t, got.StatusUnavailable)
+}
+
+// TestAStoppedHeadStreamIsNotAChainVerdict fails when a run whose head stream
+// died keeps reporting chain verdicts. Nothing after that point is read at all.
+func TestAStoppedHeadStreamIsNotAChainVerdict(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	tr.noteHeadReceived(8)
+	tr.noteHeadResolved(8)
+	tr.stopTracking(ctx)
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a run whose head stream died still claimed the chain left a tx out")
+	require.Zero(t, got.Expired)
+}
+
+// TestAHeadIsStampedWhenItArrivesNotWhenItIsRead fails when the arrival stamp is
+// taken at processing time.
+//
+// A tracker whose reads outrun the block interval builds a backlog. Stamping at
+// processing would fold that backlog into InclusionTime and into every latency
+// sample, so a run would report its own lateness as the chain's inclusion
+// latency, and the error would grow for the length of the run.
+func TestAHeadIsStampedWhenItArrivesNotWhenItIsRead(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	tr := newTestTracker(t, time.Minute, 100, NewMockBlockSource())
+
+	headers := make(chan *ethtypes.Header, 4)
+	arrivals := make(chan headArrival, 4)
+	go func() { _ = tr.pumpHeads(ctx, headers, arrivals) }()
+
+	sentAt := time.Now()
+	headers <- &ethtypes.Header{Number: big.NewInt(7)}
+
+	// The reader is busy. A stamp taken here would carry this delay.
+	time.Sleep(300 * time.Millisecond)
+	got := <-arrivals
+
+	require.WithinDuration(t, sentAt, got.arrival, 150*time.Millisecond,
+		"the head was stamped %s after it arrived, so every latency sample "+
+			"carries the tracker's own backlog", got.arrival.Sub(sentAt))
+
+	// And the head counts as received before anything reads its block.
+	for s := range tr.state.Lock() {
+		require.Equal(t, uint64(7), s.headsReceived)
+		require.Zero(t, s.headsResolved)
+	}
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index eaf68b4..340c0e0 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -121,6 +121,13 @@ func narrowReceipts(receipts []*ethtypes.Receipt) (out []blockReceipt, nulls int
 // arrival would add one head-to-head interval to every sample. In the two-node
 // topology this tracker recommends, every height is deferred at least once, so
 // that error would land on every sample rather than a few.
+// headArrival is one head and the moment it reached this process.
+type headArrival struct {
+	num     uint64
+	gasUsed uint64
+	arrival time.Time
+}
+
 type deferredRead struct {
 	num     uint64
 	gasUsed uint64
@@ -182,6 +189,21 @@ type inclusionState struct {
 	// duplicate registrations. It is the part the run cannot attribute to the
 	// chain.
 	statusUnavailable uint64
+	// headsReceived and headsResolved are the highest head this process has taken
+	// off the wire and the highest whose block it has finished reading. They
+	// differ whenever the tracker is behind the chain.
+	//
+	// A reap needs them. Every hole this file records is a read that failed or a
+	// head that never came, and none of that covers the case where every read
+	// succeeds and the tracker is simply late: the block carrying a transaction
+	// is read after the reap already evicted it. Without this the run reports
+	// expired, which is a claim about the chain, for a block it had in hand and
+	// had not opened yet.
+	headsReceived uint64
+	headsResolved uint64
+	// trackingStopped means the head stream ended and no later height will be
+	// read at all.
+	trackingStopped bool
 	// duplicates counts registrations of a hash already in flight. The registry
 	// holds one slot per hash, so the second one has no place to go and the run
 	// can say nothing about it.
@@ -336,11 +358,12 @@ func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoin
 			return err
 		}
 	}
-	return scope.Run(ctx, func(ctx context.Context, s scope.Scope) error {
+	err := scope.Run(ctx, func(ctx context.Context, s scope.Scope) error {
 		client, err := ethclient.Dial(wsEndpoint)
 		if err != nil {
 			return fmt.Errorf("inclusion tracker: connect WebSocket %s: %w", wsEndpoint, err)
 		}
+		defer client.Close()
 		// Buffered: head processing is serial, and a read that takes its whole
 		// budget must not cost a head. go-ethereum buffers far more than this
 		// on its own side and fails the subscription rather than dropping
@@ -356,23 +379,51 @@ func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoin
 			if err != nil {
 				return err
 			}
-			return subErr
+			// Not the run's error. See the sentinel's own doc.
+			return fmt.Errorf("%w: %v", errHeadStreamEnded, subErr)
 		})
 		s.Spawn(func() error { return t.reapLoop(ctx) })
 
+		// One goroutine takes heads off the wire and stamps each with the moment
+		// it arrived; another reads their blocks. Stamping at the point of
+		// processing instead would fold this tracker's own backlog into
+		// InclusionTime, so a run whose reads outrun the block interval would
+		// report its own lateness as the chain's inclusion latency.
+		arrivals := make(chan headArrival, cap(headers))
+		s.SpawnBg(func() error { return t.pumpHeads(ctx, headers, arrivals) })
+
 		var lastSeen uint64 // 0 = unset; first head seeds it (no backfill).
 		// The head loop can end with heights still queued; recordUnreadAtShutdown
 		// owns what happens to them.
 		defer t.recordUnreadAtShutdown(ctx)
 		for ctx.Err() == nil {
-			header, err := utils.Recv(ctx, headers)
+			a, err := utils.Recv(ctx, arrivals)
 			if err != nil {
 				return err
 			}
-			lastSeen = t.processHead(ctx, header.Number.Uint64(), header.GasUsed, time.Now(), lastSeen)
+			// How long this head waited to be read. It is the tracker's own lag,
+			// and it is the number that separates a chain that took nothing from
+			// a run that could not keep up: both report un-included txs, and only
+			// this one says which.
+			inclusionHeadLag.Record(ctx, time.Since(a.arrival).Seconds(),
+				metric.WithAttributes(attribute.String("chain_id", t.seiChainID)))
+			lastSeen = t.processHead(ctx, a.num, a.gasUsed, a.arrival, lastSeen)
+			t.noteHeadResolved(a.num)
 		}
 		return ctx.Err()
 	})
+	// A head stream that ends mid-run stops the tracking, not the run. The
+	// senders, the generator and the report are all still doing their job, and
+	// failing the whole run over a read-only observer's transport would turn a
+	// good run red. Every later height is unread, so nothing after this can reap
+	// as a verdict about the chain.
+	if errors.Is(err, errHeadStreamEnded) {
+		log.Printf("⚠️  inclusion tracker: %v. Tracking stops here; "+
+			"outcomes after this point are status_unavailable", err)
+		t.stopTracking(ctx)
+		return nil
+	}
+	return err
 }
 
 // processHead handles one arriving head: counts any gap (no backfill), matches
@@ -437,6 +488,63 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 	}
 }
 
+// pumpHeads takes heads off the wire and stamps each with the moment it arrived.
+//
+// It is its own step so the stamp cannot drift to the moment of processing. A
+// tracker whose reads outrun the block interval builds a backlog, and a stamp
+// taken at processing would carry that backlog into InclusionTime and report
+// this run's own lateness as the chain's inclusion latency.
+func (t *InclusionTracker) pumpHeads(
+	ctx context.Context, headers <-chan *ethtypes.Header, arrivals chan<- headArrival,
+) error {
+	for ctx.Err() == nil {
+		header, err := utils.Recv(ctx, headers)
+		if err != nil {
+			return err
+		}
+		a := headArrival{
+			num:     header.Number.Uint64(),
+			gasUsed: header.GasUsed,
+			arrival: time.Now(),
+		}
+		t.noteHeadReceived(a.num)
+		if err := utils.Send(ctx, arrivals, a); err != nil {
+			return err
+		}
+	}
+	return ctx.Err()
+}
+
+// noteHeadReceived records that a head reached this process, before anything
+// reads its block.
+func (t *InclusionTracker) noteHeadReceived(num uint64) {
+	for s := range t.state.Lock() {
+		if num > s.headsReceived {
+			s.headsReceived = num
+		}
+	}
+}
+
+// noteHeadResolved records that a head's block has been read.
+func (t *InclusionTracker) noteHeadResolved(num uint64) {
+	for s := range t.state.Lock() {
+		if num > s.headsResolved {
+			s.headsResolved = num
+		}
+	}
+}
+
+// stopTracking records that no further height will be read, so nothing reaped
+// after this point can be called a verdict about the chain.
+func (t *InclusionTracker) stopTracking(ctx context.Context) {
+	for s := range t.state.Lock() {
+		s.trackingStopped = true
+	}
+	inclusionBlockFetchErrors.Add(ctx, 1, metric.WithAttributes(
+		attribute.String("chain_id", t.seiChainID),
+		attribute.String("reason", reasonTrackingStopped)))
+}
+
 // takePending removes every queued height and returns it. Both callers must take
 // and clear in one critical section, so one function owns that.
 func (t *InclusionTracker) takePending() []deferredRead {
@@ -815,10 +923,16 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 			// expired on the surface an operator reads first and as
 			// status_unavailable on the one they read second.
 			outcome := OutcomeExpired
-			if s.blindHeights > e.blindHeightsAtRegistration {
+			switch {
+			case s.blindHeights > e.blindHeightsAtRegistration,
+				s.trackingStopped,
+				s.headsResolved < s.headsReceived:
+				// Either a height went unread, or the tracker is behind and
+				// holds heights it has not opened. A transaction may be sitting
+				// in one of them, so the run cannot say the chain left it out.
 				outcome = OutcomeStatusUnavailable
 				s.statusUnavailable++
-			} else {
+			default:
 				s.expired++
 			}
 			expired = append(expired, resolvedOutcome{outcome: outcome, scenario: e.tx.Scenario})
@@ -885,9 +999,12 @@ const (
 	// differs: a node behind the head is a topology problem, a head the run
 	// never saw is a subscription problem, and a receipt the node could not
 	// produce is a chain problem.
-	reasonBehind      = "receipt_node_behind"
-	reasonMissedHead  = "missed_head"
-	reasonNullReceipt = "null_receipt"
+	reasonBehind = "receipt_node_behind"
+	// reasonTrackingStopped is the head stream ending mid-run. Every later
+	// height is unread, so nothing after it is evidence about the chain.
+	reasonTrackingStopped = "tracking_stopped"
+	reasonMissedHead      = "missed_head"
+	reasonNullReceipt     = "null_receipt"
 )
 
 // readTimeout bounds one first read of a height. A re-read gets whatever is left
@@ -900,6 +1017,13 @@ const (
 // two is what a previous shape got wrong.
 const readTimeout = 10 * time.Second
 
+// errHeadStreamEnded marks the head subscription ending on its own, which stops
+// the tracking and not the run. It is separate from ErrEndpointUnusable because
+// the two want opposite outcomes: an endpoint that cannot serve the run should
+// stop it before it wastes gas, and a transport that drops mid-run should not
+// fail a run whose senders are still working.
+var errHeadStreamEnded = errors.New("head stream ended")
+
 // ErrEndpointUnusable means the receipt endpoint cannot serve this run, and the
 // run stops rather than reporting numbers it did not measure.
 //
diff --git a/stats/metrics.go b/stats/metrics.go
index 8111508..d2e417e 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -84,6 +84,12 @@ var (
 		metric.WithUnit("s"),
 		metric.WithExplicitBucketBoundaries(0.25, 0.5, 1, 2, 5, 10)))
 
+	inclusionHeadLag = must(meter.Float64Histogram(
+		"head_lag",
+		metric.WithDescription("How long a head waited between reaching this process and its block being read. This is the tracker's own lag: un-included txs with this near zero are the chain's doing, and with this climbing they are the run's"),
+		metric.WithUnit("s"),
+		metric.WithExplicitBucketBoundaries(0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30)))
+
 	inclusionEmptyWithGas = must(meter.Int64Counter(
 		"block_empty_with_gas",
 		metric.WithDescription("Blocks that returned no receipts while the head reported gas burned. Causes: non-EVM traffic, an EVM tx that failed the ante handler and got no receipt, or receipts going missing from the store. Only the last is a defect, and this counter cannot tell them apart"),

From 5d7427f8bf7073d2285c15f92bd57ca484f28e52 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 11:25:19 -0700
Subject: [PATCH 13/17] fix(stats): a budget must leave room for the thing it
 budgets

Three correctness findings, all mine from the last two commits, all measured by
the reviewer and reproduced here.

The sweep budget was three seconds and one read's budget was ten, so capping a
re-read at whatever the sweep had left capped every re-read at three. A node
answering in four seconds, well inside its own budget, never finished one: cut
short, requeued, cut short again, then retired as though the node were behind. In
the topology where every height is deferred that is every height in the run, and
expired becomes unreachable. A sweep budget is now not smaller than a read's, and
a test pins that relationship rather than the two numbers.

The same class one layer up, found by a test I wrote for something else. The wait
budget was shorter than a sweep, so a sweep could outlive it and retire its own
tail: heights called receipt_node_behind for time this process spent on the
heights ahead of them. The wait budget is now twice the sweep, and that is pinned
too.

Ordering the requeue by age starved the queue. A sweep walks front to back, so
what it re-deferred got a turn and the tail did not, and putting the tail behind
them meant the same front entries consumed every sweep. Measured: three queued
heights, a node holding only the third, and the third retired without the run
ever completing a read of it. Order here is service fairness, and the earlier
change to age order was a mistake I took from a review without testing what it
cost.

A reap could still claim the chain while a height sat queued. The previous commit
compared heads received against heads read, which a queued height passes: it was
received and read once, the node did not have it, and the run is waiting to ask
again. A transaction may be in it.

Two guards were vacuous. One asserted a reap outcome on a tracker whose reap
window was a minute, so nothing was ever old enough to reap; making it real
showed the defect above. One asserted a sweep's bound with a source whose delay
scaled with the constant under test, so shrinking that constant kept the test
green.

The shutdown sweep now raises the attribution watermark without emitting a
failure. Those heights genuinely went unread, so nothing reaped afterwards may
speak for the chain whatever order shutdown runs in; but a healthy run draining a
queue it was always going to drain is not the run going blind, and the
operator-facing counter should not say it was.

Also: a single missed head went unrecorded, since the gap tests used two and
forty-nine. And the guarantee that the first entry of a sweep gets a full read
was dead code once the budgets were ordered correctly.

Eleven guards proven by breaking what they cover.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/doc.go                    |  11 ++-
 stats/inclusion_outcome_test.go | 165 ++++++++++++++++++++++++--------
 stats/inclusion_tracker.go      |  74 ++++++++++----
 stats/inclusion_tracker_test.go |  53 ++++++++++
 4 files changed, 237 insertions(+), 66 deletions(-)

diff --git a/stats/doc.go b/stats/doc.go
index 550e094..0201176 100644
--- a/stats/doc.go
+++ b/stats/doc.go
@@ -94,7 +94,12 @@
 // partition every accepted transaction.
 //
 // The deferred-read queue's bounds are guarded too:
-// TestOneHeadCannotSpendUnboundedTimeReReading holds one head's sweep inside
-// rereadSweepBudget, and TestAReceiptNodeThatStaysBehindBecomesAHole holds a
-// height inside deferredReadBudget.
+// TestOneHeadIsBoundedAndLosesNoHeight holds one head's sweep inside its budget
+// and proves no queued height leaves unrecorded.
+// TestAHeightOutOfBudgetBecomesAHole is the one that exercises the wait budget;
+// TestAReceiptNodeThatStaysBehindBecomesAHole exercises the queue cap, which is
+// the bound that fires first on a fast chain.
+// TestAWaitBudgetOutlastsASweep pins the two budgets in the order the design
+// needs, since either inverted turns this run's own scheduling into a verdict
+// about the serving node.
 package stats
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 3d98ad2..a95d303 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -586,7 +586,9 @@ func TestADeferredHeightKeepsItsOwnArrival(t *testing.T) {
 func TestAPendingHeightAtShutdownIsNotAChainVerdict(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Minute, 100, src)
+	// Nanosecond, so the reap below actually evicts. A minute here would make
+	// the closing assertion unfailable.
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
 	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
 
 	tx := loadTx(1, time.Unix(1000, 0))
@@ -597,19 +599,24 @@ func TestAPendingHeightAtShutdownIsNotAChainVerdict(t *testing.T) {
 	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
 	tr.recordUnreadAtShutdown(ctx)
 
+	// The watermark rises, because those heights really did go unread and
+	// nothing reaped afterwards may speak for the chain. block_fetch_errors does
+	// not, because a healthy run draining a queue it was always going to drain
+	// is not the run going blind.
 	for s := range tr.state.Lock() {
-		require.Zero(t, s.blindHeights,
-			"a healthy shutdown marked a hole, so the series that answers "+
-				"'was this run blind?' is non-zero on every healthy run")
+		require.Equal(t, uint64(1), s.blindHeights,
+			"an unread height at shutdown left the attribution watermark alone")
 	}
 	require.Equal(t, uint64(1), tr.Summary().InflightAtShutdown,
 		"the transaction was lost rather than counted as in flight")
 
-	// And a reap after that shutdown, were one to happen, still must not call it
-	// a chain verdict.
+	// And a reap after that shutdown still must not call it a chain verdict. The
+	// tracker received a head it never resolved, so it cannot speak for the
+	// chain whatever the queue now holds.
 	tr.reap(ctx)
 	got := tr.collector.GetOperationStats()[key]
 	require.Zero(t, got.Expired, "a pending height became a chain verdict")
+	require.Equal(t, uint64(1), got.StatusUnavailable)
 }
 
 // TestASkippedHeadIsCountedAsAHole fails when a height the run never saw a head
@@ -780,46 +787,62 @@ func TestAGapIsRecordedOnce(t *testing.T) {
 	}
 }
 
-// TestOneHeadCannotSpendUnboundedTimeReReading fails when a sweep gives every
-// waiting height its own read budget.
+// TestOneHeadIsBoundedAndLosesNoHeight fails when a sweep runs unbounded, or
+// when a height it did not reach leaves the queue with nothing recorded.
 //
-// Head processing is serial. A dozen heights can be waiting at once, so a
-// per-read budget would let one hanging node spend minutes inside a single head
-// while the chain moves on.
-func TestOneHeadCannotSpendUnboundedTimeReReading(t *testing.T) {
+// Asserting only that the queue is non-empty is not enough: the heights the
+// sweep did read re-defer themselves and refill it while the ones it dropped go
+// unnoticed.
+func TestOneHeadIsBoundedAndLosesNoHeight(t *testing.T) {
 	ctx := context.Background()
-	// Each read takes most of the whole sweep budget, so the second one starts
-	// with almost none left. A budget that only gates the next read rather than
-	// bounding the one in progress lets that second read run to its own full
-	// length, which is the overshoot this guards.
-	src := NewSlowBlockSource(ethereum.NotFound, rereadSweepBudget*9/10)
+	// Each read takes a fifth of the sweep, so the budget runs out partway down
+	// the queue and the rest must survive. The node holds none of them, so every
+	// read comes back not-found and every height belongs back in the queue.
+	src := NewCountingSlowSource(rereadSweepBudget/5, 1_000_000)
 	tr := newInclusionTrackerWithSource(
 		NewInclusionTracker("test-chain", time.Minute, 100, true, NewCollector()), src)
 
-	// Queue several heights the node has not reached, without paying for a read
-	// per queueing.
 	now := time.Now()
-	for h := uint64(2); h <= 8; h++ {
+	var want []uint64
+	for h := uint64(2); h <= 12; h++ {
 		require.True(t, tr.deferHeight(deferredRead{num: h, arrival: now, deferredAt: now}))
+		want = append(want, h)
 	}
 
 	start := time.Now()
 	tr.rereadDeferred(ctx)
 	spent := time.Since(start)
 
-	require.Less(t, spent, rereadSweepBudget+time.Second,
-		"one head spent %s re-reading; the sweep budget is %s", spent, rereadSweepBudget)
-
-	// Every height is still accounted for. Asserting only that the queue is
-	// non-empty passes while heights vanish, because the ones the sweep did read
-	// re-defer themselves and fill it.
-	require.ElementsMatch(t, []uint64{2, 3, 4, 5, 6, 7, 8}, pendingHeights(tr),
-		"a height the sweep ran out of time for left the queue")
+	// One full read may exceed the remainder, because the first entry is
+	// guaranteed a fair chance. Nothing beyond that.
+	require.Less(t, spent, rereadSweepBudget+readTimeout,
+		"one head spent %s re-reading", spent)
+	require.ElementsMatch(t, want, pendingHeights(tr),
+		"a height the sweep did not reach left the queue")
 	for s := range tr.state.Lock() {
-		require.Zero(t, s.blindHeights, "a height that is still queued was also called a hole")
+		require.Zero(t, s.blindHeights, "a height still queued was also called a hole")
 	}
 }
 
+// TestAWaitBudgetOutlastsASweep fails when a height can be retired for a delay
+// the sweep itself caused.
+//
+// A sweep may spend its whole budget on the heights ahead of one. If the wait
+// budget is the shorter of the two, the tail ages out mid-sweep and the run
+// reports the serving node as behind on heights it was never asked about.
+func TestAWaitBudgetOutlastsASweep(t *testing.T) {
+	require.Greater(t, deferredReadBudget, rereadSweepBudget,
+		"a height can age out inside one sweep, so the run would call the node "+
+			"behind for time this process spent elsewhere")
+
+	// And a sweep leaves room for the read it is budgeting for. Below this a
+	// re-read is capped under one read's budget and can never clear it, so a
+	// node answering just above the cap loses every height: in the topology
+	// where every height is deferred, that is every height in the run.
+	require.GreaterOrEqual(t, rereadSweepBudget, readTimeout,
+		"every re-read is capped below one read's budget")
+}
+
 // pendingHeights reads the queued heights, for a test that has to name them all.
 func pendingHeights(tr *InclusionTracker) []uint64 {
 	for s := range tr.state.Lock() {
@@ -832,25 +855,29 @@ func pendingHeights(tr *InclusionTracker) []uint64 {
 	panic("unreachable")
 }
 
-// TestTheOldestQueuedHeightIsReadFirst fails when a sweep puts the heights it
-// did not reach ahead of the ones it re-deferred. A sweep walks oldest first, so
-// what it re-deferred is older, and prepending would serve the newest first and
-// let the oldest age out against a budget it was never given a turn under.
-func TestTheOldestQueuedHeightIsReadFirst(t *testing.T) {
+// TestAnUnreachedHeightGoesToTheFrontOfTheQueue fails when a sweep puts the
+// heights it did not reach behind the ones it read.
+//
+// A sweep walks front to back, so what it re-deferred got a turn and the tail
+// did not. Putting the tail last means the same front entries consume every
+// sweep, and a height further back is never served at all: measured with three
+// queued heights and a node holding only the third, the third was retired as
+// though the node were behind on it without the run ever completing a read of
+// it. Order here is service fairness, not age.
+func TestAnUnreachedHeightGoesToTheFrontOfTheQueue(t *testing.T) {
 	ctx := context.Background()
-	src := NewSlowBlockSource(ethereum.NotFound, rereadSweepBudget/2)
-	tr := newInclusionTrackerWithSource(
-		NewInclusionTracker("test-chain", time.Minute, 100, true, NewCollector()), src)
+	tr := newTestTracker(t, time.Minute, 100, NewMockBlockSource())
 
 	now := time.Now()
-	for h := uint64(2); h <= 8; h++ {
+	// Two heights the sweep already read and re-deferred.
+	for _, h := range []uint64{10, 11} {
 		require.True(t, tr.deferHeight(deferredRead{num: h, arrival: now, deferredAt: now}))
 	}
-	tr.rereadDeferred(ctx)
+	// One it never reached.
+	tr.requeue(ctx, []deferredRead{{num: 12, arrival: now, deferredAt: now}})
 
-	got := pendingHeights(tr)
-	require.Equal(t, []uint64{2, 3, 4, 5, 6, 7, 8}, got,
-		"the queue came back out of age order as %v", got)
+	require.Equal(t, []uint64{12, 10, 11}, pendingHeights(tr),
+		"the height that got no turn was put behind the ones that did")
 }
 
 // TestAHeightOutOfBudgetBecomesAHole fails when the wait budget never retires a
@@ -1046,3 +1073,57 @@ func TestAHeadIsStampedWhenItArrivesNotWhenItIsRead(t *testing.T) {
 		require.Zero(t, s.headsResolved)
 	}
 }
+
+// TestAQueuedHeightBlocksAChainVerdict fails when a reap calls the chain while
+// the run is still waiting on a height it queued.
+//
+// It is the same claim as a backlog in the head channel, one step further along:
+// the height was received and read once, the node did not have it, and the run
+// is waiting to ask again. A transaction may be in it.
+func TestAQueuedHeightBlocksAChainVerdict(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	// Everything received has been read, so only the queue stands between the
+	// run and a chain verdict.
+	tr.noteHeadReceived(8)
+	tr.noteHeadResolved(8)
+	now := time.Now()
+	require.True(t, tr.deferHeight(deferredRead{num: 8, arrival: now, deferredAt: now}))
+
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a transaction was blamed on the chain while a height sat queued to be read")
+	require.Zero(t, got.Expired)
+}
+
+// TestASingleMissedHeadIsCountedAsAHole fails when a gap of exactly one height
+// goes unrecorded. One missed head is the common case, and its transactions
+// would reap as expired for a block nothing read.
+func TestASingleMissedHeadIsCountedAsAHole(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	last := tr.processHead(ctx, 10, 0, time.Unix(1002, 0), 0)
+	tr.processHead(ctx, 12, 0, time.Unix(1003, 0), last) // 11 alone was missed
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"one missed head went unrecorded, so its txs were blamed on the chain")
+	require.Zero(t, got.Expired)
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 340c0e0..03b5661 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -157,12 +157,26 @@ const (
 // hole. A duration is the quantity that actually matters, which is how far the
 // receipt node trails the head node.
 const (
-	deferredReadBudget = 5 * time.Second
+	// deferredReadBudget is how long a height may wait before the run calls the
+	// serving node behind.
+	//
+	// It is larger than rereadSweepBudget, and that relationship is load-bearing
+	// rather than a coincidence of two tuned numbers. A sweep can spend its whole
+	// budget on the heights ahead of one, so a wait budget shorter than a sweep
+	// retires the tail for a delay this process caused and reports it as the
+	// node being behind. Anything queued gets at least one sweep after the one
+	// it was queued in.
+	deferredReadBudget = 2 * rereadSweepBudget
 	maxDeferredReads   = 64
 	// rereadSweepBudget bounds what one head spends re-reading, whatever is
-	// waiting. Head processing is serial, so this and one first read are what a
-	// head costs at worst.
-	rereadSweepBudget = 3 * time.Second
+	// waiting.
+	//
+	// It is not smaller than readTimeout, and that is load-bearing. A sweep
+	// budget below one read's budget caps every re-read below it, so a node
+	// answering in four seconds never finishes one, and in the topology where
+	// every height is deferred that makes every height a hole. The budget must
+	// leave room for the read it is budgeting for.
+	rereadSweepBudget = readTimeout
 )
 
 type entry struct {
@@ -470,6 +484,11 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 		// The deadline bounds the read, not just the decision to start one.
 		// Checking the clock before each read and then letting that read run its
 		// own full budget is not a bound: it is the budget plus one whole read.
+		//
+		// The first entry always gets a full read, because the sweep budget is
+		// not smaller than one read's. That is what guarantees progress: every
+		// sweep finishes at least one height, so a queue whose front entries are
+		// slow cannot spend every sweep on partial reads and finish none.
 		left := time.Until(sweepUntil)
 		if left <= 0 {
 			t.requeue(ctx, due[i:])
@@ -556,12 +575,16 @@ func (t *InclusionTracker) takePending() []deferredRead {
 	panic("unreachable")
 }
 
-// requeue puts back the heights a sweep ran out of time for.
+// requeue puts back the heights a sweep ran out of time for, ahead of what the
+// sweep already read.
 //
-// They go behind what the sweep already re-deferred, not in front of it. A sweep
-// walks oldest first, so anything it re-deferred is older than anything it did
-// not reach, and prepending would put the newest heights at the head of the
-// queue and serve the oldest last.
+// Order here is service fairness, not age. A sweep walks front to back, so the
+// entries it re-deferred are exactly the ones that got a turn and the tail is
+// exactly the ones that did not. Putting the tail behind them means the same
+// front entries consume every sweep and a height further back is never asked
+// for at all, then retires on its wait budget as though the node were behind on
+// it. Measured: with three queued heights and a node holding only the third, the
+// third was retired without the run ever issuing a request for it.
 //
 // A height past the cap is recorded rather than dropped. deferHeight already
 // holds the queue at the cap, so nothing reaches that branch today; a height
@@ -571,7 +594,7 @@ func (t *InclusionTracker) takePending() []deferredRead {
 func (t *InclusionTracker) requeue(ctx context.Context, left []deferredRead) {
 	var dropped []deferredRead
 	for s := range t.state.Lock() {
-		s.pending = append(s.pending, left...)
+		s.pending = append(left, s.pending...)
 		if len(s.pending) > maxDeferredReads {
 			dropped = s.pending[maxDeferredReads:]
 			s.pending = s.pending[:maxDeferredReads]
@@ -584,14 +607,21 @@ func (t *InclusionTracker) requeue(ctx context.Context, left []deferredRead) {
 
 // recordUnreadAtShutdown records that the run ended with heights it never read.
 //
-// It does not mark a hole. The head loop only ends when the run's context is
-// done, and the reap loop ends on the same signal, so no reap follows this and
-// the transactions in those heights are counted as in flight at shutdown, which
-// is already not a verdict about the chain. Marking a hole here would put a
-// failure on the series that answers "was this run blind?" at the end of every
-// healthy run.
+// It raises the watermark and emits no failure, and the split is deliberate. The
+// watermark is internal and decides attribution: these heights went unread, so
+// nothing reaped afterwards may be called a verdict about the chain, whatever
+// order the shutdown happens to run in. block_fetch_errors is operator-facing
+// and answers "did this run go blind?", and a healthy run draining a queue it
+// was always going to drain is not that.
 func (t *InclusionTracker) recordUnreadAtShutdown(ctx context.Context) {
-	for _, d := range t.takePending() {
+	due := t.takePending()
+	if len(due) == 0 {
+		return
+	}
+	for s := range t.state.Lock() {
+		s.blindHeights++
+	}
+	for _, d := range due {
 		inclusionDeferredWait.Record(ctx, time.Since(d.deferredAt).Seconds(),
 			metric.WithAttributes(
 				attribute.String("chain_id", t.seiChainID),
@@ -926,10 +956,12 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 			switch {
 			case s.blindHeights > e.blindHeightsAtRegistration,
 				s.trackingStopped,
-				s.headsResolved < s.headsReceived:
-				// Either a height went unread, or the tracker is behind and
-				// holds heights it has not opened. A transaction may be sitting
-				// in one of them, so the run cannot say the chain left it out.
+				s.headsResolved < s.headsReceived,
+				len(s.pending) > 0:
+				// Either a height went unread, or the run is holding one it has
+				// not opened: still in the head queue, or waiting on a receipt
+				// node that has not reached it. A transaction may be sitting in
+				// any of them, so the run cannot say the chain left it out.
 				outcome = OutcomeStatusUnavailable
 				s.statusUnavailable++
 			default:
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index 89be2bb..ab3e305 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -2,6 +2,7 @@ package stats
 
 import (
 	"context"
+	"github.com/ethereum/go-ethereum"
 	"math/big"
 	"sync"
 	"sync/atomic"
@@ -398,6 +399,58 @@ func TestInclusion_ConcurrentRaceSafe(t *testing.T) {
 		"conservation holds under concurrency")
 }
 
+// CountingSlowSource records which heights were asked for, and answers each
+// after a delay. heldFrom names the first height it actually holds; anything
+// below that answers not-found, so a test can drive a node whose availability
+// is not monotone with the queue's age order.
+type CountingSlowSource struct {
+	mu       sync.Mutex
+	delay    time.Duration
+	heldFrom uint64
+	asked    map[uint64]int
+	receipts map[uint64][]blockReceipt
+}
+
+func NewCountingSlowSource(delay time.Duration, heldFrom uint64) *CountingSlowSource {
+	return &CountingSlowSource{
+		delay: delay, heldFrom: heldFrom,
+		asked: map[uint64]int{}, receipts: map[uint64][]blockReceipt{},
+	}
+}
+
+func (c *CountingSlowSource) Hold(n uint64, rs ...blockReceipt) *CountingSlowSource {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.receipts[n] = rs
+	return c
+}
+
+func (c *CountingSlowSource) Asked() map[uint64]int {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	out := make(map[uint64]int, len(c.asked))
+	for k, v := range c.asked {
+		out[k] = v
+	}
+	return out
+}
+
+func (c *CountingSlowSource) BlockReceipts(ctx context.Context, n uint64) ([]blockReceipt, int, error) {
+	c.mu.Lock()
+	c.asked[n]++
+	delay, held, rs := c.delay, n >= c.heldFrom, c.receipts[n]
+	c.mu.Unlock()
+	select {
+	case <-time.After(delay):
+	case <-ctx.Done():
+		return nil, 0, ctx.Err()
+	}
+	if !held {
+		return nil, 0, ethereum.NotFound
+	}
+	return rs, 0, nil
+}
+
 // SlowBlockSource fails every read after a delay, so a test can measure what a
 // hanging endpoint costs a head.
 type SlowBlockSource struct {

From 0b2a408572db17c56326ec70fef0b5ec04e3f6fd Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 12:51:29 -0700
Subject: [PATCH 14/17] fix(stats): take the head signal from the node the run
 reads status from

A traceability review, the first anyone has run on this work, and it found what
six rounds of correctness review could not: the code is careful and it does not
match its specification.

TOT-022 is a MUST. The run takes its head signal from the same node it reads
status from, because a head carries the raw committed height while a status read
resolves through a watermark behind it. The two disagree even on one node, and
taking them from separate nodes adds peer lag on top, which a node inside its
readiness threshold carries for minutes without reporting unhealthy. The
requirement's own verification row names the failure as "the head signal and the
status read come from different nodes". That is what this code did, with a
comment justifying it on cost grounds, answering a question the requirement did
not ask.

The cost of that violation is most of the machinery built since. The deferred
read queue exists because heights arrived before the reading node held them,
which is the condition TOT-022 forbids creating. It stays for now, because one
node still disagrees with itself across the watermark, but it should be rare
rather than constant.

And it had made expired unreachable a second time. The reap refuses a chain
verdict while a height sits queued, which is right, and it is only safe because
the queue is normally empty. Split the nodes and the queue never empties, so a
run against a healthy chain that took nothing reports "I could not see". That is
the defect of two rounds ago, reintroduced through the topology this change
recommended. A test now drives the single-node steady state and asserts expired
is reachable.

TOT-023 says the run must not read a block older than the deadline it gives a
transaction to reach one. The read budget was a fixed twenty seconds while the
deadline is operator-configurable, so a run reaping at five seconds re-read
heights four times past its own bound. The budget answers to the deadline now.

The traceability mechanism was absent rather than incomplete. One requirement ID
appeared in the whole test suite, in a comment, from the previous PR, while the
repo already does this properly for another feature. Forty-two guards now name
what they cover. Eight requirements are cited by nothing, and that is the point
of doing it: TOT-005, 007, 012 and 019 are the report, TOT-010, 014 and 018 are
the hand-off channel, and all seven belong to later phases. TOT-013 is
contradicted rather than deferred, and it is called out below.

Also: sender/doc.go stated the identity with six terms where the data model has
seven; the README advertised a report carrying committed and reverted, which it
does not; and the duplicate-registration collision is filed under
status_unavailable, which is the closest state the spec defines and not a clean
fit, now said out loud where it happens.

Open against the spec and not fixed here: the preflight refuses a run, and the
spec's design section says nothing in this feature fails a run. Refusing is
probably better than a silently blind run, but it is an unrecorded amendment,
and it makes TOT-013 and SC-008 unreachable by construction.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 README.md                       |   7 ++-
 config/config.go                |   4 +-
 main.go                         |  24 ++++----
 sender/doc.go                   |   7 ++-
 stats/inclusion_outcome_test.go | 101 +++++++++++++++++++++++++++++++-
 stats/inclusion_tracker.go      |  53 +++++++++++++----
 6 files changed, 164 insertions(+), 32 deletions(-)

diff --git a/README.md b/README.md
index 7d3dc46..653be04 100644
--- a/README.md
+++ b/README.md
@@ -40,7 +40,10 @@ Edit `my-config.json`:
 ```
 
 `endpoints` take the load. `receiptEndpoint` is the node the inclusion tracker
-reads receipts from, and it should be a node taking no send load. On seid, a
+uses, and it should be a node taking no send load. The tracker takes both its
+head subscription and its receipt reads from that one node: a head carries the
+raw committed height while a status read resolves through a watermark behind it,
+so heads from one node and reads from another disagree by that pair's peer lag. On seid, a
 receipts read costs the serving node work that grows with the square of the
 block's transaction count, so a node doing both degrades, and the tracker
 reports that degradation as a chain result. Leave it out and the tracker reads
@@ -67,7 +70,7 @@ than the block interval makes the tracker blind, and a blind tracker reports
 | `--buffer-size, -b` | 1000 | Sender queue size |
 | `--dry-run` | false | Simulate without sending |
 | `--debug` | false | Log each transaction |
-| `--track-receipts` | false | Read each block's receipts and report what the chain did with every transaction: committed, reverted, expired, status-unavailable, dropped-at-cap, or still in flight. Set `receiptEndpoint` in the config with it |
+| `--track-receipts` | false | Read each block's receipts and record what the chain did with every transaction: committed, reverted, expired, status-unavailable, dropped-at-cap, or still in flight. The counts reach the `inclusion_outcome` metric; the closing log line carries the inclusion tally. The run report does not carry them yet. Set `receiptEndpoint` in the config with it |
 | `--inclusion-reap-after` | 30s | How long an un-included tx waits before reaping as expired (tune to expected inclusion time on congested chains) |
 | `--track-blocks` | false | Track block statistics |
 | `--track-user-latency` | false | Track user latency metrics |
diff --git a/config/config.go b/config/config.go
index e6121b2..59c3a7d 100644
--- a/config/config.go
+++ b/config/config.go
@@ -33,7 +33,9 @@ type LoadConfig struct {
 	// SeiChainID is the textual chain ID used for tagging metric collection.
 	SeiChainID string   `json:"seiChainID,omitempty"`
 	Endpoints  []string `json:"endpoints"`
-	// ReceiptEndpoint is the node the inclusion tracker reads receipts from.
+	// ReceiptEndpoint is the node the inclusion tracker uses. It takes both the
+	// head subscription and the receipt reads from it, because a head from one
+	// node and a status read from another disagree by that pair's peer lag.
 	// Empty uses Endpoints[0].
 	//
 	// Name a node that takes no send load. On seid, eth_getBlockReceipts costs
diff --git a/main.go b/main.go
index 99f749e..6dfe72e 100644
--- a/main.go
+++ b/main.go
@@ -294,19 +294,23 @@ func runLoadTest(ctx context.Context, cmd *cobra.Command) error {
 				collector,
 			)
 			inclusion = utils.Some(inclusionTracker)
-			// Heads come from the load endpoint, which is already subscribed
-			// and cheap to read. Receipts come from receiptEndpoint when the
-			// run names one, so the read work stays off the box under load.
-			receiptEndpoint := cfg.ReceiptEndpoint
-			if receiptEndpoint == "" {
-				receiptEndpoint = cfg.Endpoints[0]
-				log.Printf("⚠️  Reading receipts from the load endpoint %s. "+
-					"Set receiptEndpoint to a node taking no send load: a receipts "+
+			// Heads and receipts come from the same node, always. A head
+			// notification carries the raw committed height while a status read
+			// resolves through a watermark behind it, so the two disagree even on
+			// one node; taking them from different nodes adds peer lag on top,
+			// and a node trailing its peers by less than the readiness threshold
+			// stays in service while it does. Reading both from one node makes
+			// the run internally consistent, which is what TOT-022 requires.
+			trackingEndpoint := cfg.ReceiptEndpoint
+			if trackingEndpoint == "" {
+				trackingEndpoint = cfg.Endpoints[0]
+				log.Printf("⚠️  Tracking from the load endpoint %s. Set "+
+					"receiptEndpoint to a node taking no send load: a receipts "+
 					"read costs the serving node work that grows with the block's "+
-					"transaction count.", receiptEndpoint)
+					"transaction count.", trackingEndpoint)
 			}
 			s.SpawnBgNamed("inclusion tracker", func() error {
-				return inclusionTracker.Run(ctx, cfg.Endpoints[0], receiptEndpoint)
+				return inclusionTracker.Run(ctx, trackingEndpoint)
 			})
 		}
 
diff --git a/sender/doc.go b/sender/doc.go
index 28c5218..bfdb8cc 100644
--- a/sender/doc.go
+++ b/sender/doc.go
@@ -102,13 +102,14 @@
 // Conservation. Over [stats.Outcome]'s terminal states,
 //
 //	accepted == committed + reverted + status_unavailable
-//	          + expired + dropped_at_cap + inflight_at_shutdown
+//	          + expired + dropped_at_cap + dropped_at_handoff
+//	          + inflight_at_shutdown
 //
 // and accepted ⊆ succeeded (only a successful send is accepted). The left side
 // is accepted rather than registered, because a tx refused at the cap is
 // counted by dropped_at_cap and never entered the registry.
-// dropped_at_handoff joins the identity with the hand-off channel; nothing
-// produces it yet. The inclusion
+// dropped_at_handoff is a term of the identity and nothing produces it yet; the
+// hand-off channel is what will. The inclusion
 // denominator is succeeded (txs_accepted), never a minted "registered" series;
 // dropped_at_cap txs are excluded from it. inflight_at_shutdown is read only
 // after both the senders and the tracker have joined.
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index a95d303..82e0a32 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -20,6 +20,7 @@ import (
 
 // TestAllRevertedReceiptsYieldZeroCommitted fails when a reverted transaction and a
 // committed one land in the same terminal state.
+// Requirements: TOT-001 and SC-001.
 func TestAllRevertedReceiptsYieldZeroCommitted(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
@@ -48,6 +49,7 @@ func TestAllRevertedReceiptsYieldZeroCommitted(t *testing.T) {
 
 // TestReceiptsSeparateCommittedFromReverted covers the mixed block, which is the
 // shape a real run produces. A block carrying both must split them.
+// Requirements: TOT-001 and TOT-002.
 func TestReceiptsSeparateCommittedFromReverted(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
@@ -76,6 +78,7 @@ func TestReceiptsSeparateCommittedFromReverted(t *testing.T) {
 // TestOutcomesCarryTheOperation fails when the tracker labels an outcome by
 // scenario alone. Two operations in one scenario would then share a count, and
 // neither the report nor a dashboard could say which one degraded.
+// Requirements: TOT-017 and TOT-006.
 func TestOutcomesCarryTheOperation(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
@@ -106,6 +109,7 @@ func TestOutcomesCarryTheOperation(t *testing.T) {
 // It measures requests this process issues. It says nothing about what one
 // request costs the node that answers it, which is a separate constraint and a
 // separate measurement.
+// Requirements: TOT-009, TOT-015 and SC-004.
 func TestRequestsPerBlockDoNotTrackVolume(t *testing.T) {
 	for _, volume := range []int{1, 50, 500} {
 		t.Run(strconv.Itoa(volume), func(t *testing.T) {
@@ -135,6 +139,7 @@ func TestRequestsPerBlockDoNotTrackVolume(t *testing.T) {
 // TestReapedTransactionsReachTheCollector covers the path that does not go
 // through a receipt. A transaction nothing named before the deadline is expired,
 // and the collector has to hear about it or the states stop partitioning.
+// Requirements: TOT-016.
 func TestReapedTransactionsReachTheCollector(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Nanosecond, 100, src)
@@ -154,6 +159,7 @@ func TestReapedTransactionsReachTheCollector(t *testing.T) {
 // TestUnreadableBlockIsNotAChainVerdict fails when a block the run could not
 // read is reported as a chain that left the transaction out. Expired is a claim
 // about the chain; a read that failed supports no such claim.
+// Requirements: TOT-004, TOT-020 and SC-011.
 func TestUnreadableBlockIsNotAChainVerdict(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Nanosecond, 100, src)
@@ -180,6 +186,7 @@ func TestUnreadableBlockIsNotAChainVerdict(t *testing.T) {
 // mainnet blocks return an empty array. Counting those as holes makes expired
 // unreachable, and a chain that stopped accepting work produces nothing but
 // idle blocks, which is the one run where expired is the answer.
+// Requirements: TOT-020.
 func TestAnIdleBlockIsNotAHole(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Nanosecond, 100, src)
@@ -203,6 +210,7 @@ func TestAnIdleBlockIsNotAHole(t *testing.T) {
 // transactions sharing one hash report one terminal state between them. The
 // registry holds one slot per hash, and a resend after a send timeout produces
 // exactly that collision.
+// Requirements: TOT-003.
 func TestDuplicateRegistrationReachesATerminalState(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
@@ -228,6 +236,7 @@ func TestDuplicateRegistrationReachesATerminalState(t *testing.T) {
 // TestTransactionsRegisteredAfterTheHoleStillExpire fails when one unreadable
 // block turns every later transaction into status_unavailable. The hole covers
 // the transactions in flight across it, and nothing after.
+// Requirements: TOT-020 and SC-011.
 func TestTransactionsRegisteredAfterTheHoleStillExpire(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Nanosecond, 100, src)
@@ -250,6 +259,7 @@ func TestTransactionsRegisteredAfterTheHoleStillExpire(t *testing.T) {
 // TestReceiptWithoutStatusIsNotAFailure fails when a receipt that carries a
 // post-state root instead of a status is read as an execution failure. It says
 // the transaction executed; it does not say how it ended.
+// Requirements: TOT-004.
 func TestReceiptWithoutStatusIsNotAFailure(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
@@ -277,6 +287,7 @@ func TestReceiptWithoutStatusIsNotAFailure(t *testing.T) {
 // Every reachable terminal state has a leg here. Leaving the reap out was worth
 // a mutation test on its own, because dropping the whole reap report loop left
 // a partition guard that still passed.
+// Requirements: TOT-003 and SC-007.
 func TestOutcomesPartitionEveryAcceptedTx(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -345,6 +356,7 @@ func TestOutcomesPartitionEveryAcceptedTx(t *testing.T) {
 // a node that serves no receipts starts anyway. Such a run completes, reports
 // every transaction un-included, and exits zero, which reads as a chain that
 // accepted nothing.
+// Requirements: TOT-021.
 func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 	cases := []struct {
 		name    string
@@ -386,6 +398,7 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 // not name. It is not a hole for the ones it did: discarding a receipt the
 // endpoint actually sent would throw away a known outcome to describe an
 // unknown one.
+// Requirements: TOT-020.
 func TestNilReceiptDoesNotEndTheRun(t *testing.T) {
 	hash := loadTx(1, time.Unix(1000, 0)).EthTx.Hash()
 
@@ -408,6 +421,7 @@ func TestNilReceiptDoesNotEndTheRun(t *testing.T) {
 // TestAPartlyNullReadMatchesWhatArrivedAndCountsTheRest fails when a read that
 // was partly unreadable either throws away the part that arrived or hides the
 // part that did not.
+// Requirements: TOT-020.
 func TestAPartlyNullReadMatchesWhatArrivedAndCountsTheRest(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -443,6 +457,7 @@ func TestAPartlyNullReadMatchesWhatArrivedAndCountsTheRest(t *testing.T) {
 // arrives from one node and the other has not committed that height yet.
 // Counting it would mark a hole per block, which is how expired became
 // unreachable before.
+// Requirements: TOT-020.
 func TestALaggingReceiptNodeIsRetriedNotCountedAsAHole(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -474,6 +489,7 @@ func TestALaggingReceiptNodeIsRetriedNotCountedAsAHole(t *testing.T) {
 // TestAReceiptNodeThatStaysBehindBecomesAHole fails when a node that never
 // catches up is retried forever. One re-read is the bound: a node still behind
 // a block interval later is behind rather than busy.
+// Requirements: TOT-020.
 func TestAReceiptNodeThatStaysBehindBecomesAHole(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -502,6 +518,7 @@ func TestAReceiptNodeThatStaysBehindBecomesAHole(t *testing.T) {
 //
 // The buckets decide whether a run is refused, so a false refusal on a busy
 // node is worse than the blind run the refusal exists to prevent.
+// Requirements: TOT-021.
 func TestFetchFailureReasonReadsTypedErrors(t *testing.T) {
 	cases := []struct {
 		name string
@@ -549,6 +566,7 @@ func TestFetchFailureReasonReadsTypedErrors(t *testing.T) {
 // Arrival becomes the inclusion latency sample. In the two-node topology this
 // tracker recommends, every height is deferred at least once, so the error
 // would land on every sample rather than a few.
+// Requirements: TOT-022.
 func TestADeferredHeightKeepsItsOwnArrival(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -583,6 +601,7 @@ func TestADeferredHeightKeepsItsOwnArrival(t *testing.T) {
 // which is already not a claim about the chain. Marking a hole would put a
 // failure on the series that answers "was this run blind?" at the end of every
 // healthy run in the topology this tracker recommends.
+// Requirements: TOT-003 and TOT-004.
 func TestAPendingHeightAtShutdownIsNotAChainVerdict(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -622,6 +641,7 @@ func TestAPendingHeightAtShutdownIsNotAChainVerdict(t *testing.T) {
 // TestASkippedHeadIsCountedAsAHole fails when a height the run never saw a head
 // for goes uncounted. Its transactions would reap as expired, which is a claim
 // about the chain for a block nothing read.
+// Requirements: TOT-004.
 func TestASkippedHeadIsCountedAsAHole(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -644,6 +664,7 @@ func TestASkippedHeadIsCountedAsAHole(t *testing.T) {
 
 // TestTheDuplicateLegReachesBothLedgers fails when a duplicate registration is
 // counted on one operator-facing surface and not the other.
+// Requirements: TOT-016.
 func TestTheDuplicateLegReachesBothLedgers(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -663,6 +684,7 @@ func TestTheDuplicateLegReachesBothLedgers(t *testing.T) {
 
 // TestTheDeferredQueueDrains fails when a height stays queued after it is read,
 // which would make every later head re-read every height the run ever deferred.
+// Requirements: TOT-011.
 func TestTheDeferredQueueDrains(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -688,6 +710,7 @@ func TestTheDeferredQueueDrains(t *testing.T) {
 // normally and exits zero. A refusal that wraps one is therefore a Complete run
 // that carried no load, which anything downstream reads as a chain that included
 // nothing: the false accusation this whole tracker exists to remove.
+// Requirements: TOT-021.
 func TestARefusedRunDoesNotLookLikeAFinishedOne(t *testing.T) {
 	for _, tc := range []struct {
 		name string
@@ -712,6 +735,7 @@ func TestARefusedRunDoesNotLookLikeAFinishedOne(t *testing.T) {
 
 // TestPreflightNeedsEveryAttemptToAgree fails when one attempt's verdict speaks
 // for the run. An endpoint that answers at all, even with an error, is there.
+// Requirements: TOT-021.
 func TestPreflightNeedsEveryAttemptToAgree(t *testing.T) {
 	src := NewMockBlockSource()
 	tr := newTestTracker(t, time.Minute, 100, src)
@@ -741,6 +765,7 @@ func TestPreflightNeedsEveryAttemptToAgree(t *testing.T) {
 
 // TestATemporaryResolverFailureDoesNotRefuseTheRun fails when a resolver blip
 // ends a run. Only a name that does not exist is permanent.
+// Requirements: TOT-021.
 func TestATemporaryResolverFailureDoesNotRefuseTheRun(t *testing.T) {
 	servfail := &net.DNSError{Err: "server misbehaving", Name: "rpc-0", IsTemporary: true}
 	require.Equal(t, reasonOther, fetchFailureReason(servfail))
@@ -752,6 +777,7 @@ func TestATemporaryResolverFailureDoesNotRefuseTheRun(t *testing.T) {
 // TestTheSummaryTermsAreDisjoint fails when one transaction lands in two terms
 // of the closing log line, which invites an operator to add them up and get more
 // than the run accepted.
+// Requirements: TOT-003 and SC-010.
 func TestTheSummaryTermsAreDisjoint(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -773,6 +799,7 @@ func TestTheSummaryTermsAreDisjoint(t *testing.T) {
 // TestAGapIsRecordedOnce fails when a missed-head range logs and counts per
 // height. A long gap would then push the run's own summary out of any bounded log
 // tail, which is the only diagnostic a failed run carries.
+// Requirements: TOT-004.
 func TestAGapIsRecordedOnce(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -793,6 +820,7 @@ func TestAGapIsRecordedOnce(t *testing.T) {
 // Asserting only that the queue is non-empty is not enough: the heights the
 // sweep did read re-defer themselves and refill it while the ones it dropped go
 // unnoticed.
+// Requirements: TOT-011 and TOT-020.
 func TestOneHeadIsBoundedAndLosesNoHeight(t *testing.T) {
 	ctx := context.Background()
 	// Each read takes a fifth of the sweep, so the budget runs out partway down
@@ -830,8 +858,9 @@ func TestOneHeadIsBoundedAndLosesNoHeight(t *testing.T) {
 // A sweep may spend its whole budget on the heights ahead of one. If the wait
 // budget is the shorter of the two, the tail ages out mid-sweep and the run
 // reports the serving node as behind on heights it was never asked about.
+// Requirements: TOT-020 and TOT-023.
 func TestAWaitBudgetOutlastsASweep(t *testing.T) {
-	require.Greater(t, deferredReadBudget, rereadSweepBudget,
+	require.Greater(t, maxDeferredReadBudget, rereadSweepBudget,
 		"a height can age out inside one sweep, so the run would call the node "+
 			"behind for time this process spent elsewhere")
 
@@ -864,6 +893,7 @@ func pendingHeights(tr *InclusionTracker) []uint64 {
 // queued heights and a node holding only the third, the third was retired as
 // though the node were behind on it without the run ever completing a read of
 // it. Order here is service fairness, not age.
+// Requirements: TOT-020.
 func TestAnUnreachedHeightGoesToTheFrontOfTheQueue(t *testing.T) {
 	ctx := context.Background()
 	tr := newTestTracker(t, time.Minute, 100, NewMockBlockSource())
@@ -883,13 +913,14 @@ func TestAnUnreachedHeightGoesToTheFrontOfTheQueue(t *testing.T) {
 // TestAHeightOutOfBudgetBecomesAHole fails when the wait budget never retires a
 // height. The budget is the whole reason the bound is a duration rather than a
 // count of reads, and the queue cap must not be what fires instead.
+// Requirements: TOT-020 and TOT-023.
 func TestAHeightOutOfBudgetBecomesAHole(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource().SetFetchErr(ethereum.NotFound)
 	tr := newTestTracker(t, time.Minute, 100, src)
 
 	// One height, so the queue cap cannot be what retires it.
-	old := time.Now().Add(-deferredReadBudget - time.Second)
+	old := time.Now().Add(-tr.deferredReadBudget() - time.Second)
 	require.True(t, tr.deferHeight(deferredRead{num: 7, arrival: old, deferredAt: old}))
 	tr.rereadDeferred(ctx)
 
@@ -904,12 +935,13 @@ func TestAHeightOutOfBudgetBecomesAHole(t *testing.T) {
 // clock. The budget measures how far the receipt node trails the head node, and
 // restarting it per read would mean a node behind forever is never called
 // behind.
+// Requirements: TOT-023.
 func TestTheWaitBudgetDoesNotRestartOnEachRead(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource().SetFetchErr(ethereum.NotFound)
 	tr := newTestTracker(t, time.Minute, 100, src)
 
-	first := time.Now().Add(-deferredReadBudget / 2)
+	first := time.Now().Add(-tr.deferredReadBudget() / 2)
 	require.True(t, tr.deferHeight(deferredRead{num: 7, arrival: first, deferredAt: first}))
 	tr.rereadDeferred(ctx) // re-read fails, re-defers
 
@@ -927,6 +959,7 @@ func TestTheWaitBudgetDoesNotRestartOnEachRead(t *testing.T) {
 // TestADuplicateHeadStillDrainsTheQueue fails when a repeated or out-of-order
 // head returns before the sweep. Nothing else drains the queue, so a height left
 // in it reaps as a chain verdict for a block never read.
+// Requirements: TOT-020.
 func TestADuplicateHeadStillDrainsTheQueue(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -953,6 +986,7 @@ func TestADuplicateHeadStillDrainsTheQueue(t *testing.T) {
 // TestAnAnswerAfterATimeoutDoesNotRefuseTheRun fails when one timed-out attempt
 // decides the run. A later attempt that answers at all proves the endpoint is
 // there, whatever it said.
+// Requirements: TOT-021.
 func TestAnAnswerAfterATimeoutDoesNotRefuseTheRun(t *testing.T) {
 	src := NewMockBlockSource().SetErrSequence(
 		context.DeadlineExceeded, ethereum.NotFound, ethereum.NotFound,
@@ -971,6 +1005,7 @@ func TestAnAnswerAfterATimeoutDoesNotRefuseTheRun(t *testing.T) {
 // the reap evicts the transaction before the block carrying it is opened.
 // Reporting that as expired is a verdict about the chain drawn from a block the
 // run had in hand and had not read.
+// Requirements: TOT-004 and TOT-008.
 func TestABacklogIsNotAChainVerdict(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -997,6 +1032,7 @@ func TestABacklogIsNotAChainVerdict(t *testing.T) {
 // the chain signal. A run that has read every head it received and still never
 // saw the transaction is entitled to say the chain left it out, and that is the
 // whole point of keeping the two states apart.
+// Requirements: TOT-008 and SC-003.
 func TestACaughtUpTrackerStillReportsExpired(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -1019,6 +1055,7 @@ func TestACaughtUpTrackerStillReportsExpired(t *testing.T) {
 
 // TestAStoppedHeadStreamIsNotAChainVerdict fails when a run whose head stream
 // died keeps reporting chain verdicts. Nothing after that point is read at all.
+// Requirements: TOT-004.
 func TestAStoppedHeadStreamIsNotAChainVerdict(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -1047,6 +1084,7 @@ func TestAStoppedHeadStreamIsNotAChainVerdict(t *testing.T) {
 // processing would fold that backlog into InclusionTime and into every latency
 // sample, so a run would report its own lateness as the chain's inclusion
 // latency, and the error would grow for the length of the run.
+// Requirements: TOT-022.
 func TestAHeadIsStampedWhenItArrivesNotWhenItIsRead(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -1080,6 +1118,7 @@ func TestAHeadIsStampedWhenItArrivesNotWhenItIsRead(t *testing.T) {
 // It is the same claim as a backlog in the head channel, one step further along:
 // the height was received and read once, the node did not have it, and the run
 // is waiting to ask again. A transaction may be in it.
+// Requirements: TOT-004 and TOT-008.
 func TestAQueuedHeightBlocksAChainVerdict(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -1108,6 +1147,7 @@ func TestAQueuedHeightBlocksAChainVerdict(t *testing.T) {
 // TestASingleMissedHeadIsCountedAsAHole fails when a gap of exactly one height
 // goes unrecorded. One missed head is the common case, and its transactions
 // would reap as expired for a block nothing read.
+// Requirements: TOT-004.
 func TestASingleMissedHeadIsCountedAsAHole(t *testing.T) {
 	ctx := context.Background()
 	src := NewMockBlockSource()
@@ -1127,3 +1167,58 @@ func TestASingleMissedHeadIsCountedAsAHole(t *testing.T) {
 		"one missed head went unrecorded, so its txs were blamed on the chain")
 	require.Zero(t, got.Expired)
 }
+
+// TestTheReadBudgetNeverOutlastsTheReapDeadline covers TOT-023.
+//
+// A transaction past reapAfter is already counted, so a read that returns after
+// it answers a question the run has closed. Holding to the bound also puts a
+// floor under how much history the tracking node has to keep.
+// Requirements: TOT-023 and SC-015.
+func TestTheReadBudgetNeverOutlastsTheReapDeadline(t *testing.T) {
+	// A run that reaps quickly must not keep asking for old heights.
+	short := newTestTracker(t, 2*time.Second, 100, NewMockBlockSource())
+	require.Equal(t, 2*time.Second, short.deferredReadBudget(),
+		"the run reads back further than the deadline it gave its transactions")
+
+	// A run that reaps slowly is bounded by the queue's own budget instead.
+	long := newTestTracker(t, time.Hour, 100, NewMockBlockSource())
+	require.Equal(t, maxDeferredReadBudget, long.deferredReadBudget())
+}
+
+// TestExpiredIsReachableInTheRecommendedTopology covers TOT-008 and SC-003, and
+// guards the failure the reap's queue rule can cause.
+//
+// Refusing a chain verdict while a height sits queued is right, and it is only
+// safe because the queue is normally empty. Taking heads and receipts from one
+// node is what keeps it empty: a height cannot arrive as a head before the node
+// that sent it holds the block. Split them, and the queue never empties, and
+// expired stops being reachable at all.
+// Requirements: TOT-008, TOT-022 and SC-003.
+func TestExpiredIsReachableInTheRecommendedTopology(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	// One node: every head the run sees is a block that node already holds, so
+	// each read resolves and nothing is left queued.
+	for h := uint64(7); h <= 10; h++ {
+		src.SetReceipts(h)
+		tr.noteHeadReceived(h)
+		tr.processHead(ctx, h, 0, time.Unix(1002, 0), h-1)
+		tr.noteHeadResolved(h)
+	}
+	for s := range tr.state.Lock() {
+		require.Empty(t, s.pending, "a single-node run left heights queued")
+	}
+	tr.reap(ctx)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Expired,
+		"a run that read every block it saw could not say the chain left a tx out")
+	require.Zero(t, got.StatusUnavailable)
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 03b5661..1ce2d96 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -157,8 +157,9 @@ const (
 // hole. A duration is the quantity that actually matters, which is how far the
 // receipt node trails the head node.
 const (
-	// deferredReadBudget is how long a height may wait before the run calls the
-	// serving node behind.
+	// maxDeferredReadBudget caps how long a height may wait before the run calls
+	// the serving node behind. The live value is deferredReadBudget, which also
+	// answers to the run's reap deadline.
 	//
 	// It is larger than rereadSweepBudget, and that relationship is load-bearing
 	// rather than a coincidence of two tuned numbers. A sweep can spend its whole
@@ -166,8 +167,8 @@ const (
 	// retires the tail for a delay this process caused and reports it as the
 	// node being behind. Anything queued gets at least one sweep after the one
 	// it was queued in.
-	deferredReadBudget = 2 * rereadSweepBudget
-	maxDeferredReads   = 64
+	maxDeferredReadBudget = 2 * rereadSweepBudget
+	maxDeferredReads      = 64
 	// rereadSweepBudget bounds what one head spends re-reading, whatever is
 	// waiting.
 	//
@@ -320,6 +321,12 @@ func (t *InclusionTracker) Register(ctx context.Context, tx *types.LoadTx) {
 		// would overwrite the first and the run would report one terminal state
 		// for two accepted transactions. A resend after a send timeout produces
 		// exactly that. Count the duplicate rather than lose it.
+		//
+		// status_unavailable is the closest of the states the spec defines, and
+		// it is not a clean fit: the run could read this transaction's status
+		// perfectly well, it just has nowhere to put a second copy. The state
+		// exists for a status the run could not read. Naming this collision
+		// properly needs a state the spec does not have.
 		if _, dup := s.inflight[hash]; dup {
 			s.duplicates++
 			outcome = OutcomeStatusUnavailable
@@ -350,25 +357,32 @@ func (t *InclusionTracker) meterOutcome(ctx context.Context, outcome Outcome, sc
 	))
 }
 
-// Run subscribes to new heads on headEndpoint and reads each arriving block's
-// receipts from receiptEndpoint. Pass the same string for both to run against
-// one node.
+// Run subscribes to new heads on endpoint and reads each arriving block's
+// receipts from the same endpoint.
+//
+// One node for both, and that is a correctness rule rather than a convenience.
+// A head notification carries the raw committed height; a status read resolves
+// through a watermark that also waits on the receipt store. The two disagree
+// even on one node, and taking them from separate nodes adds peer lag on top of
+// that, which a node inside its readiness threshold can carry for minutes
+// without reporting unhealthy. Every height would then arrive before the reading
+// node held it.
 //
 // The tracker reads the height it just received as a head, and re-reads a height
 // the serving node had not reached yet. A height is only re-read while it is
 // younger than deferredReadBudget, so the reach back is seconds and the serving
 // node's receipt retention does not bound it. A change that reaches further back
 // does.
-func (t *InclusionTracker) Run(ctx context.Context, headEndpoint, receiptEndpoint string) error {
-	wsEndpoint := utils.GetWSEndpoint(headEndpoint)
+func (t *InclusionTracker) Run(ctx context.Context, endpoint string) error {
+	wsEndpoint := utils.GetWSEndpoint(endpoint)
 	if t.source == nil {
-		client, err := ethclient.Dial(receiptEndpoint)
+		client, err := ethclient.Dial(endpoint)
 		if err != nil {
-			return fmt.Errorf("inclusion tracker: dial %s: %w", receiptEndpoint, err)
+			return fmt.Errorf("inclusion tracker: dial %s: %w", endpoint, err)
 		}
 		defer client.Close()
 		t.source = ethReceiptSource{client: client}
-		if err := t.preflight(ctx, receiptEndpoint); err != nil {
+		if err := t.preflight(ctx, endpoint); err != nil {
 			return err
 		}
 	}
@@ -494,7 +508,7 @@ func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
 			t.requeue(ctx, due[i:])
 			return
 		}
-		if waited := time.Since(d.deferredAt); waited > deferredReadBudget {
+		if waited := time.Since(d.deferredAt); waited > t.deferredReadBudget() {
 			// The node has had its budget. It is behind rather than busy, and
 			// the run has to say it could not read this height.
 			inclusionDeferredWait.Record(ctx, waited.Seconds(), metric.WithAttributes(
@@ -564,6 +578,19 @@ func (t *InclusionTracker) stopTracking(ctx context.Context) {
 		attribute.String("reason", reasonTrackingStopped)))
 }
 
+// deferredReadBudget is how long a height may wait to be read.
+//
+// It never outlasts the deadline the run gives a transaction to reach a block. A
+// transaction past reapAfter is already counted, so a read that comes back later
+// answers a question the run has closed, and it would ask a node for history the
+// run's own bound says it need not keep.
+func (t *InclusionTracker) deferredReadBudget() time.Duration {
+	if t.reapAfter < maxDeferredReadBudget {
+		return t.reapAfter
+	}
+	return maxDeferredReadBudget
+}
+
 // takePending removes every queued height and returns it. Both callers must take
 // and clear in one critical section, so one function owns that.
 func (t *InclusionTracker) takePending() []deferredRead {

From b7677762aeb6d531737b9b2d2c0b0a933764b32e Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 13:28:04 -0700
Subject: [PATCH 15/17] refactor(stats): delete the machinery the topology fix
 made unnecessary

Roughly 750 lines out, 110 in. The tracker drops from 1219 to 884 lines, its
tests from 1224 to 784, and the suite from 20 seconds to 11.

The deferred-read queue, its sweep budget, its requeue, its starvation rule, its
three interacting constants, its shutdown drain and its wait budget derived from
reapAfter all existed for one reason: heads arrived from a node that was not the
one being read, so a height was routinely announced before the reader held it.
Fixing that violation removed the cause and left the apparatus.

The cause is not entirely gone, and that is why a retry stays. seid publishes a
head from Commit while the receipt store's writer is still asynchronous, so a
height answers null for the gap between the two. Our nodes run the pebbledb
receipt store with an async write buffer of a hundred, set in the fleet's own
defaults, so the window is real and bounded by that queue rather than by the
chain. It is one write, not one block, so waiting in place resolves it in
milliseconds where the queue waited for the next head.

Deleting the retry as well was the tempting move and it is wrong. A hole raises
a watermark that only grows, so one hole anywhere inside a transaction's reap
window converts it. At a null rate of five percent, every transaction in a run
reports status_unavailable and expired never fires: the tool loses the one
verdict it exists to deliver, and a run that can never say the chain dropped
anything is not a load test.

The preflight loses its refusal table and most of its classifier. The probe asks
for height 0, which sei-chain answers from a constant ahead of its watermark and
its receipt store, so a healthy EVM RPC cannot fail it and the reason for a
failure cannot change the verdict. That also closes a hole the larger version
had: an endpoint answering not-found to genesis is not a Sei EVM RPC at all, and
it used to be admitted.

deferred_read_wait becomes block_read_wait and loses its disposition label. It
now measures one thing, which is how far a node's watermark trails its own head,
and that is the number nobody has and the one that says whether even the retry
earns its place. Nothing in the platform repo reads either name.

Fourteen tests lost their subject with the code they covered. Three guards
replace them: the gap is waited out rather than written off, the retry gives up
rather than holding the head loop, and expired stays reachable.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_outcome_test.go | 590 +++++---------------------------
 stats/inclusion_tracker.go      | 481 ++++----------------------
 stats/inclusion_tracker_test.go |  12 +-
 stats/metrics.go                |   8 +-
 4 files changed, 170 insertions(+), 921 deletions(-)

diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 82e0a32..2630e70 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/sei-protocol/sei-load/types"
 
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
@@ -38,7 +37,7 @@ func TestAllRevertedReceiptsYieldZeroCommitted(t *testing.T) {
 		})
 	}
 	src.SetReceipts(7, receipts...)
-	tr.matchBlock(context.Background(), 7, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 7, 0, time.Unix(1002, 0))
 
 	got := tr.collector.GetOperationStats()[key]
 	require.Zero(t, got.Committed, "a failed transaction counted as committed")
@@ -67,7 +66,7 @@ func TestReceiptsSeparateCommittedFromReverted(t *testing.T) {
 		receipts = append(receipts, blockReceipt{Hash: tx.EthTx.Hash(), Status: status, HasStatus: true})
 	}
 	src.SetReceipts(9, receipts...)
-	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 9, 0, time.Unix(1002, 0))
 
 	got := tr.collector.GetOperationStats()[key]
 	require.Equal(t, uint64(2), got.Committed)
@@ -94,7 +93,7 @@ func TestOutcomesCarryTheOperation(t *testing.T) {
 		blockReceipt{Hash: read.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true},
 		blockReceipt{Hash: write.EthTx.Hash(), Status: ethtypes.ReceiptStatusFailed, HasStatus: true},
 	)
-	tr.matchBlock(context.Background(), 3, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 3, 0, time.Unix(1002, 0))
 
 	stats := tr.collector.GetOperationStats()
 	require.Equal(t, uint64(1), stats[OperationKey{Scenario: "storagerw", Operation: "read"}].Committed)
@@ -127,7 +126,7 @@ func TestRequestsPerBlockDoNotTrackVolume(t *testing.T) {
 				})
 			}
 			src.SetReceipts(11, receipts...)
-			tr.matchBlock(context.Background(), 11, time.Unix(1002, 0))
+			tr.matchBlock(context.Background(), 11, 0, time.Unix(1002, 0))
 
 			require.Equal(t, int64(1), src.FetchCount(),
 				"a block carrying %d transactions cost %d requests, not one",
@@ -170,7 +169,7 @@ func TestUnreadableBlockIsNotAChainVerdict(t *testing.T) {
 	tr.Register(context.Background(), tx)
 
 	src.SetFetchErr(errors.New("connection refused"))
-	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 9, 0, time.Unix(1002, 0))
 	tr.reap(context.Background())
 
 	got := tr.collector.GetOperationStats()[key]
@@ -197,7 +196,7 @@ func TestAnIdleBlockIsNotAHole(t *testing.T) {
 	tr.Register(context.Background(), tx)
 
 	src.SetReceipts(9) // the node holds the block and it carried nothing
-	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 9, 0, time.Unix(1002, 0))
 	tr.reap(context.Background())
 
 	got := tr.collector.GetOperationStats()[key]
@@ -224,7 +223,7 @@ func TestDuplicateRegistrationReachesATerminalState(t *testing.T) {
 	src.SetReceipts(3, blockReceipt{
 		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
 	})
-	tr.matchBlock(context.Background(), 3, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 3, 0, time.Unix(1002, 0))
 
 	got := tr.collector.GetOperationStats()[key]
 	sum := got.Committed + got.Reverted + got.Expired + got.DroppedAtCap +
@@ -243,7 +242,7 @@ func TestTransactionsRegisteredAfterTheHoleStillExpire(t *testing.T) {
 	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
 
 	src.SetFetchErr(errors.New("connection refused"))
-	tr.matchBlock(context.Background(), 9, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 9, 0, time.Unix(1002, 0))
 
 	tx := loadTx(2, time.Unix(1000, 0))
 	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
@@ -269,7 +268,7 @@ func TestReceiptWithoutStatusIsNotAFailure(t *testing.T) {
 	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
 	tr.Register(context.Background(), tx)
 	src.SetReceipts(4, blockReceipt{Hash: tx.EthTx.Hash(), HasStatus: false})
-	tr.matchBlock(context.Background(), 4, time.Unix(1002, 0))
+	tr.matchBlock(context.Background(), 4, 0, time.Unix(1002, 0))
 
 	got := tr.collector.GetOperationStats()[key]
 	require.Equal(t, uint64(1), got.StatusUnavailable,
@@ -320,12 +319,12 @@ func TestOutcomesPartitionEveryAcceptedTx(t *testing.T) {
 	register(6)
 
 	src.SetReceipts(5, receipts...)
-	tr.matchBlock(ctx, 5, time.Unix(1002, 0))
+	tr.matchBlock(ctx, 5, 0, time.Unix(1002, 0))
 
 	// A read that fails puts the run in a state it cannot attribute, so the
 	// fourth reaps as status_unavailable rather than expired.
 	src.SetFetchErr(errors.New("connection refused"))
-	tr.matchBlock(ctx, 6, time.Unix(1003, 0))
+	tr.matchBlock(ctx, 6, 0, time.Unix(1003, 0))
 	time.Sleep(40 * time.Millisecond)
 	tr.reap(ctx)
 
@@ -353,7 +352,12 @@ func TestOutcomesPartitionEveryAcceptedTx(t *testing.T) {
 }
 
 // TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer fails when a run against
-// a node that serves no receipts starts anyway. Such a run completes, reports
+// a node that cannot serve receipts starts anyway.
+//
+// The probe asks for height 0, which sei-chain answers from a constant with no
+// watermark, receipt store or sync state involved. A healthy Sei EVM RPC cannot
+// fail it, so an error that survives every attempt disqualifies the endpoint
+// whatever the error was, and the reason does not change the verdict. Such a run completes, reports
 // every transaction un-included, and exits zero, which reads as a chain that
 // accepted nothing.
 // Requirements: TOT-021.
@@ -367,8 +371,11 @@ func TestPreflightFailsTheRunOnAnEndpointThatCannotAnswer(t *testing.T) {
 		{"not_listening", errors.New("dial tcp 10.0.0.1:8545: connect: connection refused"), true},
 		{"no_such_host", errors.New("dial tcp: lookup rpc-0: no such host"), true},
 		{"answers_nothing", context.DeadlineExceeded, true},
-		{"node_behind", ethereum.NotFound, false},
-		{"connection_reset", errors.New("read tcp 10.0.0.1:8545: read: connection reset by peer"), false},
+		{"answers_prose", errors.New("invalid character '<' looking for beginning of value"), true},
+		// A Sei node serves genesis from a constant, ahead of its watermark and
+		// its receipt store, so not-found to height 0 is not a Sei EVM RPC.
+		{"not_found_at_genesis", ethereum.NotFound, true},
+		{"reset_every_attempt", errors.New("read tcp 10.0.0.1:8545: read: connection reset by peer"), true},
 	}
 	for _, tc := range cases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -439,7 +446,7 @@ func TestAPartlyNullReadMatchesWhatArrivedAndCountsTheRest(t *testing.T) {
 	src.SetNulls(1).SetReceipts(4, blockReceipt{
 		Hash: named.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
 	})
-	tr.matchBlock(ctx, 4, time.Unix(1002, 0))
+	tr.matchBlock(ctx, 4, 0, time.Unix(1002, 0))
 	tr.reap(ctx)
 
 	got := tr.collector.GetOperationStats()[key]
@@ -450,194 +457,6 @@ func TestAPartlyNullReadMatchesWhatArrivedAndCountsTheRest(t *testing.T) {
 	require.Zero(t, got.Expired)
 }
 
-// TestALaggingReceiptNodeIsRetriedNotCountedAsAHole fails when a height the
-// receipt node has not reached becomes a hole on first sight.
-//
-// It is the normal case once receiptEndpoint names a second node: the head
-// arrives from one node and the other has not committed that height yet.
-// Counting it would mark a hole per block, which is how expired became
-// unreachable before.
-// Requirements: TOT-020.
-func TestALaggingReceiptNodeIsRetriedNotCountedAsAHole(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Minute, 100, src)
-	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
-
-	tx := loadTx(1, time.Unix(1000, 0))
-	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-	tr.Register(ctx, tx)
-
-	// Head 7 arrives before the receipt node holds it.
-	src.SetFetchErr(ethereum.NotFound)
-	last := tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
-	require.Equal(t, uint64(7), last)
-
-	// By the next head it has caught up, and the re-read finds the transaction.
-	src.SetFetchErr(nil)
-	src.SetReceipts(7, blockReceipt{
-		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
-	})
-	tr.processHead(ctx, 8, 0, time.Unix(1003, 0), 7)
-
-	got := tr.collector.GetOperationStats()[key]
-	require.Equal(t, uint64(1), got.Committed,
-		"a height the node had not reached was written off instead of re-read")
-	require.Zero(t, got.StatusUnavailable)
-}
-
-// TestAReceiptNodeThatStaysBehindBecomesAHole fails when a node that never
-// catches up is retried forever. One re-read is the bound: a node still behind
-// a block interval later is behind rather than busy.
-// Requirements: TOT-020.
-func TestAReceiptNodeThatStaysBehindBecomesAHole(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Nanosecond, 100, src)
-	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
-
-	tx := loadTx(1, time.Unix(1000, 0))
-	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-	tr.Register(ctx, tx)
-
-	src.SetFetchErr(ethereum.NotFound)
-	// Every head gives the height one more read. Past the budget it is a hole.
-	for h := uint64(7); h <= 7+maxDeferredReads+1; h++ {
-		tr.processHead(ctx, h, 0, time.Unix(1002, 0), h-1)
-	}
-	tr.reap(ctx)
-
-	got := tr.collector.GetOperationStats()[key]
-	require.Equal(t, uint64(1), got.StatusUnavailable,
-		"a node that never caught up was never counted as a hole")
-	require.Zero(t, got.Expired)
-}
-
-// TestFetchFailureReasonReadsTypedErrors fails when the classifier's typed
-// checks are removed, which the substring fallback would otherwise hide.
-//
-// The buckets decide whether a run is refused, so a false refusal on a busy
-// node is worse than the blind run the refusal exists to prevent.
-// Requirements: TOT-021.
-func TestFetchFailureReasonReadsTypedErrors(t *testing.T) {
-	cases := []struct {
-		name string
-		err  error
-		want string
-	}{
-		{"rpc_method_not_found", rpc.HTTPError{
-			StatusCode: 403, Status: "403 Forbidden",
-			Body: []byte(`{"error":{"code":-32601,"message":"not whitelisted"}}`),
-		}, reasonMethodUnavailable},
-		{"rate_limited_body_holding_the_digits", rpc.HTTPError{
-			StatusCode: 429, Status: "429 Too Many Requests",
-			Body: []byte(`{"error":{"code":-32005},"id":"req-32601-a"}`),
-		}, reasonOther},
-		{"ingress_404_mid_reconcile", rpc.HTTPError{
-			StatusCode: 404, Status: "404 Not Found", Body: []byte("404 page not found"),
-		}, reasonOther},
-		{"bad_gateway", rpc.HTTPError{
-			StatusCode: 502, Status: "502 Bad Gateway",
-		}, reasonOther},
-		{"not_found_sentinel", ethereum.NotFound, reasonNotFound},
-		{"deadline", context.DeadlineExceeded, reasonTimeout},
-		{"dial_refused", &net.OpError{
-			Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED,
-		}, reasonUnreachable},
-		{"name_does_not_resolve", &net.DNSError{
-			Err: "no such host", Name: "rpc-0", IsNotFound: true,
-		}, reasonUnreachable},
-		{"reset_by_a_busy_node", &net.OpError{
-			Op: "read", Net: "tcp", Err: syscall.ECONNRESET,
-		}, reasonOther},
-		{"pruned_before_availability", errors.New(
-			"receipts have been pruned; earliest available is 100"), reasonPruned},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			require.Equal(t, tc.want, fetchFailureReason(tc.err))
-		})
-	}
-}
-
-// TestADeferredHeightKeepsItsOwnArrival fails when a re-read stamps the
-// transaction with the arrival of the head that triggered it.
-//
-// Arrival becomes the inclusion latency sample. In the two-node topology this
-// tracker recommends, every height is deferred at least once, so the error
-// would land on every sample rather than a few.
-// Requirements: TOT-022.
-func TestADeferredHeightKeepsItsOwnArrival(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Minute, 100, src)
-
-	tx := loadTx(1, time.Unix(1000, 0))
-	tx.IntendedSendTime = time.Unix(1000, 0)
-	tr.Register(ctx, tx)
-
-	blockSeven := time.Unix(1002, 0)
-	blockEight := blockSeven.Add(400 * time.Millisecond)
-
-	src.SetFetchErr(ethereum.NotFound)
-	tr.processHead(ctx, 7, 0, blockSeven, 6)
-
-	src.SetFetchErr(nil)
-	src.SetReceipts(7, blockReceipt{
-		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
-	})
-	tr.processHead(ctx, 8, 0, blockEight, 7)
-
-	require.Equal(t, blockSeven, tx.InclusionTime,
-		"the transaction was stamped with a later block's arrival, so every "+
-			"latency sample in this topology is inflated by one block")
-}
-
-// TestAPendingHeightAtShutdownIsNotAChainVerdict fails when a height still
-// waiting to be read at shutdown is turned into a failure.
-//
-// The head loop and the reap loop end on the same signal, so no reap follows the
-// shutdown sweep and those transactions are counted as in flight at shutdown,
-// which is already not a claim about the chain. Marking a hole would put a
-// failure on the series that answers "was this run blind?" at the end of every
-// healthy run in the topology this tracker recommends.
-// Requirements: TOT-003 and TOT-004.
-func TestAPendingHeightAtShutdownIsNotAChainVerdict(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource()
-	// Nanosecond, so the reap below actually evicts. A minute here would make
-	// the closing assertion unfailable.
-	tr := newTestTracker(t, time.Nanosecond, 100, src)
-	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
-
-	tx := loadTx(1, time.Unix(1000, 0))
-	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-	tr.Register(ctx, tx)
-
-	src.SetFetchErr(ethereum.NotFound)
-	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
-	tr.recordUnreadAtShutdown(ctx)
-
-	// The watermark rises, because those heights really did go unread and
-	// nothing reaped afterwards may speak for the chain. block_fetch_errors does
-	// not, because a healthy run draining a queue it was always going to drain
-	// is not the run going blind.
-	for s := range tr.state.Lock() {
-		require.Equal(t, uint64(1), s.blindHeights,
-			"an unread height at shutdown left the attribution watermark alone")
-	}
-	require.Equal(t, uint64(1), tr.Summary().InflightAtShutdown,
-		"the transaction was lost rather than counted as in flight")
-
-	// And a reap after that shutdown still must not call it a chain verdict. The
-	// tracker received a head it never resolved, so it cannot speak for the
-	// chain whatever the queue now holds.
-	tr.reap(ctx)
-	got := tr.collector.GetOperationStats()[key]
-	require.Zero(t, got.Expired, "a pending height became a chain verdict")
-	require.Equal(t, uint64(1), got.StatusUnavailable)
-}
-
 // TestASkippedHeadIsCountedAsAHole fails when a height the run never saw a head
 // for goes uncounted. Its transactions would reap as expired, which is a claim
 // about the chain for a block nothing read.
@@ -682,27 +501,6 @@ func TestTheDuplicateLegReachesBothLedgers(t *testing.T) {
 		"the closing log line and the metric disagree about the duplicate")
 }
 
-// TestTheDeferredQueueDrains fails when a height stays queued after it is read,
-// which would make every later head re-read every height the run ever deferred.
-// Requirements: TOT-011.
-func TestTheDeferredQueueDrains(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Minute, 100, src)
-
-	src.FailTimes(1, ethereum.NotFound)
-	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
-	tr.processHead(ctx, 8, 0, time.Unix(1003, 0), 7)
-
-	for s := range tr.state.Lock() {
-		require.Empty(t, s.pending, "a height stayed queued after it was read")
-	}
-	before := src.FetchCount()
-	tr.processHead(ctx, 9, 0, time.Unix(1004, 0), 8)
-	require.Equal(t, int64(1), src.FetchCount()-before,
-		"a later head re-read a height that was already done")
-}
-
 // TestARefusedRunDoesNotLookLikeAFinishedOne fails when the preflight's refusal
 // wraps a context sentinel.
 //
@@ -733,47 +531,6 @@ func TestARefusedRunDoesNotLookLikeAFinishedOne(t *testing.T) {
 	}
 }
 
-// TestPreflightNeedsEveryAttemptToAgree fails when one attempt's verdict speaks
-// for the run. An endpoint that answers at all, even with an error, is there.
-// Requirements: TOT-021.
-func TestPreflightNeedsEveryAttemptToAgree(t *testing.T) {
-	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Minute, 100, src)
-
-	// The first attempt times out; the second says the node is merely behind,
-	// which proves it answers and speaks the method.
-	src.FailTimes(1, context.DeadlineExceeded)
-	require.NoError(t, tr.preflight(context.Background(), "http://node:8545"))
-
-	// Two attempts, two different permanent-looking causes. That is not one
-	// permanent cause, so the run proceeds and reports what it finds.
-	// Three, one per attempt, so the loop cannot fall through to a success and
-	// reach the same verdict by another route.
-	refused := &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED}
-	mixed := NewMockBlockSource().SetErrSequence(
-		context.DeadlineExceeded, refused, refused,
-	)
-	trMixed := newTestTracker(t, time.Minute, 100, mixed)
-	require.NoError(t, trMixed.preflight(context.Background(), "http://node:8545"),
-		"two different causes were treated as one settled verdict")
-
-	// Every attempt the same permanent cause is a refusal.
-	src.SetFetchErr(context.DeadlineExceeded)
-	require.ErrorIs(t, tr.preflight(context.Background(), "http://node:8545"),
-		ErrEndpointUnusable)
-}
-
-// TestATemporaryResolverFailureDoesNotRefuseTheRun fails when a resolver blip
-// ends a run. Only a name that does not exist is permanent.
-// Requirements: TOT-021.
-func TestATemporaryResolverFailureDoesNotRefuseTheRun(t *testing.T) {
-	servfail := &net.DNSError{Err: "server misbehaving", Name: "rpc-0", IsTemporary: true}
-	require.Equal(t, reasonOther, fetchFailureReason(servfail))
-
-	missing := &net.DNSError{Err: "no such host", Name: "rpc-0", IsNotFound: true}
-	require.Equal(t, reasonUnreachable, fetchFailureReason(missing))
-}
-
 // TestTheSummaryTermsAreDisjoint fails when one transaction lands in two terms
 // of the closing log line, which invites an operator to add them up and get more
 // than the run accepted.
@@ -788,7 +545,7 @@ func TestTheSummaryTermsAreDisjoint(t *testing.T) {
 	// A receipt carrying a post-state root instead of a status: it was in a
 	// block, and what it did there cannot be read.
 	src.SetReceipts(4, blockReceipt{Hash: tx.EthTx.Hash(), HasStatus: false})
-	tr.matchBlock(ctx, 4, time.Unix(1002, 0))
+	tr.matchBlock(ctx, 4, 0, time.Unix(1002, 0))
 
 	s := tr.Summary()
 	total := s.Included + s.Expired + s.StatusUnavailable + s.DroppedAtCap + s.InflightAtShutdown
@@ -814,188 +571,6 @@ func TestAGapIsRecordedOnce(t *testing.T) {
 	}
 }
 
-// TestOneHeadIsBoundedAndLosesNoHeight fails when a sweep runs unbounded, or
-// when a height it did not reach leaves the queue with nothing recorded.
-//
-// Asserting only that the queue is non-empty is not enough: the heights the
-// sweep did read re-defer themselves and refill it while the ones it dropped go
-// unnoticed.
-// Requirements: TOT-011 and TOT-020.
-func TestOneHeadIsBoundedAndLosesNoHeight(t *testing.T) {
-	ctx := context.Background()
-	// Each read takes a fifth of the sweep, so the budget runs out partway down
-	// the queue and the rest must survive. The node holds none of them, so every
-	// read comes back not-found and every height belongs back in the queue.
-	src := NewCountingSlowSource(rereadSweepBudget/5, 1_000_000)
-	tr := newInclusionTrackerWithSource(
-		NewInclusionTracker("test-chain", time.Minute, 100, true, NewCollector()), src)
-
-	now := time.Now()
-	var want []uint64
-	for h := uint64(2); h <= 12; h++ {
-		require.True(t, tr.deferHeight(deferredRead{num: h, arrival: now, deferredAt: now}))
-		want = append(want, h)
-	}
-
-	start := time.Now()
-	tr.rereadDeferred(ctx)
-	spent := time.Since(start)
-
-	// One full read may exceed the remainder, because the first entry is
-	// guaranteed a fair chance. Nothing beyond that.
-	require.Less(t, spent, rereadSweepBudget+readTimeout,
-		"one head spent %s re-reading", spent)
-	require.ElementsMatch(t, want, pendingHeights(tr),
-		"a height the sweep did not reach left the queue")
-	for s := range tr.state.Lock() {
-		require.Zero(t, s.blindHeights, "a height still queued was also called a hole")
-	}
-}
-
-// TestAWaitBudgetOutlastsASweep fails when a height can be retired for a delay
-// the sweep itself caused.
-//
-// A sweep may spend its whole budget on the heights ahead of one. If the wait
-// budget is the shorter of the two, the tail ages out mid-sweep and the run
-// reports the serving node as behind on heights it was never asked about.
-// Requirements: TOT-020 and TOT-023.
-func TestAWaitBudgetOutlastsASweep(t *testing.T) {
-	require.Greater(t, maxDeferredReadBudget, rereadSweepBudget,
-		"a height can age out inside one sweep, so the run would call the node "+
-			"behind for time this process spent elsewhere")
-
-	// And a sweep leaves room for the read it is budgeting for. Below this a
-	// re-read is capped under one read's budget and can never clear it, so a
-	// node answering just above the cap loses every height: in the topology
-	// where every height is deferred, that is every height in the run.
-	require.GreaterOrEqual(t, rereadSweepBudget, readTimeout,
-		"every re-read is capped below one read's budget")
-}
-
-// pendingHeights reads the queued heights, for a test that has to name them all.
-func pendingHeights(tr *InclusionTracker) []uint64 {
-	for s := range tr.state.Lock() {
-		out := make([]uint64, 0, len(s.pending))
-		for _, d := range s.pending {
-			out = append(out, d.num)
-		}
-		return out
-	}
-	panic("unreachable")
-}
-
-// TestAnUnreachedHeightGoesToTheFrontOfTheQueue fails when a sweep puts the
-// heights it did not reach behind the ones it read.
-//
-// A sweep walks front to back, so what it re-deferred got a turn and the tail
-// did not. Putting the tail last means the same front entries consume every
-// sweep, and a height further back is never served at all: measured with three
-// queued heights and a node holding only the third, the third was retired as
-// though the node were behind on it without the run ever completing a read of
-// it. Order here is service fairness, not age.
-// Requirements: TOT-020.
-func TestAnUnreachedHeightGoesToTheFrontOfTheQueue(t *testing.T) {
-	ctx := context.Background()
-	tr := newTestTracker(t, time.Minute, 100, NewMockBlockSource())
-
-	now := time.Now()
-	// Two heights the sweep already read and re-deferred.
-	for _, h := range []uint64{10, 11} {
-		require.True(t, tr.deferHeight(deferredRead{num: h, arrival: now, deferredAt: now}))
-	}
-	// One it never reached.
-	tr.requeue(ctx, []deferredRead{{num: 12, arrival: now, deferredAt: now}})
-
-	require.Equal(t, []uint64{12, 10, 11}, pendingHeights(tr),
-		"the height that got no turn was put behind the ones that did")
-}
-
-// TestAHeightOutOfBudgetBecomesAHole fails when the wait budget never retires a
-// height. The budget is the whole reason the bound is a duration rather than a
-// count of reads, and the queue cap must not be what fires instead.
-// Requirements: TOT-020 and TOT-023.
-func TestAHeightOutOfBudgetBecomesAHole(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource().SetFetchErr(ethereum.NotFound)
-	tr := newTestTracker(t, time.Minute, 100, src)
-
-	// One height, so the queue cap cannot be what retires it.
-	old := time.Now().Add(-tr.deferredReadBudget() - time.Second)
-	require.True(t, tr.deferHeight(deferredRead{num: 7, arrival: old, deferredAt: old}))
-	tr.rereadDeferred(ctx)
-
-	require.Empty(t, pendingHeights(tr), "a height past its budget stayed queued")
-	for s := range tr.state.Lock() {
-		require.Equal(t, uint64(1), s.blindHeights,
-			"a height past its budget left the queue without being called a hole")
-	}
-}
-
-// TestTheWaitBudgetDoesNotRestartOnEachRead fails when a re-read resets the
-// clock. The budget measures how far the receipt node trails the head node, and
-// restarting it per read would mean a node behind forever is never called
-// behind.
-// Requirements: TOT-023.
-func TestTheWaitBudgetDoesNotRestartOnEachRead(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource().SetFetchErr(ethereum.NotFound)
-	tr := newTestTracker(t, time.Minute, 100, src)
-
-	first := time.Now().Add(-tr.deferredReadBudget() / 2)
-	require.True(t, tr.deferHeight(deferredRead{num: 7, arrival: first, deferredAt: first}))
-	tr.rereadDeferred(ctx) // re-read fails, re-defers
-
-	queued := func() deferredRead {
-		for s := range tr.state.Lock() {
-			require.Len(t, s.pending, 1)
-			return s.pending[0]
-		}
-		panic("unreachable")
-	}()
-	require.Equal(t, first, queued.deferredAt,
-		"the wait restarted, so the height can never reach its budget")
-}
-
-// TestADuplicateHeadStillDrainsTheQueue fails when a repeated or out-of-order
-// head returns before the sweep. Nothing else drains the queue, so a height left
-// in it reaps as a chain verdict for a block never read.
-// Requirements: TOT-020.
-func TestADuplicateHeadStillDrainsTheQueue(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Minute, 100, src)
-
-	tx := loadTx(1, time.Unix(1000, 0))
-	tr.Register(ctx, tx)
-
-	// Head 7 arrives before the receipt node holds it.
-	src.SetFetchErr(ethereum.NotFound)
-	tr.processHead(ctx, 7, 0, time.Unix(1002, 0), 6)
-
-	// The same head again. The node has caught up by now.
-	src.SetFetchErr(nil)
-	src.SetReceipts(7, blockReceipt{
-		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
-	})
-	tr.processHead(ctx, 7, 0, time.Unix(1003, 0), 7)
-
-	require.Equal(t, uint64(1), tr.Summary().Included,
-		"a repeated head returned without draining the queue")
-}
-
-// TestAnAnswerAfterATimeoutDoesNotRefuseTheRun fails when one timed-out attempt
-// decides the run. A later attempt that answers at all proves the endpoint is
-// there, whatever it said.
-// Requirements: TOT-021.
-func TestAnAnswerAfterATimeoutDoesNotRefuseTheRun(t *testing.T) {
-	src := NewMockBlockSource().SetErrSequence(
-		context.DeadlineExceeded, ethereum.NotFound, ethereum.NotFound,
-	)
-	tr := newTestTracker(t, time.Minute, 100, src)
-	require.NoError(t, tr.preflight(context.Background(), "http://node:8545"),
-		"an endpoint that answered was refused because an earlier attempt timed out")
-}
-
 // TestABacklogIsNotAChainVerdict fails when a transaction is called expired
 // because the tracker had not got to its block yet.
 //
@@ -1112,38 +687,6 @@ func TestAHeadIsStampedWhenItArrivesNotWhenItIsRead(t *testing.T) {
 	}
 }
 
-// TestAQueuedHeightBlocksAChainVerdict fails when a reap calls the chain while
-// the run is still waiting on a height it queued.
-//
-// It is the same claim as a backlog in the head channel, one step further along:
-// the height was received and read once, the node did not have it, and the run
-// is waiting to ask again. A transaction may be in it.
-// Requirements: TOT-004 and TOT-008.
-func TestAQueuedHeightBlocksAChainVerdict(t *testing.T) {
-	ctx := context.Background()
-	src := NewMockBlockSource()
-	tr := newTestTracker(t, time.Nanosecond, 100, src)
-	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
-
-	tx := loadTx(1, time.Unix(1000, 0))
-	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
-	tr.Register(ctx, tx)
-
-	// Everything received has been read, so only the queue stands between the
-	// run and a chain verdict.
-	tr.noteHeadReceived(8)
-	tr.noteHeadResolved(8)
-	now := time.Now()
-	require.True(t, tr.deferHeight(deferredRead{num: 8, arrival: now, deferredAt: now}))
-
-	tr.reap(ctx)
-
-	got := tr.collector.GetOperationStats()[key]
-	require.Equal(t, uint64(1), got.StatusUnavailable,
-		"a transaction was blamed on the chain while a height sat queued to be read")
-	require.Zero(t, got.Expired)
-}
-
 // TestASingleMissedHeadIsCountedAsAHole fails when a gap of exactly one height
 // goes unrecorded. One missed head is the common case, and its transactions
 // would reap as expired for a block nothing read.
@@ -1168,23 +711,6 @@ func TestASingleMissedHeadIsCountedAsAHole(t *testing.T) {
 	require.Zero(t, got.Expired)
 }
 
-// TestTheReadBudgetNeverOutlastsTheReapDeadline covers TOT-023.
-//
-// A transaction past reapAfter is already counted, so a read that returns after
-// it answers a question the run has closed. Holding to the bound also puts a
-// floor under how much history the tracking node has to keep.
-// Requirements: TOT-023 and SC-015.
-func TestTheReadBudgetNeverOutlastsTheReapDeadline(t *testing.T) {
-	// A run that reaps quickly must not keep asking for old heights.
-	short := newTestTracker(t, 2*time.Second, 100, NewMockBlockSource())
-	require.Equal(t, 2*time.Second, short.deferredReadBudget(),
-		"the run reads back further than the deadline it gave its transactions")
-
-	// A run that reaps slowly is bounded by the queue's own budget instead.
-	long := newTestTracker(t, time.Hour, 100, NewMockBlockSource())
-	require.Equal(t, maxDeferredReadBudget, long.deferredReadBudget())
-}
-
 // TestExpiredIsReachableInTheRecommendedTopology covers TOT-008 and SC-003, and
 // guards the failure the reap's queue rule can cause.
 //
@@ -1204,17 +730,14 @@ func TestExpiredIsReachableInTheRecommendedTopology(t *testing.T) {
 	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
 	tr.Register(ctx, tx)
 
-	// One node: every head the run sees is a block that node already holds, so
-	// each read resolves and nothing is left queued.
+	// One node: every head the run sees is a block that node already holds, or
+	// holds within a retry, so every read resolves.
 	for h := uint64(7); h <= 10; h++ {
 		src.SetReceipts(h)
 		tr.noteHeadReceived(h)
 		tr.processHead(ctx, h, 0, time.Unix(1002, 0), h-1)
 		tr.noteHeadResolved(h)
 	}
-	for s := range tr.state.Lock() {
-		require.Empty(t, s.pending, "a single-node run left heights queued")
-	}
 	tr.reap(ctx)
 
 	got := tr.collector.GetOperationStats()[key]
@@ -1222,3 +745,64 @@ func TestExpiredIsReachableInTheRecommendedTopology(t *testing.T) {
 		"a run that read every block it saw could not say the chain left a tx out")
 	require.Zero(t, got.StatusUnavailable)
 }
+
+// Requirements: TOT-020 and TOT-004.
+// TestTheWatermarkGapIsWaitedOutNotWrittenOff fails when a height the node has
+// not finished writing is called a hole instead of waited for.
+//
+// seid publishes a head from Commit while the receipt store's writer is still
+// async, so a height answers null for the gap between the two. That gap is one
+// async write, not a chain fault: writing it off would raise the blind
+// watermark, and because that watermark only grows, one such gap inside a
+// transaction's reap window converts it. At any appreciable rate that makes
+// expired unreachable and the run can never say the chain dropped anything.
+func TestTheWatermarkGapIsWaitedOutNotWrittenOff(t *testing.T) {
+	ctx := context.Background()
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	src := NewMockBlockSource()
+	tr := newTestTracker(t, time.Nanosecond, 100, src)
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(ctx, tx)
+
+	// The node has the head but not yet the receipts, then catches up.
+	src.FailTimes(2, ethereum.NotFound)
+	src.SetReceipts(7, blockReceipt{
+		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
+	})
+	tr.matchBlock(ctx, 7, 0, time.Unix(1002, 0))
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Committed,
+		"a height the node had not finished writing was written off")
+	for s := range tr.state.Lock() {
+		require.Zero(t, s.blindHeights,
+			"the watermark rose for a gap that resolved, which makes expired "+
+				"unreachable for every transaction in the window")
+	}
+}
+
+// Requirements: TOT-020 and TOT-023.
+// TestARetryGivesUpAndBecomesAHole fails when a node that never catches up
+// holds the head loop indefinitely.
+//
+// Head processing is serial, so a read that waits forever stops the run reading
+// anything. Past its budget the height is a hole, which is honest and lets the
+// loop move on.
+func TestARetryGivesUpAndBecomesAHole(t *testing.T) {
+	ctx := context.Background()
+	src := NewMockBlockSource().SetFetchErr(ethereum.NotFound)
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	start := time.Now()
+	tr.matchBlock(ctx, 7, 0, time.Unix(1002, 0))
+	spent := time.Since(start)
+
+	require.Less(t, spent, readTimeout*2,
+		"a node that never caught up held the head loop for %s", spent)
+	for s := range tr.state.Lock() {
+		require.Equal(t, uint64(1), s.blindHeights,
+			"a height the node never served was not counted as unread")
+	}
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 1ce2d96..23a6b19 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -1,16 +1,12 @@
 package stats
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
 	"log"
 	"net"
-	"net/http"
-	"strconv"
 	"strings"
-	"syscall"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
@@ -113,14 +109,6 @@ func narrowReceipts(receipts []*ethtypes.Receipt) (out []blockReceipt, nulls int
 	return out, nulls
 }
 
-// deferredRead is a height the receipt node had not reached, waiting to be read
-// again.
-//
-// It carries the arrival of its own head. The re-read happens on a later head,
-// and arrival becomes the inclusion latency sample, so passing the later head's
-// arrival would add one head-to-head interval to every sample. In the two-node
-// topology this tracker recommends, every height is deferred at least once, so
-// that error would land on every sample rather than a few.
 // headArrival is one head and the moment it reached this process.
 type headArrival struct {
 	num     uint64
@@ -128,58 +116,6 @@ type headArrival struct {
 	arrival time.Time
 }
 
-type deferredRead struct {
-	num     uint64
-	gasUsed uint64
-	arrival time.Time
-	// deferredAt is when the run first found the node had not reached this
-	// height, and zero on a first read. The wait is measured from here rather
-	// than from arrival, so a slow first read does not eat the budget and a
-	// re-read does not restart it.
-	deferredAt time.Time
-}
-
-// What became of a height that waited to be read. These are label values on
-// deferred_read_wait. They are not Outcome values and the label is not called
-// outcome, because one label name meaning two disjoint sets across two
-// instruments is the same wire hazard Outcome's own type exists to prevent.
-const (
-	waitRead             = "read"
-	waitAbandoned        = "abandoned"
-	waitUnreadAtShutdown = "unread_at_shutdown"
-)
-
-// How long a height waits to be read, and how many entries can wait.
-//
-// The wait is a duration rather than a count of heads. A count of heads means a
-// different tolerance on every chain, and it makes the failure a cliff: at one
-// head under the bound every block reads, at one head over it every block is a
-// hole. A duration is the quantity that actually matters, which is how far the
-// receipt node trails the head node.
-const (
-	// maxDeferredReadBudget caps how long a height may wait before the run calls
-	// the serving node behind. The live value is deferredReadBudget, which also
-	// answers to the run's reap deadline.
-	//
-	// It is larger than rereadSweepBudget, and that relationship is load-bearing
-	// rather than a coincidence of two tuned numbers. A sweep can spend its whole
-	// budget on the heights ahead of one, so a wait budget shorter than a sweep
-	// retires the tail for a delay this process caused and reports it as the
-	// node being behind. Anything queued gets at least one sweep after the one
-	// it was queued in.
-	maxDeferredReadBudget = 2 * rereadSweepBudget
-	maxDeferredReads      = 64
-	// rereadSweepBudget bounds what one head spends re-reading, whatever is
-	// waiting.
-	//
-	// It is not smaller than readTimeout, and that is load-bearing. A sweep
-	// budget below one read's budget caps every re-read below it, so a node
-	// answering in four seconds never finishes one, and in the topology where
-	// every height is deferred that makes every height a hole. The budget must
-	// leave room for the read it is budgeting for.
-	rereadSweepBudget = readTimeout
-)
-
 type entry struct {
 	tx           *types.LoadTx
 	registeredAt time.Time
@@ -222,12 +158,7 @@ type inclusionState struct {
 	// duplicates counts registrations of a hash already in flight. The registry
 	// holds one slot per hash, so the second one has no place to go and the run
 	// can say nothing about it.
-	duplicates uint64
-	// pending holds heights the receipt node had not reached, waiting to be read
-	// again. It is bounded by maxDeferredReads entries, and an entry leaves once
-	// it is read or once its deferredReadBudget runs out, so a node that stays
-	// behind produces holes rather than a growing queue.
-	pending            []deferredRead
+	duplicates         uint64
 	inflight           map[common.Hash]*entry
 	included           uint64
 	expired            uint64
@@ -421,9 +352,6 @@ func (t *InclusionTracker) Run(ctx context.Context, endpoint string) error {
 		s.SpawnBg(func() error { return t.pumpHeads(ctx, headers, arrivals) })
 
 		var lastSeen uint64 // 0 = unset; first head seeds it (no backfill).
-		// The head loop can end with heights still queued; recordUnreadAtShutdown
-		// owns what happens to them.
-		defer t.recordUnreadAtShutdown(ctx)
 		for ctx.Err() == nil {
 			a, err := utils.Recv(ctx, arrivals)
 			if err != nil {
@@ -457,10 +385,6 @@ func (t *InclusionTracker) Run(ctx context.Context, endpoint string) error {
 // processHead handles one arriving head: counts any gap (no backfill), matches
 // the block, and returns the new lastSeen. lastSeen==0 seeds on the first head.
 func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64, arrival time.Time, lastSeen uint64) uint64 {
-	// The re-read runs before the early return. A repeated or out-of-order head
-	// is no reason to leave a height waiting: nothing else drains the queue, and
-	// a height left in it reaps as a chain verdict for a block never read.
-	t.rereadDeferred(ctx)
 	if lastSeen != 0 && num <= lastSeen {
 		return lastSeen // duplicate or out-of-order head: no re-fetch, no spurious gap.
 	}
@@ -478,49 +402,10 @@ func (t *InclusionTracker) processHead(ctx context.Context, num, gasUsed uint64,
 		// diagnostic a failed run carries.
 		t.recordBlindGap(ctx, lastSeen+1, num-1)
 	}
-	t.matchBlockAttempt(ctx, deferredRead{num: num, gasUsed: gasUsed, arrival: arrival}, readTimeout)
+	t.matchBlock(ctx, num, gasUsed, arrival)
 	return num
 }
 
-// rereadDeferred reads the heights the receipt node had not reached, using each
-// height's own arrival. A height out of budget becomes a hole here, because
-// nothing downstream would notice it was never read.
-//
-// Head processing is serial, so the whole sweep shares one budget rather than
-// each read carrying its own. A dozen heights can be waiting at once, and giving
-// each a full read budget would let one hanging node spend minutes inside a
-// single head. A height the sweep does not reach stays queued for the next one,
-// which costs a head of delay and not a hole.
-func (t *InclusionTracker) rereadDeferred(ctx context.Context) {
-	due := t.takePending()
-	sweepUntil := time.Now().Add(rereadSweepBudget)
-	for i, d := range due {
-		// The deadline bounds the read, not just the decision to start one.
-		// Checking the clock before each read and then letting that read run its
-		// own full budget is not a bound: it is the budget plus one whole read.
-		//
-		// The first entry always gets a full read, because the sweep budget is
-		// not smaller than one read's. That is what guarantees progress: every
-		// sweep finishes at least one height, so a queue whose front entries are
-		// slow cannot spend every sweep on partial reads and finish none.
-		left := time.Until(sweepUntil)
-		if left <= 0 {
-			t.requeue(ctx, due[i:])
-			return
-		}
-		if waited := time.Since(d.deferredAt); waited > t.deferredReadBudget() {
-			// The node has had its budget. It is behind rather than busy, and
-			// the run has to say it could not read this height.
-			inclusionDeferredWait.Record(ctx, waited.Seconds(), metric.WithAttributes(
-				attribute.String("chain_id", t.seiChainID),
-				attribute.String("disposition", waitAbandoned)))
-			t.recordBlindFetch(ctx, d.num, reasonBehind, nil)
-			continue
-		}
-		t.matchBlockAttempt(ctx, d, left)
-	}
-}
-
 // pumpHeads takes heads off the wire and stamps each with the moment it arrived.
 //
 // It is its own step so the stamp cannot drift to the moment of processing. A
@@ -578,128 +463,21 @@ func (t *InclusionTracker) stopTracking(ctx context.Context) {
 		attribute.String("reason", reasonTrackingStopped)))
 }
 
-// deferredReadBudget is how long a height may wait to be read.
-//
-// It never outlasts the deadline the run gives a transaction to reach a block. A
-// transaction past reapAfter is already counted, so a read that comes back later
-// answers a question the run has closed, and it would ask a node for history the
-// run's own bound says it need not keep.
-func (t *InclusionTracker) deferredReadBudget() time.Duration {
-	if t.reapAfter < maxDeferredReadBudget {
-		return t.reapAfter
-	}
-	return maxDeferredReadBudget
-}
-
-// takePending removes every queued height and returns it. Both callers must take
-// and clear in one critical section, so one function owns that.
-func (t *InclusionTracker) takePending() []deferredRead {
-	for s := range t.state.Lock() {
-		due := s.pending
-		s.pending = nil
-		return due
-	}
-	panic("unreachable")
-}
-
-// requeue puts back the heights a sweep ran out of time for, ahead of what the
-// sweep already read.
+// preflight proves the endpoint serves eth_getBlockReceipts before the run
+// spends gas. Without it the run completes, reports every transaction
+// status_unavailable, and exits zero — a blind run that reads as a chain that
+// accepted nothing.
 //
-// Order here is service fairness, not age. A sweep walks front to back, so the
-// entries it re-deferred are exactly the ones that got a turn and the tail is
-// exactly the ones that did not. Putting the tail behind them means the same
-// front entries consume every sweep and a height further back is never asked
-// for at all, then retires on its wait budget as though the node were behind on
-// it. Measured: with three queued heights and a node holding only the third, the
-// third was retired without the run ever issuing a request for it.
-//
-// A height past the cap is recorded rather than dropped. deferHeight already
-// holds the queue at the cap, so nothing reaches that branch today; a height
-// that vanished from it silently would leave blindHeights unmoved and let a tx
-// from that block reap as a verdict about the chain, which is the one thing this
-// file must never do quietly.
-func (t *InclusionTracker) requeue(ctx context.Context, left []deferredRead) {
-	var dropped []deferredRead
-	for s := range t.state.Lock() {
-		s.pending = append(left, s.pending...)
-		if len(s.pending) > maxDeferredReads {
-			dropped = s.pending[maxDeferredReads:]
-			s.pending = s.pending[:maxDeferredReads]
-		}
-	}
-	for _, d := range dropped {
-		t.recordBlindFetch(ctx, d.num, reasonBehind, nil)
-	}
-}
-
-// recordUnreadAtShutdown records that the run ended with heights it never read.
-//
-// It raises the watermark and emits no failure, and the split is deliberate. The
-// watermark is internal and decides attribution: these heights went unread, so
-// nothing reaped afterwards may be called a verdict about the chain, whatever
-// order the shutdown happens to run in. block_fetch_errors is operator-facing
-// and answers "did this run go blind?", and a healthy run draining a queue it
-// was always going to drain is not that.
-func (t *InclusionTracker) recordUnreadAtShutdown(ctx context.Context) {
-	due := t.takePending()
-	if len(due) == 0 {
-		return
-	}
-	for s := range t.state.Lock() {
-		s.blindHeights++
-	}
-	for _, d := range due {
-		inclusionDeferredWait.Record(ctx, time.Since(d.deferredAt).Seconds(),
-			metric.WithAttributes(
-				attribute.String("chain_id", t.seiChainID),
-				attribute.String("disposition", waitUnreadAtShutdown)))
-	}
-}
-
-// deferHeight queues a height to be read again, and reports whether it took it.
-// It refuses once the queue is full, so a node far behind produces holes instead
-// of a queue that grows with the run.
-func (t *InclusionTracker) deferHeight(d deferredRead) bool {
-	for s := range t.state.Lock() {
-		if len(s.pending) >= maxDeferredReads {
-			return false
-		}
-		// A re-deferral keeps the moment the run first found the node behind, so
-		// the budget measures the node's drift rather than restarting per read.
-		if d.deferredAt.IsZero() {
-			d.deferredAt = time.Now()
-		}
-		s.pending = append(s.pending, d)
-		return true
-	}
-	panic("unreachable")
-}
-
-// preflight proves the endpoint can answer a receipts read before the tracker
-// starts matching. Without it the run completes, reports every transaction
-// un-included, and exits zero, which reads as a chain that accepted nothing.
-//
-// It asks for height 0, which proves the endpoint speaks the method and nothing
-// about what state it holds: Sei answers genesis from a constant, ahead of its
-// watermark and its receipt store. That is the point. A probe of the head would
-// fail on a node that is merely a block behind, which is ordinary and not a
-// reason to refuse a run.
-//
-// Two failures refuse the run, and they are the two that stay broken for its
-// whole length. A node that does not answer at all is one: validator and seed
-// modes serve no EVM HTTP, so a run pointed at either finds nothing listening.
-// A node that answers and refuses the method is the other. Anything else is a
-// node busy or behind, which the run reports as it finds.
+// It asks for height 0. sei-chain answers that from a constant, ahead of its
+// watermark and its receipt store, so the probe tests the endpoint and says
+// nothing about the node's sync state: a node merely behind still passes.
+// Nothing but a broken endpoint fails it, which is why any error that survives
+// every attempt refuses the run and no error classifier is needed here.
 func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error {
 	// Several attempts, because one verdict stands for the whole run. A single
 	// reset or timeout from a node that is merely busy must not end a run before
 	// it starts.
-	// A refusal has to hold across every attempt. One attempt that answered at
-	// all, even with an error, proves the endpoint is there and speaks the
-	// method, and that evidence must not be erased by a later timeout.
 	var err error
-	// The empty string means no verdict yet.
-	var refusing string
 	for attempt := range preflightAttempts {
 		if attempt > 0 {
 			if _, waitErr := utils.Recv(ctx, time.After(preflightBackoff)); waitErr != nil {
@@ -712,94 +490,51 @@ func (t *InclusionTracker) preflight(ctx context.Context, endpoint string) error
 		if err == nil {
 			return nil
 		}
-		reason := fetchFailureReason(err)
-		if _, permanent := permanentReasons[reason]; !permanent {
-			// The endpoint answered. Whatever it said, it is there.
-			log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
-			return nil
-		}
-		if refusing != "" && refusing != reason {
-			// Two different permanent-looking causes is not one permanent
-			// cause. Report and let the run say what it finds.
-			log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
-			return nil
-		}
-		refusing = reason
+		log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
 	}
-	if refuse, permanent := permanentReasons[refusing]; permanent {
-		return refuse(endpoint, preflightAttempts, err)
-	}
-	log.Printf("inclusion tracker: preflight against %s: %v", endpoint, err)
-	return nil
+	return fmt.Errorf(
+		"inclusion tracker: %w: %s failed %d eth_getBlockReceipts probes (%v). "+
+			"receiptEndpoint must name the EVM HTTP port of a node in fullNode or "+
+			"archive mode: validator and seed modes serve no EVM HTTP, and the "+
+			"metrics port, the Cosmos RPC port and an ingress error page all answer "+
+			"without serving the method",
+		ErrEndpointUnusable, endpoint, preflightAttempts, err)
 }
 
-// permanentReasons maps a reason that will still be wrong when the run ends to
-// the refusal it produces. One table, because a reason that refuses and a
-// refusal that names it are the same fact: two lists would let a reason be added
-// to one and forgotten in the other, giving either a silent stall through the
-// retry loop or a refusal nothing can reach.
+// readReceipts reads one block's receipts, waiting out a height the node has
+// not yet made readable, and reports how long it waited.
 //
-// Anything absent here is a node busy or behind, and the run reports what it
-// finds rather than refusing to start.
-var permanentReasons = map[string]func(endpoint string, attempts int, err error) error{
-	reasonMethodUnavailable: func(endpoint string, _ int, err error) error {
-		return fmt.Errorf(
-			"inclusion tracker: %w: %s answers, but not eth_getBlockReceipts (%v). "+
-				"Set receiptEndpoint in the profile to a node in fullNode or "+
-				"archive mode, which are the modes that serve EVM HTTP",
-			ErrEndpointUnusable, endpoint, err)
-	},
-	reasonUnreachable: func(endpoint string, _ int, err error) error {
-		return fmt.Errorf(
-			"inclusion tracker: %w: %s is not serving EVM JSON-RPC (%v). "+
-				"Set receiptEndpoint in the profile to a node in fullNode or "+
-				"archive mode; validator and seed modes serve no EVM HTTP",
-			ErrEndpointUnusable, endpoint, err)
-	},
-	reasonNotJSON: func(endpoint string, _ int, err error) error {
-		return fmt.Errorf(
-			"inclusion tracker: %w: %s answered, but not with JSON-RPC (%v). "+
-				"Check that receiptEndpoint names the EVM HTTP port rather than "+
-				"the metrics port, the Cosmos RPC port, or an ingress path",
-			ErrEndpointUnusable, endpoint, err)
-	},
-	reasonTimeout: func(endpoint string, attempts int, err error) error {
-		return fmt.Errorf(
-			"inclusion tracker: %w: %s did not answer %d receipt reads (%v). "+
-				"Check that receiptEndpoint names a reachable node and that "+
-				"nothing between here and it is dropping the connection",
-			ErrEndpointUnusable, endpoint, attempts, err)
-	},
-}
-
-// matchBlock fetches block num once and stamps every in-flight tx it includes
-// with the header-arrival time.
-func (t *InclusionTracker) matchBlock(ctx context.Context, num uint64, arrival time.Time) {
-	t.matchBlockAttempt(ctx, deferredRead{num: num, arrival: arrival}, readTimeout)
+// A not-found is the node's watermark trailing the head it just published, not
+// a missing block. Retrying in place costs a phase shift and no backlog: the
+// wait for height N ends when N becomes readable, which on a healthy node is
+// long before head N+1 arrives.
+func (t *InclusionTracker) readReceipts(
+	ctx context.Context, num uint64,
+) (receipts []blockReceipt, nulls int, waited time.Duration, err error) {
+	readCtx, cancel := context.WithTimeout(ctx, readTimeout)
+	defer cancel()
+	start := time.Now()
+	for {
+		receipts, nulls, err = t.source.BlockReceipts(readCtx, num)
+		if err == nil || fetchFailureReason(err) != reasonNotFound {
+			return receipts, nulls, time.Since(start), err
+		}
+		if _, waitErr := utils.Recv(readCtx, time.After(notFoundBackoff)); waitErr != nil {
+			return nil, 0, time.Since(start), err
+		}
+	}
 }
 
-// matchBlockAttempt is matchBlock over a queued read. It takes the whole
-// deferredRead rather than its fields: arrival and deferredAt are both
-// timestamps, and transposing them as arguments would compile while measuring
-// the latency and the budget from each other's instant.
-func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, d deferredRead, budget time.Duration) {
-	// Explicit per-iteration cancel (not deferred-in-loop): bound the fetch.
-	fetchCtx, cancel := context.WithTimeout(ctx, min(budget, readTimeout))
-	receipts, nulls, err := t.source.BlockReceipts(fetchCtx, d.num)
-	cancel()
+// matchBlock reads block num's receipts and stamps every in-flight tx the block
+// includes with arrival, the moment the head reached this process.
+//
+// seid publishes a head from App.Commit, and eth_getBlockReceipts resolves
+// through a watermark that also waits on the receipt store's async writer, so a
+// height can answer null for the moment between the two. readReceipts waits that
+// out; anything readTimeout does not resolve is a hole.
+func (t *InclusionTracker) matchBlock(ctx context.Context, num, gasUsed uint64, arrival time.Time) {
+	receipts, nulls, waited, err := t.readReceipts(ctx, num)
 	if err != nil {
-		// A read this sweep cut short is not evidence about the node. The
-		// height goes back in the queue: calling it a hole would blame the
-		// serving node for a deadline this process imposed on itself.
-		cutShort := budget < readTimeout && errors.Is(err, context.DeadlineExceeded)
-		if cutShort && t.deferHeight(d) {
-			return
-		}
-		if fetchFailureReason(err) == reasonNotFound && t.deferHeight(d) {
-			// The node has not reached this height. Ordinary, and not a hole
-			// until a re-read says so.
-			return
-		}
 		// The block goes unmatched, and blindHeights records that this run has
 		// a hole, so a tx in flight across it reaps as status_unavailable
 		// rather than as a verdict about the chain.
@@ -810,25 +545,23 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, d deferredRead
 		// ethereum.NotFound. Reading an empty array as a hole would mark every
 		// idle block, and a chain that stopped accepting work produces nothing
 		// but idle blocks, which is the one run where expired is the answer.
-		t.recordBlindFetch(ctx, d.num, fetchFailureReason(err), err)
+		t.recordBlindFetch(ctx, num, fetchFailureReason(err), err)
 		return
 	}
-	if !d.deferredAt.IsZero() {
-		// The node had not reached this height and now has. The distribution of
-		// these waits is how far the receipt node trails the head node, which is
-		// the quantity that decides whether this topology works.
-		inclusionDeferredWait.Record(ctx, time.Since(d.deferredAt).Seconds(),
-			metric.WithAttributes(
-				attribute.String("chain_id", t.seiChainID),
-				attribute.String("disposition", waitRead)))
+	if waited > 0 {
+		// How long the node trailed its own head on this height. The
+		// distribution is the one signal that says whether the serving node
+		// keeps up with the stream it publishes.
+		inclusionReadWait.Record(ctx, waited.Seconds(),
+			metric.WithAttributes(attribute.String("chain_id", t.seiChainID)))
 	}
 	if nulls > 0 {
 		// The read succeeded and part of it is unreadable. Match what arrived,
 		// and record the hole so a transaction the missing part would have named
 		// is not blamed on the chain.
-		t.recordBlindFetch(ctx, d.num, reasonNullReceipt, nil)
+		t.recordBlindFetch(ctx, num, reasonNullReceipt, nil)
 	}
-	if len(receipts) == 0 && d.gasUsed > 0 {
+	if len(receipts) == 0 && gasUsed > 0 {
 		// An empty array usually means the block carried no EVM transaction,
 		// and it is the answer for most blocks on an idle chain. It has one
 		// other cause: sei-chain drops a receipt its store cannot find and
@@ -860,7 +593,7 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, d deferredRead
 			}
 			// Single writer of InclusionTime, under the lock; first observation
 			// wins (delete-on-touch) — see reorg note in sender/doc.go.
-			e.tx.InclusionTime = d.arrival
+			e.tx.InclusionTime = arrival
 			delete(s.inflight, r.Hash)
 			// A receipt carries one status bit, so the outcome names what the run
 			// observed. Every failure cause shares the failed status.
@@ -892,7 +625,7 @@ func (t *InclusionTracker) matchBlockAttempt(ctx context.Context, d deferredRead
 			// bogus epoch-based duration. See LoadTx contract.
 			if t.openLoop && !e.tx.IntendedSendTime.IsZero() {
 				matched = append(matched, inclusionSample{
-					latency:  d.arrival.Sub(e.tx.IntendedSendTime).Seconds(),
+					latency:  arrival.Sub(e.tx.IntendedSendTime).Seconds(),
 					scenario: e.tx.Scenario,
 				})
 			}
@@ -983,12 +716,10 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 			switch {
 			case s.blindHeights > e.blindHeightsAtRegistration,
 				s.trackingStopped,
-				s.headsResolved < s.headsReceived,
-				len(s.pending) > 0:
+				s.headsResolved < s.headsReceived:
 				// Either a height went unread, or the run is holding one it has
-				// not opened: still in the head queue, or waiting on a receipt
-				// node that has not reached it. A transaction may be sitting in
-				// any of them, so the run cannot say the chain left it out.
+				// not opened yet. A transaction may be sitting in either, so the
+				// run cannot say the chain left it out.
 				outcome = OutcomeStatusUnavailable
 				s.statusUnavailable++
 			default:
@@ -1042,23 +773,15 @@ func (t *InclusionTracker) recordBlindFetch(ctx context.Context, num uint64, rea
 // branches on two of them, so one constant owns each string rather than a
 // literal at every site.
 const (
-	reasonMethodUnavailable = "method_unavailable"
-	reasonUnreachable       = "unreachable"
-	reasonPruned            = "pruned"
-	reasonNotFound          = "not_found"
-	reasonTimeout           = "timeout"
-	reasonOther             = "other"
-	// reasonNotJSON is an endpoint that answered with something other than
-	// JSON-RPC: an ingress error page, a metrics port, the Cosmos RPC port. It
-	// stays broken for the run's length, so it refuses the run.
-	reasonNotJSON = "not_json"
+	reasonNotFound = "not_found"
+	reasonTimeout  = "timeout"
+	reasonOther    = "other"
 
 	// These name why a height went unread rather than why a call failed. They
 	// share the counter and not the cause, because the operator's next move
 	// differs: a node behind the head is a topology problem, a head the run
 	// never saw is a subscription problem, and a receipt the node could not
 	// produce is a chain problem.
-	reasonBehind = "receipt_node_behind"
 	// reasonTrackingStopped is the head stream ending mid-run. Every later
 	// height is unread, so nothing after it is evidence about the chain.
 	reasonTrackingStopped = "tracking_stopped"
@@ -1076,6 +799,11 @@ const (
 // two is what a previous shape got wrong.
 const readTimeout = 10 * time.Second
 
+// notFoundBackoff paces the retry of a height the node has not made readable.
+// It is short relative to the block interval, because the wait it paces is the
+// node's own commit-to-watermark gap and not a block.
+const notFoundBackoff = 25 * time.Millisecond
+
 // errHeadStreamEnded marks the head subscription ending on its own, which stops
 // the tracking and not the run. It is separate from ErrEndpointUnusable because
 // the two want opposite outcomes: an endpoint that cannot serve the run should
@@ -1099,11 +827,6 @@ const (
 	preflightBackoff  = 500 * time.Millisecond
 )
 
-// methodNotFoundCode is JSON-RPC 2.0's "Method not found". Every server returns
-// it under that code whatever prose it puts beside it, so the code is what this
-// matches on.
-const methodNotFoundCode = -32601
-
 // fetchFailureReason buckets a receipt-read error so an operator reads the cause
 // off a dashboard instead of the pod log. The strings are label values: keep
 // them few, and keep them stable.
@@ -1114,28 +837,6 @@ const methodNotFoundCode = -32601
 // N" for one and "not yet available" for the other, which differ by one word, so
 // pruning is tested before anything matching on availability.
 func fetchFailureReason(err error) string {
-	var rpcErr rpc.Error
-	if errors.As(err, &rpcErr) && rpcErr.ErrorCode() == methodNotFoundCode {
-		return reasonMethodUnavailable
-	}
-	// A gateway that filters methods answers with an HTTP status, and the
-	// JSON-RPC code sits in the body where the decoder never looks.
-	//
-	// The status gates the body read. A rate-limited response carries a body
-	// too, and a request id inside it can hold these six digits by coincidence,
-	// which would refuse a run against a node that is merely busy. Only a
-	// status that means "I will not serve this" is allowed to speak for the
-	// method.
-	var httpErr rpc.HTTPError
-	if errors.As(err, &httpErr) {
-		switch httpErr.StatusCode {
-		case http.StatusBadRequest, http.StatusForbidden, http.StatusMethodNotAllowed:
-			if bytes.Contains(httpErr.Body, []byte(strconv.Itoa(methodNotFoundCode))) {
-				return reasonMethodUnavailable
-			}
-		}
-		return reasonOther
-	}
 	if errors.Is(err, ethereum.NotFound) {
 		return reasonNotFound
 	}
@@ -1146,46 +847,10 @@ func fetchFailureReason(err error) string {
 	if errors.As(err, &netErr) && netErr.Timeout() {
 		return reasonTimeout
 	}
-	// Unreachable means the endpoint is not there, and it is the only network
-	// bucket allowed to refuse a run. A refused dial and a name that does not
-	// resolve stay broken for the run's length.
-	//
-	// A connection reset does not. It is what a healthy node under load does to
-	// a caller, and reading it as unreachable would kill a run for being busy,
-	// which is worse than the blind run this bucket exists to prevent. It falls
-	// through to other.
-	//
-	// Only a name that does not exist counts. A resolver answering SERVFAIL is
-	// temporary, and refusing a run for it would end a run over a blip in DNS.
-	var dnsErr *net.DNSError
-	if errors.As(err, &dnsErr) {
-		if dnsErr.IsNotFound {
-			return reasonUnreachable
-		}
-		return reasonOther
-	}
-	var opErr *net.OpError
-	if errors.As(err, &opErr) && opErr.Op == "dial" && errors.Is(err, syscall.ECONNREFUSED) {
-		return reasonUnreachable
-	}
-	switch msg := strings.ToLower(err.Error()); {
-	case strings.Contains(msg, "pruned"):
-		return reasonPruned
-	case strings.Contains(msg, "does not exist") || strings.Contains(msg, "not available"):
-		return reasonMethodUnavailable
-	case strings.Contains(msg, "connection refused") ||
-		strings.Contains(msg, "no such host"):
-		return reasonUnreachable
-	case strings.Contains(msg, "looking for beginning of value") ||
-		strings.Contains(msg, "invalid character"):
-		return reasonNotJSON
-	case strings.Contains(msg, "not found"):
+	if strings.Contains(strings.ToLower(err.Error()), "not found") {
 		return reasonNotFound
-	case strings.Contains(msg, "deadline exceeded"):
-		return reasonTimeout
-	default:
-		return reasonOther
 	}
+	return reasonOther
 }
 
 // InclusionSummary is the conservation tally. Read only after both sender and
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index ab3e305..b1f321b 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -170,7 +170,7 @@ func TestInclusion_MatchStamps(t *testing.T) {
 
 	arrival := time.Unix(1002, 0)
 	src.SetBlock(5, tx.EthTx.Hash())
-	tr.matchBlock(context.Background(), 5, arrival)
+	tr.matchBlock(context.Background(), 5, 0, arrival)
 
 	require.Equal(t, arrival, tx.InclusionTime, "InclusionTime is the header-arrival time")
 	require.Equal(t, 0, inflightLen(t, tr), "matched tx leaves the registry")
@@ -191,7 +191,7 @@ func TestInclusion_ClosedLoopCountsNoLatency(t *testing.T) {
 	tr.Register(context.Background(), tx)
 	arrival := time.Unix(1002, 0)
 	src.SetBlock(5, tx.EthTx.Hash())
-	tr.matchBlock(context.Background(), 5, arrival)
+	tr.matchBlock(context.Background(), 5, 0, arrival)
 
 	require.Equal(t, arrival, tx.InclusionTime, "InclusionTime still stamped in closed-loop")
 	require.Equal(t, uint64(1), tr.Summary().Included, "included count tracked in closed-loop")
@@ -225,7 +225,7 @@ func TestInclusion_ReapVsLateInclusion(t *testing.T) {
 		time.Sleep(time.Millisecond)
 		tr.reap(context.Background()) // wins: expired
 		src.SetBlock(5, tx.EthTx.Hash())
-		tr.matchBlock(context.Background(), 5, time.Unix(1002, 0)) // no-op
+		tr.matchBlock(context.Background(), 5, 0, time.Unix(1002, 0)) // no-op
 		s := tr.Summary()
 		require.Equal(t, uint64(1), s.Expired)
 		require.Equal(t, uint64(0), s.Included)
@@ -237,7 +237,7 @@ func TestInclusion_ReapVsLateInclusion(t *testing.T) {
 		tx := loadTx(1, time.Unix(1000, 0))
 		tr.Register(context.Background(), tx)
 		src.SetBlock(5, tx.EthTx.Hash())
-		tr.matchBlock(context.Background(), 5, time.Unix(1002, 0)) // wins: included
+		tr.matchBlock(context.Background(), 5, 0, time.Unix(1002, 0)) // wins: included
 		time.Sleep(time.Millisecond)
 		tr.reap(context.Background()) // no-op
 		s := tr.Summary()
@@ -318,7 +318,7 @@ func TestInclusion_Conservation(t *testing.T) {
 			}
 			for i := 0; i < tc.matched; i++ {
 				src.SetBlock(uint64(i), txs[i].EthTx.Hash())
-				tr.matchBlock(context.Background(), uint64(i), time.Unix(1001, 0))
+				tr.matchBlock(context.Background(), uint64(i), 0, time.Unix(1001, 0))
 			}
 			// Reap the next `reaped` txs by forcing their registeredAt past cutoff.
 			for s := range tr.state.Lock() {
@@ -381,7 +381,7 @@ func TestInclusion_ConcurrentRaceSafe(t *testing.T) {
 	go func() {
 		defer wg.Done()
 		for i := range txs {
-			tr.matchBlock(context.Background(), uint64(i), time.Unix(1001, 0))
+			tr.matchBlock(context.Background(), uint64(i), 0, time.Unix(1001, 0))
 		}
 	}()
 	go func() {
diff --git a/stats/metrics.go b/stats/metrics.go
index d2e417e..6b6c51e 100644
--- a/stats/metrics.go
+++ b/stats/metrics.go
@@ -78,11 +78,11 @@ var (
 		metric.WithDescription("Block heights the head subscription never delivered, counted per height. Not backfilled; the gap is recorded once as a hole on block_fetch_errors{reason=missed_head}"),
 		metric.WithUnit("{blocks}")))
 
-	inclusionDeferredWait = must(meter.Float64Histogram(
-		"deferred_read_wait",
-		metric.WithDescription("How long a height waited for the receipt node to reach it, by disposition. A rising distribution is the receipt node falling behind the head node"),
+	inclusionReadWait = must(meter.Float64Histogram(
+		"block_read_wait",
+		metric.WithDescription("How long a block's receipts took to become readable after its head arrived. seid publishes a head from Commit while the receipt store's writer is still async, so this is that gap. A rising distribution is the node's watermark falling behind its own head"),
 		metric.WithUnit("s"),
-		metric.WithExplicitBucketBoundaries(0.25, 0.5, 1, 2, 5, 10)))
+		metric.WithExplicitBucketBoundaries(0.005, 0.025, 0.1, 0.25, 0.5, 1, 2, 5, 10)))
 
 	inclusionHeadLag = must(meter.Float64Histogram(
 		"head_lag",

From 6ad9351b33e668aeaf91bfc9e1f0b52b417086fc Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 13:52:26 -0700
Subject: [PATCH 16/17] test(stats): drive Run itself, and fix the two defects
 that hid behind it
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Run had no coverage. Every ordering fact seven rounds of fixes depend on — the
pump stamping a head's arrival, the head loop reading its block, the reap
deciding what to call a transaction, a dead subscription ending the tracking —
was exercised only by helpers called by hand. That is why the two worst defects
in this change survived seven rounds of review, and why four of the guards
written along the way turned out to assert nothing.

The harness is an in-process WebSocket server speaking eth_subscribe("newHeads"),
so a test drives the real subscription, the real client and the real loop. It
earned its keep immediately: the first version sent a header the go-ethereum
decoder rejects, and the run reported the subscription dying rather than the
header being wrong. Coverage of Run goes from nothing to 77%.

The first defect it exposes fires on every run. The senders and the tracker start
together, and the tracker dials and probes before it reads a block, so every run
accepts transactions during a window in which it is observing nothing. Those
transactions land in blocks the run never opened, and reaped as expired: a claim
about the chain drawn from a period the run did not watch. The registry records
when the run first read a block, and a reap will not speak for the chain about
anything accepted before that.

The second is what a dead subscription does. The reap loop ends with the head
loop, so nothing reaps afterwards, and the registry kept filling from senders
that were still working until everything reported dropped_at_cap. An operator
reads that as a cap to raise rather than a subscription that died. Tracking
stopping now settles everything in flight, and a later registration is answered
rather than stored.

That made the reap's own trackingStopped arm unreachable, which two reviews had
already called dead for a different reason. It is gone; Register is the only
reader now, and the field says so.

The registry's conservation identity omitted status_unavailable, so it could not
see a transaction migrating into that term. It has five terms now.

Guards proven by breaking what they cover: the first-read watermark and its
placement, the drain, the registration path after tracking stops, and the reap
arms in both directions.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/inclusion_run_test.go     | 287 ++++++++++++++++++++++++++++++++
 stats/inclusion_tracker.go      |  49 +++++-
 stats/inclusion_tracker_test.go |  25 ++-
 3 files changed, 351 insertions(+), 10 deletions(-)
 create mode 100644 stats/inclusion_run_test.go

diff --git a/stats/inclusion_run_test.go b/stats/inclusion_run_test.go
new file mode 100644
index 0000000..c97affa
--- /dev/null
+++ b/stats/inclusion_run_test.go
@@ -0,0 +1,287 @@
+package stats
+
+import (
+	"context"
+	"math/big"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	ethtypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/stretchr/testify/require"
+)
+
+// headServer serves eth_subscribe("newHeads") over WebSocket, so a test can
+// drive InclusionTracker.Run itself rather than the helpers underneath it.
+//
+// Run is where the ordering lives: the pump stamps a head's arrival, the head
+// loop reads its block, the reap decides what to call a transaction, and a dead
+// subscription ends the tracking. None of that is reachable from a helper, and
+// the defects that reached round seven were all in it.
+type headServer struct {
+	srv   *httptest.Server
+	heads chan *ethtypes.Header
+
+	mu    sync.Mutex
+	conns []net.Conn
+}
+
+type headSubscription struct{ heads chan *ethtypes.Header }
+
+// NewHeads backs eth_subscribe("newHeads"): go-ethereum takes the subscription
+// name from the method name.
+func (h *headSubscription) NewHeads(ctx context.Context) (*rpc.Subscription, error) {
+	notifier, ok := rpc.NotifierFromContext(ctx)
+	if !ok {
+		return nil, rpc.ErrNotificationsUnsupported
+	}
+	sub := notifier.CreateSubscription()
+	go func() {
+		for {
+			select {
+			case hdr := <-h.heads:
+				_ = notifier.Notify(sub.ID, hdr)
+			case <-sub.Err():
+				return
+			}
+		}
+	}()
+	return sub, nil
+}
+
+func newHeadServer(t *testing.T) *headServer {
+	t.Helper()
+	heads := make(chan *ethtypes.Header, 64)
+	rpcSrv := rpc.NewServer()
+	require.NoError(t, rpcSrv.RegisterName("eth", &headSubscription{heads: heads}))
+
+	hs := &headServer{heads: heads}
+	hs.srv = httptest.NewUnstartedServer(rpcSrv.WebsocketHandler([]string{"*"}))
+	// A WebSocket is a hijacked connection, which CloseClientConnections does
+	// not reach. Hold the raw conns so a test can drop one the way a node
+	// restart or a proxy eviction does.
+	hs.srv.Config.ConnState = func(c net.Conn, state http.ConnState) {
+		if state != http.StateNew {
+			return
+		}
+		hs.mu.Lock()
+		defer hs.mu.Unlock()
+		hs.conns = append(hs.conns, c)
+	}
+	hs.srv.Start()
+	t.Cleanup(hs.srv.Close)
+	return hs
+}
+
+// endpoint is what a run would be configured with. GetWSEndpoint rewrites the
+// scheme and leaves this port alone, so the tracker dials this same server.
+func (h *headServer) endpoint() string {
+	return strings.Replace(h.srv.URL, "ws://", "http://", 1)
+}
+
+func (h *headServer) send(t *testing.T, num uint64) {
+	t.Helper()
+	// go-ethereum's decoder requires difficulty and the base fee, so a bare
+	// header with only a number is rejected by the client before the tracker
+	// ever sees it.
+	h.heads <- &ethtypes.Header{
+		Number:     new(big.Int).SetUint64(num),
+		Difficulty: new(big.Int),
+		BaseFee:    new(big.Int),
+		Extra:      []byte{},
+	}
+}
+
+// kill drops the WebSocket, which is what a node restart or a proxy eviction
+// looks like to the run.
+func (h *headServer) kill() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	for _, c := range h.conns {
+		_ = c.Close()
+	}
+	h.conns = nil
+}
+
+// runTracker starts Run against the head server with an injected receipt source
+// and returns the tracker plus a stop function.
+func runTracker(
+	t *testing.T, hs *headServer, src receiptSource, reapAfter time.Duration,
+) (*InclusionTracker, context.CancelFunc, <-chan error) {
+	t.Helper()
+	tr := newInclusionTrackerWithSource(
+		NewInclusionTracker("test-chain", reapAfter, 1000, true, NewCollector()), src)
+	ctx, cancel := context.WithCancel(context.Background())
+	done := make(chan error, 1)
+	go func() { done <- tr.Run(ctx, hs.endpoint()) }()
+	t.Cleanup(cancel)
+	return tr, cancel, done
+}
+
+// Requirements: TOT-001, TOT-002 and TOT-022.
+// TestRunReadsAHeadAndResolvesIt is the harness's own proof: a head delivered
+// over a real subscription reaches the reader and its block's receipts resolve.
+func TestRunReadsAHeadAndResolvesIt(t *testing.T) {
+	hs := newHeadServer(t)
+	src := NewMockBlockSource()
+	tr, cancel, done := runTracker(t, hs, src, time.Minute)
+	defer cancel()
+
+	tx := loadTx(1, time.Unix(1000, 0))
+	tr.Register(context.Background(), tx)
+	src.SetReceipts(7, blockReceipt{
+		Hash: tx.EthTx.Hash(), Status: ethtypes.ReceiptStatusSuccessful, HasStatus: true,
+	})
+
+	hs.send(t, 7)
+	require.Eventually(t, func() bool { return tr.Summary().Included == 1 },
+		3*time.Second, 10*time.Millisecond,
+		"a head delivered over a real subscription never reached the reader")
+
+	cancel()
+	<-done
+}
+
+// Requirements: TOT-004 and TOT-008.
+// TestATxAcceptedBeforeTheFirstReadIsNotAChainVerdict fails when a transaction
+// accepted before the tracker read anything reaps as expired.
+//
+// The senders and the tracker start together, and the tracker dials and
+// preflights before it reads a block, so every run accepts transactions during
+// a window in which it is looking at nothing. Those transactions land in blocks
+// the run never opened. Calling that expired is a claim about the chain drawn
+// from a period the run did not observe, and on the shipped profiles it happens
+// on every run.
+func TestATxAcceptedBeforeTheFirstReadIsNotAChainVerdict(t *testing.T) {
+	hs := newHeadServer(t)
+	src := NewMockBlockSource()
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tr, cancel, done := runTracker(t, hs, src, 30*time.Millisecond)
+	defer cancel()
+
+	// Accepted while the tracker is still starting: its block is already behind
+	// the first head the run will ever see.
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), tx)
+
+	// The run starts looking from here. Block 99, which held the transaction, is
+	// never read.
+	for h := uint64(100); h <= 103; h++ {
+		src.SetReceipts(h)
+		hs.send(t, h)
+	}
+
+	require.Eventually(t, func() bool {
+		g := tr.collector.GetOperationStats()[key]
+		return g.Expired+g.StatusUnavailable == 1
+	}, 5*time.Second, 10*time.Millisecond, "the transaction never reached a terminal state")
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a transaction accepted before the run read anything was blamed on the chain")
+	require.Zero(t, got.Expired)
+
+	cancel()
+	<-done
+}
+
+// Requirements: TOT-004.
+// TestADeadSubscriptionStopsTheRunSayingAnything fails when a run whose head
+// stream dies keeps producing chain verdicts, or reports the failure as
+// something else.
+//
+// Nothing after the stream dies is read, so nothing after it is evidence about
+// the chain. The run continues, because the senders are still working and a
+// read-only observer must not fail them.
+func TestADeadSubscriptionStopsTheRunSayingAnything(t *testing.T) {
+	hs := newHeadServer(t)
+	src := NewMockBlockSource()
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	tr, cancel, done := runTracker(t, hs, src, 30*time.Millisecond)
+	defer cancel()
+
+	src.SetReceipts(7)
+	hs.send(t, 7)
+	require.Eventually(t, func() bool { return tr.Summary().Included == 0 && headsSeen(tr) >= 7 },
+		5*time.Second, 10*time.Millisecond, "the first head never landed")
+
+	// Accepted while the run is healthy, then the socket drops.
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), tx)
+	hs.kill()
+
+	// The run does not fail: a read-only observer must not kill the senders.
+	select {
+	case err := <-done:
+		require.NoError(t, err, "a dropped subscription failed the whole run")
+	case <-time.After(5 * time.Second):
+		t.Fatal("Run did not return after the head stream died")
+	}
+
+	time.Sleep(60 * time.Millisecond) // past reapAfter, so the tx is due
+	tr.reap(context.Background())
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.Expired+got.StatusUnavailable,
+		"the transaction never reached a terminal state")
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a run whose head stream died still claimed the chain left a tx out")
+	require.Zero(t, got.Expired)
+}
+
+// headsSeen reports the highest head the tracker has taken off the wire.
+func headsSeen(tr *InclusionTracker) uint64 {
+	for s := range tr.state.Lock() {
+		return s.headsReceived
+	}
+	panic("unreachable")
+}
+
+// Requirements: TOT-003 and TOT-004.
+// TestADeadSubscriptionSettlesEverythingInFlight fails when a run whose head
+// stream died leaves transactions to pile up.
+//
+// The reap loop ends with the head loop, so nothing reaps afterwards. Without a
+// drain the registry fills from senders that are still working and every later
+// transaction reports dropped_at_cap, which tells an operator to raise a cap
+// when the real cause was a dead subscription.
+func TestADeadSubscriptionSettlesEverythingInFlight(t *testing.T) {
+	hs := newHeadServer(t)
+	src := NewMockBlockSource()
+	key := OperationKey{Scenario: "erc20", Operation: "erc20_transfer"}
+
+	// A reap window longer than the test: nothing may depend on a reap running.
+	tr, cancel, done := runTracker(t, hs, src, time.Hour)
+	defer cancel()
+
+	src.SetReceipts(7)
+	hs.send(t, 7)
+	require.Eventually(t, func() bool { return headsSeen(tr) >= 7 },
+		5*time.Second, 10*time.Millisecond, "the first head never landed")
+
+	inFlight := loadTx(1, time.Unix(1000, 0))
+	inFlight.Scenario.Name, inFlight.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), inFlight)
+
+	hs.kill()
+	require.NoError(t, <-done)
+
+	// A sender that has not noticed yet keeps handing transactions over.
+	after := loadTx(2, time.Unix(1000, 0))
+	after.Scenario.Name, after.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), after)
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(2), got.StatusUnavailable,
+		"a dead subscription left transactions unsettled")
+	require.Zero(t, got.DroppedAtCap, "the cause was reported as a full registry")
+	require.Zero(t, tr.Summary().InflightAtShutdown)
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index 23a6b19..c028dc7 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -152,8 +152,18 @@ type inclusionState struct {
 	// had not opened yet.
 	headsReceived uint64
 	headsResolved uint64
+	// firstReadAt is when the run first read a block. Before it the tracker is
+	// dialling and probing while the senders are already sending, so every run
+	// accepts transactions during a window in which it observes nothing.
+	//
+	// A reap compares it against the transaction's own registration. A
+	// transaction accepted before the run started looking may have landed in a
+	// block the run never opened, so the chain cannot be blamed for it.
+	firstReadAt time.Time
 	// trackingStopped means the head stream ended and no later height will be
-	// read at all.
+	// read at all. Register reads it; a reap does not need to, because
+	// stopTracking settles everything in flight and Register admits nothing
+	// after it, so no reap can find a transaction from that period.
 	trackingStopped bool
 	// duplicates counts registrations of a hash already in flight. The registry
 	// holds one slot per hash, so the second one has no place to go and the run
@@ -242,6 +252,13 @@ func (t *InclusionTracker) Register(ctx context.Context, tx *types.LoadTx) {
 	hash := tx.EthTx.Hash()
 	var outcome Outcome
 	for s := range t.state.Lock() {
+		// Nothing further is read, so holding this would only fill the registry
+		// and report dropped_at_cap, which names the wrong cause.
+		if s.trackingStopped {
+			s.statusUnavailable++
+			outcome = OutcomeStatusUnavailable
+			break
+		}
 		// Cap check and insert share one critical section: race-free admission.
 		if len(s.inflight) >= t.maxInflight {
 			s.droppedAtCap++
@@ -452,11 +469,27 @@ func (t *InclusionTracker) noteHeadResolved(num uint64) {
 	}
 }
 
-// stopTracking records that no further height will be read, so nothing reaped
-// after this point can be called a verdict about the chain.
+// stopTracking records that no further height will be read, and settles
+// everything still in flight.
+//
+// The draining is the point. The reap loop ends with the head loop, so nothing
+// reaps after this: without a drain the registry keeps filling from senders that
+// are still working, and every later transaction reports dropped_at_cap. An
+// operator reads that as "raise the cap" rather than "the subscription died".
 func (t *InclusionTracker) stopTracking(ctx context.Context) {
+	var stranded []resolvedOutcome
 	for s := range t.state.Lock() {
 		s.trackingStopped = true
+		for h, e := range s.inflight {
+			delete(s.inflight, h)
+			s.statusUnavailable++
+			stranded = append(stranded, resolvedOutcome{
+				outcome: OutcomeStatusUnavailable, scenario: e.tx.Scenario,
+			})
+		}
+	}
+	for _, r := range stranded {
+		t.report(ctx, r.outcome, r.scenario)
 	}
 	inclusionBlockFetchErrors.Add(ctx, 1, metric.WithAttributes(
 		attribute.String("chain_id", t.seiChainID),
@@ -586,6 +619,11 @@ func (t *InclusionTracker) matchBlock(ctx context.Context, num, gasUsed uint64,
 	matched := make([]inclusionSample, 0, len(receipts))
 	resolved := make([]resolvedOutcome, 0, len(receipts))
 	for s := range t.state.Lock() {
+		if s.firstReadAt.IsZero() {
+			// The run is looking from here. Anything accepted earlier could be
+			// in a block it never opened.
+			s.firstReadAt = time.Now()
+		}
 		for _, r := range receipts {
 			e, ok := s.inflight[r.Hash]
 			if !ok {
@@ -715,8 +753,9 @@ func (t *InclusionTracker) reap(ctx context.Context) {
 			outcome := OutcomeExpired
 			switch {
 			case s.blindHeights > e.blindHeightsAtRegistration,
-				s.trackingStopped,
-				s.headsResolved < s.headsReceived:
+				s.headsResolved < s.headsReceived,
+				s.firstReadAt.IsZero(),
+				s.firstReadAt.After(e.registeredAt):
 				// Either a height went unread, or the run is holding one it has
 				// not opened yet. A transaction may be sitting in either, so the
 				// run cannot say the chain left it out.
diff --git a/stats/inclusion_tracker_test.go b/stats/inclusion_tracker_test.go
index b1f321b..7a9ffb9 100644
--- a/stats/inclusion_tracker_test.go
+++ b/stats/inclusion_tracker_test.go
@@ -120,10 +120,22 @@ func newTestTracker(t *testing.T, reapAfter time.Duration, maxInflight int, src
 	return newTestTrackerLoop(t, reapAfter, maxInflight, src, true)
 }
 
+// newTestTrackerLoop builds a tracker that has already read a block.
+//
+// A run that has not read anything yet cannot call a transaction expired: it may
+// have landed in a block the run never opened, which is what happens on every
+// real run while the tracker is still dialling. These tests are about what
+// happens once the run is underway, so they start there.
+// TestATxAcceptedBeforeTheFirstReadIsNotAChainVerdict covers the other case
+// through Run itself.
 func newTestTrackerLoop(t *testing.T, reapAfter time.Duration, maxInflight int, src receiptSource, openLoop bool) *InclusionTracker {
 	t.Helper()
-	return newInclusionTrackerWithSource(
+	tr := newInclusionTrackerWithSource(
 		NewInclusionTracker("test-chain", reapAfter, maxInflight, openLoop, NewCollector()), src)
+	for s := range tr.state.Lock() {
+		s.firstReadAt = time.Now().Add(-time.Hour)
+	}
+	return tr
 }
 
 // loadTx builds a LoadTx with a deterministic hash from nonce and an intended
@@ -334,11 +346,14 @@ func TestInclusion_Conservation(t *testing.T) {
 			tr.reap(context.Background())
 
 			s := tr.Summary()
-			// dropped_at_cap is excluded from the registered set, so every Register
-			// attempt is accounted by exactly one of the four buckets.
+			// Every Register attempt is accounted by exactly one bucket.
+			// status_unavailable is one of them: a reap reaches it whenever the
+			// run could not see, and an identity that omits it cannot notice a
+			// leg migrating there.
 			require.Equal(t, uint64(tc.attempts),
-				s.Included+s.Expired+s.InflightAtShutdown+s.DroppedAtCap,
-				"attempts == included + expired + inflight_at_shutdown + dropped_at_cap")
+				s.Included+s.Expired+s.StatusUnavailable+s.InflightAtShutdown+s.DroppedAtCap,
+				"attempts == included + expired + status_unavailable + "+
+					"inflight_at_shutdown + dropped_at_cap")
 		})
 	}
 }

From a44f34cf0eeafa784bd0e6fdc30278be2fdf6277 Mon Sep 17 00:00:00 2001
From: bdchatham <bdchatham@gmail.com>
Date: Thu, 27 Aug 2026 14:58:22 -0700
Subject: [PATCH 17/17] test(stats): guard the two mechanisms that keep expired
 reachable

Round eight found that both could be disabled with the suite green, and that is
the over-reporting direction: every un-matched transaction in every run reports
status_unavailable, and nothing says so.

One transaction cannot catch it. The first-read watermark and marking a head
resolved both move a single transaction's verdict the same way, so a test with
one subject cannot tell a working watermark from a dead one. The guard drives Run
with two: one accepted before the run read anything, which must not be blamed on
the chain, and one accepted after and never included, which must be. A third
covers a subscription that is up and has delivered no head, where every other
watermark reads as healthy because the run is trivially caught up with the
nothing it has seen.

The drain reached the metric ledger and not the closing log line, and no test
compared them. That is one transaction counted under two names, which the reap
path has been guarded against for several rounds and the drain path had not.

An ingress error page was retried three hundred and eighty-five times over ten
seconds. rpc.HTTPError renders as "404 Not Found: 404 page not found", so the
substring check read it as a height the node had not reached yet. Deleting the
classifier's typed checks left that substring first to match. The typed checks
are back above it, and a table pins which reasons the read retries: only the one
that means the node will answer differently in a moment.

The package doc still described the deleted queue, named four tests that no
longer exist, and said two goroutines drive the run where there are three. It
also claimed everything in flight at shutdown is inflight_at_shutdown, which
stopped being true when a dead stream started settling them.

Guards proven by breaking what they cover: the watermark unset, its arm removed,
heads never resolved, the drain's counter, and the error page.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 stats/doc.go                    | 37 +++++++-------
 stats/inclusion_outcome_test.go | 56 ++++++++++++++++++++
 stats/inclusion_run_test.go     | 90 +++++++++++++++++++++++++++++++++
 stats/inclusion_tracker.go      | 36 ++++++++-----
 4 files changed, 189 insertions(+), 30 deletions(-)

diff --git a/stats/doc.go b/stats/doc.go
index 0201176..1686b91 100644
--- a/stats/doc.go
+++ b/stats/doc.go
@@ -42,10 +42,9 @@
 //     RecordOutcome take it themselves; recordOperation is called with it
 //     already held.
 //   - TPSWindow.mu guards the rolling rate window.
-//   - InclusionTracker.state guards the in-flight registry and the queue of
-//     heights waiting to be read again. That queue holds at most
-//     maxDeferredReads entries, and an entry leaves it once the height is read
-//     or once deferredReadBudget runs out.
+//   - InclusionTracker.state guards the in-flight registry and the watermarks a
+//     reap consults: the first height the run read, the highest head it has
+//     taken off the wire, and the highest whose block it has read.
 //
 // The rule: report an outcome to the collector outside the tracker's registry
 // lock. The sender blocks on that lock at every send completion, so work held
@@ -66,11 +65,17 @@
 // that never answers, refuses the method, or replies with something other than
 // JSON-RPC ends the run with ErrEndpointUnusable rather than letting it report
 // numbers it did not measure. Otherwise Run subscribes to new heads, and then
-// two goroutines drive it: the head loop matches each arriving block once, and the reap loop
-// evicts transactions that outlived reapAfter. Both end when the run context
-// does. A transaction still in the registry at that point reached no terminal
-// state and is counted as inflight_at_shutdown, which is why the conservation
-// identity is only readable after both have joined.
+// three goroutines drive it: a pump stamping each head's arrival, the head loop
+// matching each arriving block once, and the reap loop evicting transactions
+// that outlived reapAfter. All three end when the run context does, and a
+// transaction still in the registry then is counted as inflight_at_shutdown,
+// which is why the conservation identity is only readable after they have
+// joined.
+//
+// A head stream that ends on its own is the exception. It stops the tracking and
+// not the run, and everything in flight is settled as status_unavailable rather
+// than left to fill the registry: nothing later is read, so nothing later is
+// evidence about the chain.
 //
 // # Ownership boundaries
 //
@@ -93,13 +98,9 @@
 // TestOutcomesPartitionEveryAcceptedTx asserts that Outcome's terminal states
 // partition every accepted transaction.
 //
-// The deferred-read queue's bounds are guarded too:
-// TestOneHeadIsBoundedAndLosesNoHeight holds one head's sweep inside its budget
-// and proves no queued height leaves unrecorded.
-// TestAHeightOutOfBudgetBecomesAHole is the one that exercises the wait budget;
-// TestAReceiptNodeThatStaysBehindBecomesAHole exercises the queue cap, which is
-// the bound that fires first on a fast chain.
-// TestAWaitBudgetOutlastsASweep pins the two budgets in the order the design
-// needs, since either inverted turns this run's own scheduling into a verdict
-// about the serving node.
+// The watermarks that keep expired reachable are guarded through Run itself, by
+// TestTheWatermarksKeepExpiredReachable: it drives one transaction accepted
+// before the run read anything and one accepted after, because a single
+// transaction moves both ways at once and cannot tell a working watermark from
+// a dead one.
 package stats
diff --git a/stats/inclusion_outcome_test.go b/stats/inclusion_outcome_test.go
index 2630e70..3e1cd7d 100644
--- a/stats/inclusion_outcome_test.go
+++ b/stats/inclusion_outcome_test.go
@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/sei-protocol/sei-load/types"
 
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
@@ -806,3 +807,58 @@ func TestARetryGivesUpAndBecomesAHole(t *testing.T) {
 			"a height the node never served was not counted as unread")
 	}
 }
+
+// Requirements: TOT-020.
+// TestOnlyAnUnreachedHeightIsRetried pins which errors the read retries.
+//
+// not_found is the one reason that loops, because it means the node has not
+// finished writing that height and will answer differently in a moment.
+// Anything else answered as well as it ever will, and retrying it spends the
+// whole read budget on hundreds of requests against a node that is already
+// wrong — pointed at a method whose cost grows with the block's transaction
+// count.
+func TestOnlyAnUnreachedHeightIsRetried(t *testing.T) {
+	cases := []struct {
+		name    string
+		err     error
+		want    string
+		retried bool
+	}{
+		{"node_has_not_reached_it", ethereum.NotFound, reasonNotFound, true},
+		{"ingress_404_page", rpc.HTTPError{
+			StatusCode: 404, Status: "404 Not Found", Body: []byte("404 page not found"),
+		}, reasonOther, false},
+		{"gateway_filters_the_method", rpc.HTTPError{
+			StatusCode: 403, Status: "403 Forbidden",
+			Body: []byte(`{"error":{"code":-32601,"message":"Method not found"}}`),
+		}, reasonOther, false},
+		{"read_timed_out", context.DeadlineExceeded, reasonTimeout, false},
+		{"connection_reset", errors.New("read tcp: connection reset by peer"), reasonOther, false},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := fetchFailureReason(tc.err)
+			require.Equal(t, tc.want, got)
+			require.Equal(t, tc.retried, got == reasonNotFound,
+				"an error that answered as well as it ever will is retried")
+		})
+	}
+}
+
+// Requirements: TOT-020.
+// TestAnErrorThatWillNotChangeCostsOneRequest fails when a permanent answer is
+// retried for the whole read budget.
+func TestAnErrorThatWillNotChangeCostsOneRequest(t *testing.T) {
+	src := NewMockBlockSource().SetFetchErr(rpc.HTTPError{
+		StatusCode: 404, Status: "404 Not Found", Body: []byte("404 page not found"),
+	})
+	tr := newTestTracker(t, time.Minute, 100, src)
+
+	start := time.Now()
+	tr.matchBlock(context.Background(), 7, 0, time.Unix(1002, 0))
+	spent := time.Since(start)
+
+	require.Equal(t, int64(1), src.FetchCount(),
+		"a permanent answer cost %d requests", src.FetchCount())
+	require.Less(t, spent, time.Second, "a permanent answer held the head loop for %s", spent)
+}
diff --git a/stats/inclusion_run_test.go b/stats/inclusion_run_test.go
index c97affa..957da32 100644
--- a/stats/inclusion_run_test.go
+++ b/stats/inclusion_run_test.go
@@ -282,6 +282,96 @@ func TestADeadSubscriptionSettlesEverythingInFlight(t *testing.T) {
 	got := tr.collector.GetOperationStats()[key]
 	require.Equal(t, uint64(2), got.StatusUnavailable,
 		"a dead subscription left transactions unsettled")
+	require.Equal(t, got.StatusUnavailable, tr.Summary().StatusUnavailable,
+		"the drain reached the metric ledger and not the closing log line, so "+
+			"one transaction is counted under two names")
 	require.Zero(t, got.DroppedAtCap, "the cause was reported as a full registry")
 	require.Zero(t, tr.Summary().InflightAtShutdown)
 }
+
+// Requirements: TOT-004 and TOT-008.
+// TestTheWatermarksKeepExpiredReachable drives two transactions through Run: one
+// accepted before the run read anything, one after.
+//
+// One transaction cannot tell these apart. Both mechanisms that make expired
+// reachable — the first-read watermark and marking a head resolved — can be
+// disabled without a single-transaction test noticing, because every verdict
+// moves the same way. Then a run reports status_unavailable for everything it
+// ever reaps, which is the over-reporting direction, and it is silent.
+func TestTheWatermarksKeepExpiredReachable(t *testing.T) {
+	hs := newHeadServer(t)
+	src := NewMockBlockSource()
+	tr, cancel, done := runTracker(t, hs, src, 40*time.Millisecond)
+	defer cancel()
+
+	before := OperationKey{Scenario: "s", Operation: "before"}
+	after := OperationKey{Scenario: "s", Operation: "after"}
+
+	// Accepted while the run is still starting. Its block is behind the first
+	// head the run will ever see.
+	early := loadTx(1, time.Unix(1000, 0))
+	early.Scenario.Name, early.Scenario.Operation = before.Scenario, before.Operation
+	tr.Register(context.Background(), early)
+
+	// The run starts looking here.
+	src.SetReceipts(100)
+	hs.send(t, 100)
+	require.Eventually(t, func() bool { return headsSeen(tr) >= 100 },
+		5*time.Second, 5*time.Millisecond, "the first head never landed")
+
+	// Accepted with the run underway, and never included.
+	late := loadTx(2, time.Unix(1000, 0))
+	late.Scenario.Name, late.Scenario.Operation = after.Scenario, after.Operation
+	tr.Register(context.Background(), late)
+
+	for h := uint64(101); h <= 104; h++ {
+		src.SetReceipts(h)
+		hs.send(t, h)
+	}
+	require.Eventually(t, func() bool { return headsSeen(tr) >= 104 },
+		5*time.Second, 5*time.Millisecond, "the later heads never landed")
+
+	time.Sleep(80 * time.Millisecond)
+	tr.reap(context.Background())
+
+	stats := tr.collector.GetOperationStats()
+	require.Equal(t, uint64(1), stats[before].StatusUnavailable,
+		"a transaction accepted before the run read anything was blamed on the chain")
+	require.Equal(t, uint64(1), stats[after].Expired,
+		"a run that read every head it saw could not say the chain left a tx out; "+
+			"expired is unreachable")
+
+	cancel()
+	<-done
+}
+
+// Requirements: TOT-004.
+// TestARunThatReadNothingCannotSpeakForTheChain fails when a run whose
+// subscription is up but has delivered no head calls a transaction expired.
+//
+// It is reachable: the socket connects, the chain is quiet or the node is not
+// publishing, and reapAfter elapses. Every other watermark reads as healthy —
+// nothing failed, no head was missed, and the run is trivially caught up with
+// the nothing it has seen.
+func TestARunThatReadNothingCannotSpeakForTheChain(t *testing.T) {
+	hs := newHeadServer(t)
+	tr, cancel, done := runTracker(t, hs, NewMockBlockSource(), 30*time.Millisecond)
+	defer cancel()
+
+	key := OperationKey{Scenario: "s", Operation: "o"}
+	tx := loadTx(1, time.Unix(1000, 0))
+	tx.Scenario.Name, tx.Scenario.Operation = key.Scenario, key.Operation
+	tr.Register(context.Background(), tx)
+
+	// No head is ever sent.
+	time.Sleep(60 * time.Millisecond)
+	tr.reap(context.Background())
+
+	got := tr.collector.GetOperationStats()[key]
+	require.Equal(t, uint64(1), got.StatusUnavailable,
+		"a run that read no block at all claimed the chain left a tx out")
+	require.Zero(t, got.Expired)
+
+	cancel()
+	<-done
+}
diff --git a/stats/inclusion_tracker.go b/stats/inclusion_tracker.go
index c028dc7..6e21400 100644
--- a/stats/inclusion_tracker.go
+++ b/stats/inclusion_tracker.go
@@ -316,11 +316,10 @@ func (t *InclusionTracker) meterOutcome(ctx context.Context, outcome Outcome, sc
 // without reporting unhealthy. Every height would then arrive before the reading
 // node held it.
 //
-// The tracker reads the height it just received as a head, and re-reads a height
-// the serving node had not reached yet. A height is only re-read while it is
-// younger than deferredReadBudget, so the reach back is seconds and the serving
-// node's receipt retention does not bound it. A change that reaches further back
-// does.
+// The tracker reads the height it just received as a head, waiting in place if
+// the node has not finished writing it. It never asks for an older height, so
+// the serving node's receipt retention does not bound it. A change that reaches
+// further back does.
 func (t *InclusionTracker) Run(ctx context.Context, endpoint string) error {
 	wsEndpoint := utils.GetWSEndpoint(endpoint)
 	if t.source == nil {
@@ -828,14 +827,12 @@ const (
 	reasonNullReceipt     = "null_receipt"
 )
 
-// readTimeout bounds one first read of a height. A re-read gets whatever is left
-// of the sweep instead, so the sweep's budget is a real ceiling on what one head
-// costs rather than a ceiling plus one more read.
+// readTimeout bounds one height's whole read, retries included. Head processing
+// is serial, so this is what one head costs at worst.
 //
-// A first read is not shortened: in the two-node topology this tracker
-// recommends, a height's first read returns not-found cheaply and the re-read is
-// the one that carries the receipts, so a shorter budget on the wrong one of the
-// two is what a previous shape got wrong.
+// It should stay below the run's reap deadline. A read that returns after the
+// reap has already evicted a transaction answers a question the run has closed,
+// and converts a real inclusion into status_unavailable.
 const readTimeout = 10 * time.Second
 
 // notFoundBackoff paces the retry of a height the node has not made readable.
@@ -876,6 +873,9 @@ const (
 // N" for one and "not yet available" for the other, which differ by one word, so
 // pruning is tested before anything matching on availability.
 func fetchFailureReason(err error) string {
+	// not_found is the only reason the read retries, so anything that reaches it
+	// by accident is retried for the whole read budget against a node that will
+	// never answer differently.
 	if errors.Is(err, ethereum.NotFound) {
 		return reasonNotFound
 	}
@@ -886,6 +886,18 @@ func fetchFailureReason(err error) string {
 	if errors.As(err, &netErr) && netErr.Timeout() {
 		return reasonTimeout
 	}
+	// An answer with a status is the server refusing, whatever prose it chose.
+	// rpc.HTTPError.Error() renders as "404 Not Found: 404 page not found", so
+	// the substring below would read an ingress error page as a height the node
+	// has not reached, and retry it hundreds of times.
+	var httpErr rpc.HTTPError
+	if errors.As(err, &httpErr) {
+		return reasonOther
+	}
+	var rpcErr rpc.Error
+	if errors.As(err, &rpcErr) {
+		return reasonOther
+	}
 	if strings.Contains(strings.ToLower(err.Error()), "not found") {
 		return reasonNotFound
 	}