Fix defects found since 0.7.0

changelog.d/post-070-review.md

@@ -0,0 +1,34 @@
+Fixed
+
+- **Esplora `after_txid` is 422 when that tx is not in the script's history.**
+  A cursor that exists somewhere else on the chain used to restart page 1.
+  `/txs`, `/txs/chain`, `/txs/summary`, the address routes, and a multi
+  POST now return `after_txid not found` and no rows.
+- **`estimatesmartfee`, `estimaterawfee`, and Electrum `blockchain.estimatefee`
+  use the 2-block rate for target 2.** Target 0 is still the next-block
+  horizon. Target 1 stays the 1-block rate.
+- **A shared orphan survives the other announcer's reserve.** Evicting
+  peer B drops only B. Peer A's copy is still delivered when the parent
+  arrives.
+- **A tip shrink clamps the spend-durable marker.** Open revalidation and
+  spend replay lower a marker that sits above the surviving tip, so a
+  later confirm still annotates spends. A checkpoint cannot publish the
+  pre-disconnect height over that clamp.
+- **A newest-first scripthash page stops at the page edge.** An unspent
+  tail no longer re-reads every older create.
+- **A tip or compact block with a repeated transaction pair is not a
+  block.** The merkle root can still match (CVE-2012-2459). Tip follow
+  disconnects that peer. Compact reconstruction returns the hash to
+  `getdata`.
+- **`submitblock` of a sibling that spends a coin the tip also spent is
+  inconclusive.** That header is not cached as `duplicate-invalid`.
+- **A refused local I2P SAM port does not rotate the session.** The dial
+  error is no longer classified as a dead `STREAM CONNECT`. A SAM reply
+  of `INVALID_ID` still is.
+- **Outbound dial keeps one onion or I2P seat when clearnet fills the
+  batch.** A dead overlay peer is recorded on its real address. An
+  unspecified version socket is not inserted into addrman.
+- **An Esplora singleflight join does not put an older scripthash back
+  over a newer last-1** for the same client. That includes a leader that
+  is still inside its handler when the newer script finishes, and a waiter
+  that resumes after it.

crates/rbitcoin-consensus/src/confirm_run/write.rs

@@ -399,6 +399,20 @@ pub fn replay_spend_annotations(query: &Query) -> Result<u32, ConsensusError> {
     let Some(tip) = query.tip_height().map(|h| h.0) else {
         return Ok(0);
     };
+    // A marker above the tip is not a cursor for this chain. Shrink and a
+    // crash between the tip flush and the clamp both leave that file behind.
+    // Lower it before the `a == tip` skip, or the next confirm's spends are
+    // never rewritten.
+    let annotated = query
+        .store()
+        .spend_annotated_through()
+        .map_err(ConsensusError::from)?;
+    if annotated.is_some_and(|h| h > tip) {
+        query
+            .store()
+            .clamp_spend_durable()
+            .map_err(ConsensusError::from)?;
+    }
     let annotated = query
         .store()
         .spend_annotated_through()

crates/rbitcoin-electrum/src/server.rs

@@ -1785,14 +1785,17 @@ fn dispatch_pinned(
                 sh_join,
                 |q, view| {
                     q.scripthash_history_filtered_in(&sh, &filter, view)
+                        .map(|page| page.rows)
                         .map_err(|e| e.to_string())
                 },
                 |q, slot, view| {
                     q.scripthash_history_filtered_slot_in(&sh, &filter, slot, view)
+                        .map(|page| page.rows)
                         .map_err(|e| e.to_string())
                 },
                 |q, slot| {
                     q.scripthash_history_filtered_slot(&sh, &filter, slot)
+                        .map(|page| page.rows)
                         .map_err(|e| e.to_string())
                 },
             )?;

crates/rbitcoin-esplora/src/esplora_http_journey.rs

@@ -112,6 +112,27 @@ async fn wallet_pages_after_txid(addr: SocketAddr, sh1: &str) {
     }
     let (st, body) = http_get(addr, &format!("/scripthash/{sh1}/txs?after_txid=zz")).await;
     assert_eq!(st, 422, "{body}");
+    let (st, body) = http_get(addr, &format!("/scripthash/{sh1}/txs/chain/{t3}")).await;
+    assert_eq!(st, 200, "{body}");
+    let page: Vec<Value> = serde_json::from_str(&body).unwrap();
+    let ids: Vec<&str> = page.iter().filter_map(|v| v["txid"].as_str()).collect();
+    assert_eq!(ids, vec![t1.as_str()]);
+
+    foreign_chain_cursor_is_not_this_page(addr, sh1, &t1, &t3).await;
+}
+
+async fn foreign_chain_cursor_is_not_this_page(addr: SocketAddr, sh1: &str, t1: &str, t3: &str) {
+    let t2 = block_hash_hex(&pay_txid(0x22));
+    for path in [
+        format!("/scripthash/{sh1}/txs?after_txid={t2}"),
+        format!("/scripthash/{sh1}/txs/summary?after_txid={t2}"),
+        format!("/scripthash/{sh1}/txs/chain/{t2}"),
+    ] {
+        let (st, body) = http_get(addr, &path).await;
+        assert_eq!(st, 422, "{path}: {body}");
+        assert!(body.contains("after_txid not found"), "{body}");
+        assert!(!body.contains(t1) && !body.contains(t3), "{path}: {body}");
+    }
 }
 
 async fn wallet_posts_scripthashes(addr: SocketAddr, a: [&str; 2], sh: [&str; 2]) {
@@ -150,6 +171,7 @@ async fn wallet_posts_scripthashes(addr: SocketAddr, a: [&str; 2], sh: [&str; 2]
     .await;
     assert_eq!(st, 422, "{resp}");
     assert!(resp.contains("after_txid not found"), "{resp}");
+    cursor_outside_one_script_is_422(addr, a[0], sh[0], &t1, &t2, &t3).await;
     let too: Vec<String> = (0..301).map(|_| "aa".repeat(32)).collect();
     let (st, resp) = http_post(
         addr,
@@ -161,6 +183,24 @@ async fn wallet_posts_scripthashes(addr: SocketAddr, a: [&str; 2], sh: [&str; 2]
     assert!(resp.contains("body too long"), "{resp}");
 }
 
+async fn cursor_outside_one_script_is_422(
+    addr: SocketAddr,
+    address: &str,
+    sh: &str,
+    t1: &str,
+    t2: &str,
+    t3: &str,
+) {
+    let only = serde_json::to_vec(&json!([sh])).unwrap();
+    let (st, resp) = http_post(addr, &format!("/scripthashes/txs?after_txid={t2}"), &only).await;
+    assert_eq!(st, 422, "{resp}");
+    assert!(resp.contains("after_txid not found"), "{resp}");
+    assert!(!resp.contains(t1) && !resp.contains(t3), "{resp}");
+    let (st, resp) = http_get(addr, &format!("/address/{address}/txs?after_txid={t2}")).await;
+    assert_eq!(st, 422, "{resp}");
+    assert!(resp.contains("after_txid not found"), "{resp}");
+}
+
 /// Packed SH `/txs` runs on `spawn_blocking`, so tip height still answers on
 /// the test's single worker.
 async fn tip_height_overlaps_scripthash_txs(pad: &HttpPad) {

crates/rbitcoin-esplora/src/handlers.rs

@@ -1067,19 +1067,22 @@ fn summary_page_sh(
         }
     }
     let filter = HistoryFilter::esplora_chain_page(after);
-    let (items, view) = match sh_at_view(
+    let (page, view) = match sh_at_view(
         st,
         sh,
         asof,
         |q, view| q.scripthash_history_summary_filtered_in(sh, &filter, view),
         |q, slot, view| q.scripthash_history_summary_filtered_slot_in(sh, &filter, slot, view),
-        Vec::new(),
+        cursor_page(after),
         client,
         None,
     ) {
         Ok(x) => x,
         Err(r) => return r,
     };
+    let Some(items) = rows_or_cursor_missing(page) else {
+        return after_txid_not_found();
+    };
     maybe_attach_view(
         match summaries_json(&st.query, &items) {
             Ok(v) => Json(v).into_response(),
@@ -1137,19 +1140,22 @@ fn chain_page_sh(
     client: Option<&str>,
 ) -> Response {
     let filter = HistoryFilter::esplora_chain_page(after);
-    let (items, view) = match sh_at_view(
+    let (page, view) = match sh_at_view(
         st,
         sh,
         asof,
         |q, view| q.scripthash_history_filtered_in(sh, &filter, view),
         |q, slot, view| q.scripthash_history_filtered_slot_in(sh, &filter, slot, view),
-        Vec::new(),
+        cursor_page(after),
         client,
         None,
     ) {
         Ok(x) => x,
         Err(r) => return r,
     };
+    let Some(items) = rows_or_cursor_missing(page) else {
+        return after_txid_not_found();
+    };
     maybe_attach_view(
         match history_items_to_tx_json(&st.query, &items, st.network) {
             Ok(v) => Json(v).into_response(),
@@ -1171,33 +1177,42 @@ fn combined_txs(
             return after_txid_not_found();
         }
     }
+    // Membership is this script's mempool rows. A tx sitting in the pool for
+    // some other script must not restart this script's first page.
+    let mempool_rows = if asof.is_none() {
+        mempool_txs_json(st, sh)
+    } else {
+        Vec::new()
+    };
     let after_in_mempool = after.is_some_and(|id| {
-        let tid = Txid::from_byte_array(id);
-        st.mempool.as_ref().is_some_and(|m| m.contains(&tid))
+        let hex = block_hash_hex(&id);
+        mempool_rows.iter().any(|v| v["txid"] == hex)
     });
     let mut out = Vec::new();
     if asof.is_none() && (after.is_none() || after_in_mempool) {
-        let rows = mempool_txs_json(st, sh);
         out.extend(match after {
-            Some(id) if after_in_mempool => skip_mempool_after(rows, &id),
-            _ => rows,
+            Some(id) if after_in_mempool => skip_mempool_after(mempool_rows, &id),
+            _ => mempool_rows,
         });
     }
     let chain_after = if after_in_mempool { None } else { after };
     let filter = HistoryFilter::esplora_chain_page(chain_after);
-    let (items, view) = match sh_at_view(
+    let (page, view) = match sh_at_view(
         st,
         sh,
         asof,
         |q, view| q.scripthash_history_filtered_in(sh, &filter, view),
         |q, slot, view| q.scripthash_history_filtered_slot_in(sh, &filter, slot, view),
-        Vec::new(),
+        cursor_page(chain_after),
         client,
         None,
     ) {
         Ok(x) => x,
         Err(r) => return r,
     };
+    let Some(items) = rows_or_cursor_missing(page) else {
+        return after_txid_not_found();
+    };
     maybe_attach_view(
         match history_items_to_tx_json(&st.query, &items, st.network) {
             Ok(chain) => {
@@ -1274,14 +1289,29 @@ fn sort_txs_newest_first(rows: &mut [Value]) {
     });
 }
 
-fn skip_rows_after(rows: Vec<Value>, after: Option<[u8; 32]>) -> Vec<Value> {
+/// `None`: `after` was set and is not in `rows`. The first page is not a
+/// stand-in for a cursor this response does not contain.
+fn skip_rows_after(rows: Vec<Value>, after: Option<[u8; 32]>) -> Option<Vec<Value>> {
     let Some(id) = after else {
-        return rows;
+        return Some(rows);
     };
     let hex = block_hash_hex(&id);
-    match rows.iter().position(|v| v["txid"] == hex) {
-        Some(i) => rows[i.saturating_add(1)..].to_vec(),
-        None => rows,
+    let i = rows.iter().position(|v| v["txid"] == hex)?;
+    Some(rows[i.saturating_add(1)..].to_vec())
+}
+
+fn cursor_page<T>(after: Option<[u8; 32]>) -> rbitcoin_query::FilteredHistory<T> {
+    rbitcoin_query::FilteredHistory {
+        rows: Vec::new(),
+        cursor_missing: after.is_some(),
+    }
+}
+
+fn rows_or_cursor_missing<T>(page: rbitcoin_query::FilteredHistory<T>) -> Option<Vec<T>> {
+    if page.cursor_missing {
+        None
+    } else {
+        Some(page.rows)
     }
 }
 
@@ -1311,16 +1341,17 @@ fn combined_tx_vec(
         out.extend(mempool_txs_json(st, sh));
     }
     let filter = HistoryFilter::esplora_chain_page(None);
-    let (items, _) = sh_at_view(
+    let (page, _) = sh_at_view(
         st,
         sh,
         asof,
         |q, view| q.scripthash_history_filtered_in(sh, &filter, view),
         |q, slot, view| q.scripthash_history_filtered_slot_in(sh, &filter, slot, view),
-        Vec::new(),
+        cursor_page(None),
         None,
         bag,
     )?;
+    let items = page.rows;
     let chain = history_items_to_tx_json(&st.query, &items, st.network).map_err(store_err)?;
     out.extend(chain);
     Ok(out)
@@ -1334,16 +1365,17 @@ fn summary_vec(
     bag: Option<&mut JoinBag>,
 ) -> Result<Vec<Value>, Response> {
     let filter = HistoryFilter::esplora_chain_page(None);
-    let (items, _) = sh_at_view(
+    let (page, _) = sh_at_view(
         st,
         sh,
         asof,
         |q, view| q.scripthash_history_summary_filtered_in(sh, &filter, view),
         |q, slot, view| q.scripthash_history_summary_filtered_slot_in(sh, &filter, slot, view),
-        Vec::new(),
+        cursor_page(None),
         None,
         bag,
     )?;
+    let items = page.rows;
     match summaries_json(&st.query, &items) {
         Ok(Value::Array(v)) => Ok(v),
         Ok(_) => Ok(Vec::new()),
@@ -1375,7 +1407,9 @@ fn multi_txs(
     st.promote_bulk(client, bag);
     let mut rows = dedup_txid(rows);
     sort_txs_newest_first(&mut rows);
-    let rows = skip_rows_after(rows, after);
+    let Some(rows) = skip_rows_after(rows, after) else {
+        return after_txid_not_found();
+    };
     Json(rows).into_response()
 }
 
@@ -1412,7 +1446,9 @@ fn multi_summary(
                 .cmp(a["txid"].as_str().unwrap_or(""))
         })
     });
-    let rows = skip_rows_after(rows, after);
+    let Some(rows) = skip_rows_after(rows, after) else {
+        return after_txid_not_found();
+    };
     Json(rows).into_response()
 }