From 2f14022ad794aa846939a6795a9d34986b0f2825 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim Date: Wed, 2 Sep 2026 19:28:59 -0400 Subject: [PATCH 1/2] feat(research): measure what a round trip costs on a commission-free venue (#371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alpaca Phase C's cost-fidelity half. The PRD makes this a precondition -- "no strategy claim is believed before it" (§6.3, O4) -- and the ordering earned its keep: `config.paper-equities.yaml` ships `taker_pct: 0.0`, honestly labelled as awaiting this work, and priced literally that models a FREE venue. Result: a round trip costs ~2.2bp on Alpaca equities and ~306bp on Coinbase spot, a factor of 141. keel's crypto model is validated in passing (332bp charged vs 306bp measured, conservative by 8%, from an estimator with no knowledge of Coinbase's fee schedule). Its equities pricing is 7.5x too high. Two findings mattered more than the headline. THE AGGREGATION CHOICE IS WORTH 22x, AND THE OBVIOUS ONE IS WRONG. Corwin-Schultz two-day estimates go negative constantly. Flooring each pair and averaging the survivors keeps the positive half of symmetric noise and discards the negative half, so the mean converges on E[max(X,0)] > 0 -- it reports a spread that is not there, and a bigger one the more volatile the series is. Worst, therefore, exactly on the asset class this exists to price. The paper's own procedure averages within a month and floors the MONTHLY mean; negatives then cancel inside the block. Same data, same estimator: MSFT 41.8bp -> 1.9bp. The first run of this measurement reported 43.6bp and a 4.1x ratio, and what exposed it was the implausibility of a mega-cap quoting 41.8bp, not a test. Both aggregations stay reachable, because the comparison between them IS the finding. KEEL READS MSFT AS A THIN ASSET. Cached median daily quote volume is $186M -- about 2% of MSFT's real consolidated volume, which is approximately IEX's market share, and the equities profile runs on the IEX feed. Every ticker sits below the model's $500M anchor, so every one is priced as thin. The candles are fine; the statistic is being asked a question the feed cannot answer. Bounded for cost modelling, unbounded for liquidity screens and admission floors. Filed as #696. Tests (written first, red before green): tests/research/test_spread.py -- 28 tests. The load-bearing one is `test_flooring_each_pair_biases_a_quiet_series_up`: a seeded random walk with NO spread at all, where the naive aggregation must read >5x the blocked one. `test_the_overnight_gap_adjustment_is_applied` pins §I.B -- equities gap nightly and crypto does not, so an unadjusted estimator would inflate the equities side of exactly this comparison, in the direction that flatters keel's existing prior. Mutation-verified, 12 mutants. One survived -- averaging over positive blocks only rather than all blocks -- because no test had a series with both positive and negative BLOCKS; `test_a_floored_block_still_counts_toward_the_average` was added and kills it. One further mutant proved equivalent (`sum(max(m,0))` and `sum(m for m if m>0)` over the same denominator) and is recorded as such rather than counted as a survivor. `keel/commands/research.py` gains the index row its completeness pin demands. Refs #371 Refs #696 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NzuKAe2RVrPt9acVAWjRyL --- .../2026-09-02-equities-cost-fidelity.jsonl | 29 ++ .../2026-09-02-equities-cost-fidelity.md | 156 +++++++++ .../2026-09-02-equities-cost-fidelity.py | 207 +++++++++++ keel/commands/research.py | 15 + keel/research/spread.py | 231 +++++++++++++ tests/research/test_spread.py | 321 ++++++++++++++++++ 6 files changed, 959 insertions(+) create mode 100644 docs/experiments/2026-09-02-equities-cost-fidelity.jsonl create mode 100644 docs/experiments/2026-09-02-equities-cost-fidelity.md create mode 100644 docs/experiments/2026-09-02-equities-cost-fidelity.py create mode 100644 keel/research/spread.py create mode 100644 tests/research/test_spread.py diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl b/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl new file mode 100644 index 00000000..d5b5a5ef --- /dev/null +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl @@ -0,0 +1,29 @@ +{"product_id": "MSFT-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4984", "gap_adjusted_pairs": 147, "spread_bp": "1.94", "naive_spread_bp": "41.82", "raw_spread_bp": "-32.21", "blocks": 60, "half_spread_bp": "0.97", "median_daily_quote_volume": "186462740", "modelled_slippage_bp": "8.19", "modelled_roundtrip_bp": "16.38", "regulatory_bp": "0.2323", "sec_section_31_usd": "0.022900", "taf_usd": "0.000333", "commission_bp": "0.00", "measured_roundtrip_bp": "2.17"} +{"product_id": "AAPL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4952", "gap_adjusted_pairs": 141, "spread_bp": "2.83", "naive_spread_bp": "43.55", "raw_spread_bp": "-32.96", "blocks": 60, "half_spread_bp": "1.42", "median_daily_quote_volume": "216061742", "modelled_slippage_bp": "7.61", "modelled_roundtrip_bp": "15.21", "regulatory_bp": "0.2341", "sec_section_31_usd": "0.022900", "taf_usd": "0.000511", "commission_bp": "0.00", "measured_roundtrip_bp": "3.07"} +{"product_id": "GOOGL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.5296", "gap_adjusted_pairs": 152, "spread_bp": "1.29", "naive_spread_bp": "44.83", "raw_spread_bp": "-44.21", "blocks": 60, "half_spread_bp": "0.64", "median_daily_quote_volume": "146760641", "modelled_slippage_bp": "9.23", "modelled_roundtrip_bp": "18.46", "regulatory_bp": "0.2339", "sec_section_31_usd": "0.022900", "taf_usd": "0.000494", "commission_bp": "0.00", "measured_roundtrip_bp": "1.51"} +{"product_id": "NVDA-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4952", "gap_adjusted_pairs": 164, "spread_bp": "4.50", "naive_spread_bp": "77.48", "raw_spread_bp": "-63.29", "blocks": 60, "half_spread_bp": "2.25", "median_daily_quote_volume": "300409740", "modelled_slippage_bp": "6.45", "modelled_roundtrip_bp": "12.90", "regulatory_bp": "0.2364", "sec_section_31_usd": "0.022900", "taf_usd": "0.000739", "commission_bp": "0.00", "measured_roundtrip_bp": "4.74"} +{"product_id": "COST-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4808", "gap_adjusted_pairs": 141, "spread_bp": "1.36", "naive_spread_bp": "34.81", "raw_spread_bp": "-27.66", "blocks": 60, "half_spread_bp": "0.68", "median_daily_quote_volume": "44370750", "modelled_slippage_bp": "16.78", "modelled_roundtrip_bp": "33.57", "regulatory_bp": "0.2308", "sec_section_31_usd": "0.022900", "taf_usd": "0.000178", "commission_bp": "0.00", "measured_roundtrip_bp": "1.59"} +{"product_id": "BTC-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3289", "gap_adjusted_pairs": 3, "spread_bp": "44.67", "naive_spread_bp": "100.10", "raw_spread_bp": "38.70", "blocks": 90, "half_spread_bp": "22.33", "median_daily_quote_volume": "568492017", "modelled_slippage_bp": "5.00", "modelled_roundtrip_bp": "250.00", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "284.66"} +{"product_id": "ETH-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3364", "gap_adjusted_pairs": 1, "spread_bp": "54.73", "naive_spread_bp": "132.55", "raw_spread_bp": "46.32", "blocks": 90, "half_spread_bp": "27.37", "median_daily_quote_volume": "329719539", "modelled_slippage_bp": "6.16", "modelled_roundtrip_bp": "252.31", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "294.74"} +{"product_id": "ADA-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3600", "gap_adjusted_pairs": 0, "spread_bp": "60.54", "naive_spread_bp": "167.60", "raw_spread_bp": "48.67", "blocks": 89, "half_spread_bp": "30.27", "median_daily_quote_volume": "21810985", "modelled_slippage_bp": "23.94", "modelled_roundtrip_bp": "287.88", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "300.54"} +{"product_id": "LINK-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3616", "gap_adjusted_pairs": 0, "spread_bp": "69.94", "naive_spread_bp": "180.72", "raw_spread_bp": "61.57", "blocks": 89, "half_spread_bp": "34.97", "median_daily_quote_volume": "21072345", "modelled_slippage_bp": "24.36", "modelled_roundtrip_bp": "288.71", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "309.94"} +{"product_id": "LTC-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3401", "gap_adjusted_pairs": 1, "spread_bp": "65.01", "naive_spread_bp": "153.09", "raw_spread_bp": "61.09", "blocks": 89, "half_spread_bp": "32.51", "median_daily_quote_volume": "14815056", "modelled_slippage_bp": "29.05", "modelled_roundtrip_bp": "298.09", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "305.02"} +{"product_id": "SOL-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3718", "gap_adjusted_pairs": 1, "spread_bp": "69.76", "naive_spread_bp": "192.37", "raw_spread_bp": "62.52", "blocks": 89, "half_spread_bp": "34.88", "median_daily_quote_volume": "107673223", "modelled_slippage_bp": "10.77", "modelled_roundtrip_bp": "261.55", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "309.76"} +{"product_id": "XLM-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3718", "gap_adjusted_pairs": 0, "spread_bp": "57.44", "naive_spread_bp": "148.24", "raw_spread_bp": "43.95", "blocks": 89, "half_spread_bp": "28.72", "median_daily_quote_volume": "10004921", "modelled_slippage_bp": "35.35", "modelled_roundtrip_bp": "310.69", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "297.44"} +{"product_id": "PAXG-USDT", "asset_class": "crypto", "bars": 1826, "pairs": 1825, "negative_pair_share": "0.2833", "gap_adjusted_pairs": 0, "spread_bp": "21.16", "naive_spread_bp": "39.80", "raw_spread_bp": "20.30", "blocks": 87, "half_spread_bp": "10.58", "median_daily_quote_volume": "3381426", "modelled_slippage_bp": "60.80", "modelled_roundtrip_bp": "361.60", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "261.16"} +{"product_id": "BCH-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3369", "gap_adjusted_pairs": 1, "spread_bp": "70.20", "naive_spread_bp": "158.66", "raw_spread_bp": "63.15", "blocks": 89, "half_spread_bp": "35.10", "median_daily_quote_volume": "5497843", "modelled_slippage_bp": "47.68", "modelled_roundtrip_bp": "335.36", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "310.20"} +{"product_id": "AAVE-USD", "asset_class": "crypto", "bars": 1819, "pairs": 1818, "negative_pair_share": "0.3614", "gap_adjusted_pairs": 2, "spread_bp": "67.51", "naive_spread_bp": "195.25", "raw_spread_bp": "55.45", "blocks": 87, "half_spread_bp": "33.76", "median_daily_quote_volume": "6365686", "modelled_slippage_bp": "44.31", "modelled_roundtrip_bp": "328.63", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "307.52"} +{"product_id": "DOGE-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3556", "gap_adjusted_pairs": 2, "spread_bp": "65.98", "naive_spread_bp": "173.97", "raw_spread_bp": "58.47", "blocks": 88, "half_spread_bp": "32.99", "median_daily_quote_volume": "29636691", "modelled_slippage_bp": "20.54", "modelled_roundtrip_bp": "281.07", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "305.98"} +{"product_id": "DOT-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3638", "gap_adjusted_pairs": 2, "spread_bp": "66.16", "naive_spread_bp": "174.75", "raw_spread_bp": "58.49", "blocks": 88, "half_spread_bp": "33.08", "median_daily_quote_volume": "6286489", "modelled_slippage_bp": "44.59", "modelled_roundtrip_bp": "329.18", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "306.16"} +{"product_id": "UNI-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3529", "gap_adjusted_pairs": 1, "spread_bp": "74.46", "naive_spread_bp": "192.42", "raw_spread_bp": "66.66", "blocks": 88, "half_spread_bp": "37.23", "median_daily_quote_volume": "4917765", "modelled_slippage_bp": "50.42", "modelled_roundtrip_bp": "340.83", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "314.46"} +{"product_id": "ZEC-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3455", "gap_adjusted_pairs": 2, "spread_bp": "88.94", "naive_spread_bp": "216.30", "raw_spread_bp": "73.56", "blocks": 89, "half_spread_bp": "44.47", "median_daily_quote_volume": "1278710", "modelled_slippage_bp": "98.87", "modelled_roundtrip_bp": "437.74", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "328.94"} +{"product_id": "ALGO-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3660", "gap_adjusted_pairs": 1, "spread_bp": "66.45", "naive_spread_bp": "191.71", "raw_spread_bp": "55.93", "blocks": 88, "half_spread_bp": "33.23", "median_daily_quote_volume": "3699580", "modelled_slippage_bp": "58.13", "modelled_roundtrip_bp": "356.25", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "306.46"} +{"product_id": "FET-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3611", "gap_adjusted_pairs": 2, "spread_bp": "113.21", "naive_spread_bp": "261.64", "raw_spread_bp": "103.98", "blocks": 88, "half_spread_bp": "56.60", "median_daily_quote_volume": "4661785", "modelled_slippage_bp": "51.78", "modelled_roundtrip_bp": "343.56", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "353.20"} +{"product_id": "CRV-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3399", "gap_adjusted_pairs": 1, "spread_bp": "97.98", "naive_spread_bp": "227.64", "raw_spread_bp": "87.52", "blocks": 88, "half_spread_bp": "48.99", "median_daily_quote_volume": "3322338", "modelled_slippage_bp": "61.34", "modelled_roundtrip_bp": "362.68", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "337.98"} +{"product_id": "ICP-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3461", "gap_adjusted_pairs": 2, "spread_bp": "84.89", "naive_spread_bp": "212.25", "raw_spread_bp": "79.23", "blocks": 89, "half_spread_bp": "42.45", "median_daily_quote_volume": "4813975", "modelled_slippage_bp": "50.96", "modelled_roundtrip_bp": "341.91", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "324.90"} +{"product_id": "AVAX-USD", "asset_class": "crypto", "bars": 1793, "pairs": 1792, "negative_pair_share": "0.3839", "gap_adjusted_pairs": 2, "spread_bp": "57.78", "naive_spread_bp": "181.86", "raw_spread_bp": "39.01", "blocks": 86, "half_spread_bp": "28.89", "median_daily_quote_volume": "13043019", "modelled_slippage_bp": "30.96", "modelled_roundtrip_bp": "301.92", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "297.78"} +{"product_id": "NEAR-USD", "asset_class": "crypto", "bars": 1458, "pairs": 1457, "negative_pair_share": "0.3789", "gap_adjusted_pairs": 1, "spread_bp": "68.40", "naive_spread_bp": "193.36", "raw_spread_bp": "55.23", "blocks": 70, "half_spread_bp": "34.20", "median_daily_quote_volume": "3641518", "modelled_slippage_bp": "58.59", "modelled_roundtrip_bp": "357.18", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "308.40"} +{"product_id": "XRP-USD", "asset_class": "crypto", "bars": 1104, "pairs": 1103, "negative_pair_share": "0.3509", "gap_adjusted_pairs": 1, "spread_bp": "52.56", "naive_spread_bp": "137.22", "raw_spread_bp": "44.97", "blocks": 53, "half_spread_bp": "26.28", "median_daily_quote_volume": "77210801", "modelled_slippage_bp": "12.72", "modelled_roundtrip_bp": "265.45", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "292.56"} +{"product_id": "PAXG-USD", "asset_class": "crypto", "bars": 482, "pairs": 481, "negative_pair_share": "0.3160", "gap_adjusted_pairs": 1, "spread_bp": "20.25", "naive_spread_bp": "49.45", "raw_spread_bp": "18.68", "blocks": 23, "half_spread_bp": "10.12", "median_daily_quote_volume": "1262880", "modelled_slippage_bp": "99.49", "modelled_roundtrip_bp": "438.98", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "260.24"} +{"product_id": "WLD-USD", "asset_class": "crypto", "bars": 449, "pairs": 448, "negative_pair_share": "0.3795", "gap_adjusted_pairs": 0, "spread_bp": "78.15", "naive_spread_bp": "228.26", "raw_spread_bp": "49.44", "blocks": 22, "half_spread_bp": "39.08", "median_daily_quote_volume": "1245627", "modelled_slippage_bp": "100.18", "modelled_roundtrip_bp": "440.35", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "318.16"} +{"product_id": "TON-USD", "asset_class": "crypto", "bars": 248, "pairs": 247, "negative_pair_share": "0.3522", "gap_adjusted_pairs": 0, "spread_bp": "48.29", "naive_spread_bp": "164.68", "raw_spread_bp": "31.74", "blocks": 12, "half_spread_bp": "24.15", "median_daily_quote_volume": "366376", "modelled_slippage_bp": "183.80", "modelled_roundtrip_bp": "607.60", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "288.30"} diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.md b/docs/experiments/2026-09-02-equities-cost-fidelity.md new file mode 100644 index 00000000..6385fd7d --- /dev/null +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.md @@ -0,0 +1,156 @@ +# 2026-09-02: What a round trip costs on a commission-free venue (#371, Alpaca Phase C) + +**Headline: a round trip costs ~2.2bp on Alpaca equities and ~306bp on Coinbase spot — a +factor of 141.** keel's cost model is close to right on crypto (it charges 332bp against a +measured 306bp, conservative by 8%) and wrong by 7.5× on equities, where it charges 16.4bp +against a measured 2.2bp. Both errors are now explained, and the equities one has a specific +cause: the IEX feed reports only IEX's own executions, so keel's liquidity statistic reads +MSFT as a $186M/day product when it trades several billion. + +The PRD makes this a precondition — "no strategy claim is believed before it" (§6.3, O4) — and +that ordering earned its keep. `config.paper-equities.yaml` ships `taker_pct: 0.0`, honestly +labelled as awaiting this work. Priced literally, that models a free venue, and a free venue +makes everything look profitable. + +**This measures COST ONLY.** It makes no claim about edge on either asset class. The crypto +intersection is empty; the equity rules have never been measured at all. Nothing below changes +either fact, and a cheaper venue is not an edge. + +## What was measured, on what + +- Data: the deployment's own cached ONE_DAY candles — `keel-equities.db` (5 tickers × 1,253 + bars, Alpaca IEX feed) and `keel.db` (24 assets, 248–1,871 bars, Coinbase). +- Spread: `keel.research.spread.corwin_schultz_spread` — Corwin & Schultz (2012), *"A Simple + Way to Estimate Bid-Ask Spreads from Daily High and Low Prices"*, Journal of Finance 67(2), + 719–759. **One estimator, both asset classes, from data already on disk.** A comparison + measured two different ways would be an artifact of the methods rather than a finding about + the venues, and buying a quote history would make the number unreproducible by a reader. +- Overnight-gap adjustment ON (§I.B of the paper). Equities gap nightly; crypto trades + continuously. Leaving it off inflates the equities side of exactly this comparison — the + direction that flatters keel's existing crypto-heavy prior. +- Commission and regulatory fees are **computed, not estimated**: 1.2%/leg is Coinbase's + published taker rate, $0 is Alpaca's published commission, and the sell-side pass-throughs + come from `keel_broker_alpaca.fees`, whose constants carry their own provenance. +- Driver: [`2026-09-02-equities-cost-fidelity.py`](2026-09-02-equities-cost-fidelity.py). + +## The result + +Round trip = commission (both legs) + spread (half per leg) + regulatory (sell only). + +| ticker | bars | spread bp | naive bp | raw bp | reg bp | measured RT bp | keel charges bp | +| --- | --- | --- | --- | --- | --- | --- | --- | +| GOOGL | 1253 | 1.29 | 44.83 | -44.21 | 0.2339 | **1.51** | 18.46 | +| COST | 1253 | 1.36 | 34.81 | -27.66 | 0.2308 | **1.59** | 33.57 | +| MSFT | 1253 | 1.94 | 41.82 | -32.21 | 0.2323 | **2.17** | 16.38 | +| AAPL | 1253 | 2.83 | 43.55 | -32.96 | 0.2341 | **3.07** | 15.21 | +| NVDA | 1253 | 4.50 | 77.48 | -63.29 | 0.2364 | **4.74** | 12.90 | + +| | median spread | median measured RT | median keel charges | model error | +| --- | --- | --- | --- | --- | +| equities (5) | 1.94bp | **2.17bp** | 16.38bp | **7.5× too high** | +| crypto (24) | 66.31bp | **306.31bp** | 332.27bp | 1.08× (conservative) | + +**Ratio: 141×.** The pre-declared expectation was "one to two orders of magnitude"; the +measurement landed inside it. The refutation condition was an equities spread at or above 50bp +or a material regulatory term — neither occurred. The regulatory term is real but negligible at +this account's size: on a $1,000 sell, SEC Section 31 is $0.0229 and FINRA TAF is $0.000333, +together **0.23bp**, about a tenth of the spread. + +## Finding 1 — the aggregation choice is worth 22×, and the obvious one is wrong + +This is the part that nearly produced a wrong document. The first run of this measurement +reported equities at 43.6bp and a ratio of 4.1×, and it was the *plausibility* of MSFT quoting +41.8bp — roughly twenty times the penny it actually quotes — that exposed the error rather than +any test. + +Corwin-Schultz two-day estimates go negative often, and no venue charges a negative spread, so +they must be handled. The tempting treatment is to floor every pair at zero and average the +survivors. On a series whose true spread is small relative to its volatility, the estimate is +symmetric noise about zero, and keeping the positive half while discarding the negative half +makes the mean converge on E[max(X,0)] > 0. **It reports a spread that is not there, and +reports a larger one the more volatile the series is.** The `raw bp` column is the giveaway: +every equity's unfloored mean is strongly NEGATIVE, which is the honest statement that the +series carries no spread the estimator can resolve at daily frequency. + +Corwin & Schultz's own procedure averages within a month and floors the *monthly* mean, so the +negatives cancel against the positives inside the block instead of being discarded. Same data, +same estimator: + +| | naive (floor each pair) | blocked (floor each month) | ratio | +| --- | --- | --- | --- | +| equities | 43.55bp | 1.94bp | **22.4×** | +| crypto | 177.74bp | 66.31bp | 2.7× | + +The bias is worst exactly where the true spread is smallest — which is to say, exactly on the +asset class this measurement exists to price. `keel.research.spread` keeps both aggregations +reachable, because the comparison between them is itself the finding, and the biased one is +what makes the bias demonstrable rather than folklore. + +## Finding 2 — keel reads MSFT as a thin asset, because IEX is 2% of the market + +`slippage_for_quote_volume` keys off `median_daily_quote_volume` computed from cached candles, +against a $500M/day anchor with a 5bp floor. What it sees: + +| ticker | cached median daily quote volume | modelled slippage | +| --- | --- | --- | +| MSFT | $186,462,740 | 8.19bp | +| AAPL | $216,061,742 | 7.61bp | +| GOOGL | $146,760,641 | 9.23bp | +| NVDA | $300,409,740 | 6.45bp | +| COST | $44,370,750 | 16.78bp | + +Every one sits below the anchor, so every one is priced as *thinner than the model's reference +liquidity*. MSFT's real consolidated volume is several billion dollars a day. The cached figure +is roughly 2% of it — which is approximately IEX's share of US equity volume, and the equities +profile runs on the IEX feed by configuration. + +**The candles are not wrong; the statistic is being asked a question the feed cannot answer.** +IEX reports its own executions faithfully. A model keyed on that volume will misread every US +equity as ~50× thinner than it is, and will do so silently, because nothing in the pipeline +knows the difference between "this product is thin" and "this feed sees 2% of the market". + +This is the SIP-vs-IEX data-tier implication #371 asked about, and it is more consequential than +the fee question it was filed beside. Two consequences worth separating: + +1. **For cost modelling** (this document): the 5bp floor is the binding term for mega-caps + anyway, and even the floor is ~2.5× the measured 1.94bp spread. Both the floor and the + volume statistic are crypto-shaped. +2. **For anything else keyed on volume** — liquidity screens, admission floors, the asset + scout's `--probe-liquidity` — the same 50× understatement applies, and a genuinely liquid + equity could be refused admission for thinness it does not have. That direction fails + closed, so it is safe — but safe for the wrong reason, and it will misinform any equities + universe decision. Filed as **#696**; nothing in this document changes it. + +## Finding 3 — the crypto model is validated, in passing + +332.27bp modelled against 306.31bp measured, conservative by 8%, using an estimator with no +knowledge of Coinbase's fee schedule. The two agree because the fee dominates: 240bp of the +306bp is published commission, and the spread estimate only has to be roughly right for the +total to be. This is the first independent check on keel's crypto cost assumption that does not +reuse keel's own fee constant, and it passes. + +Read the crypto spread column on its own terms, though: PAXG at ~20bp and FET at ~113bp is a +5.6× dispersion that a single global slippage figure cannot express. That is the same argument +[the per-product restatement](2026-09-01-per-product-slippage-restatement.md) made from the +volume side, arriving independently from the price side. + +## What this does and does not license + +It licenses one sentence: **the cost regime that killed every crypto result is not present on +equities.** 2.2bp is not 306bp, and a strategy needing 300bp of gross edge to break even is +playing a different game from one needing 3bp. + +It licenses nothing about edge. Every measured rule is negative on crypto for three unrelated +reasons, and only one of them — `turtle_breakout`'s real gross edge destroyed by cost — is even +addressable by a cheaper venue. `pullback_continuation` has essentially no gross edge (median +0.77) and will not acquire one on a different asset class; `rsi_meanrev` fails on sample size. +A 141× cost reduction is a necessary condition for those first-family results, not a sufficient +one, and the equity rules remain entirely unmeasured. The DCA benchmark on these same tickers is +the next step and is deliberately not in this document. + +One caveat on the equity numbers specifically: ~150 of each ticker's ~1,250 pairs are +gap-adjusted, and a gap-adjusted pair floors to zero by construction (the adjustment leaves day +2 sitting exactly atop day 1, the most trend-like geometry two ranges can have). The equity +estimates therefore rest on ~1,100 effective pairs, not 1,250. This does not bias them upward — +gap days contribute zero rather than noise — but it widens their uncertainty relative to the +crypto series, which barely gap at all (0–3 pairs each). diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.py b/docs/experiments/2026-09-02-equities-cost-fidelity.py new file mode 100644 index 00000000..eb0c31c1 --- /dev/null +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.py @@ -0,0 +1,207 @@ +"""What a round trip actually costs on a commission-free venue -- issue #371 (Alpaca Phase C). + +**The question.** Every cost claim keel has ever made is about Coinbase, where the taker fee +(1.2%/leg) dwarfs everything else and the spread is a rounding error beside it. Alpaca charges +NO commission on US equities, so the entire cost is the two terms keel has never had to measure +carefully: the sell-side regulatory pass-throughs, and the spread. The PRD (§6.3, O4) makes +this measurement a PRECONDITION -- "no strategy claim is believed before it" -- because the +failure mode is obvious and seductive: an equities backtest priced at `taker_pct: 0.0` (which is +what `config.paper-equities.yaml` ships today, honestly labelled as awaiting this work) models a +FREE venue, and a free venue makes everything look profitable. + +DECLARED BEFORE THE RUN: + +* **One estimator, both asset classes, same data type.** Corwin-Schultz (2012) on daily + highs/lows, from candles already cached. A comparison measured two different ways would be an + artifact of the methods, not a finding about the venues. Overnight-gap adjustment ON (equities + gap, crypto does not; leaving it off would inflate the equities side of exactly this + comparison -- the direction that flatters keel's existing prior). +* **The regulatory term is computed, not estimated**: `keel_broker_alpaca.fees` already encodes + SEC Section 31 and FINRA TAF from published formulas, with provenance. +* **Primary quantity: total round-trip friction in basis points**, equities vs crypto, and + against what keel's model charges each today. +* **Pre-declared expectation**: equities friction lands one to two ORDERS OF MAGNITUDE below + crypto's ~250bp, and keel's current equities pricing is wrong in BOTH directions at once -- + under-charging fees (0 instead of the pass-throughs) and over-charging slippage (the 5bp + floor of a crypto-anchored model, on books far deeper than its $500M/day anchor). +* **What would refute it**: an equities spread estimate at or above 50bp, or a regulatory term + material against the spread. Either would mean commission-free is a marketing claim rather + than a cost structure, and the equities profile would need the same null treatment as crypto. + +This measures COST ONLY. It makes no claim about edge on either asset class, and nothing here +should be read as one -- the crypto intersection is empty, and the equity rules have never been +measured at all. + +Re-run: + KEEL_EQUITIES_DB=~/keel/keel-equities.db KEEL_CRYPTO_DB=~/keel/keel.db \ + python docs/experiments/2026-09-02-equities-cost-fidelity.py +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import statistics +from decimal import Decimal +from pathlib import Path + +from keel_broker_alpaca.fees import estimate_regulatory_fees +from keel_core.types import Candle, Side + +from keel.research.spread import corwin_schultz_spread +from keel.strategy.backtest import SLIPPAGE_FLOOR_PCT, TAKER_FEE_PCT, slippage_for_quote_volume + +EQUITIES_DB = os.environ.get("KEEL_EQUITIES_DB") or str(Path.home() / "keel" / "keel-equities.db") +CRYPTO_DB = os.environ.get("KEEL_CRYPTO_DB") or str(Path.home() / "keel" / "keel.db") +OUT_DIR = Path(os.environ.get("KEEL_EXPERIMENT_OUT") or Path(__file__).resolve().parent / "_out") +OUT_DIR.mkdir(parents=True, exist_ok=True) +JSONL_PATH = OUT_DIR / "equities_cost_fidelity.jsonl" + +EQUITIES = ["MSFT-USD", "AAPL-USD", "GOOGL-USD", "NVDA-USD", "COST-USD"] +CRYPTO = [ + "BTC-USD", "ETH-USD", "ADA-USD", "LINK-USD", "LTC-USD", "SOL-USD", + "XLM-USD", "PAXG-USDT", "BCH-USD", "AAVE-USD", "DOGE-USD", "DOT-USD", + "UNI-USD", "ZEC-USD", "ALGO-USD", "FET-USD", "CRV-USD", "ICP-USD", + "AVAX-USD", "NEAR-USD", "XRP-USD", "PAXG-USD", "WLD-USD", "TON-USD", +] + +#: The order size the regulatory term is expressed at. keel's equities profile caps exposure at +#: $5,000 over five names at flat 20% weights, so ~$1,000 is the position this account actually +#: takes -- and the per-share TAF makes the answer size-dependent, so a size must be named. +ORDER_USD = Decimal("1000") + + +def load(db: str, product_id: str) -> list[Candle]: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + try: + rows = con.execute( + "SELECT ts, o, h, l, c, v FROM candles WHERE product_id=? AND granularity='ONE_DAY'" + " ORDER BY ts", + (product_id,), + ).fetchall() + finally: + con.close() + return [ + Candle(ts=r[0], open=Decimal(r[1]), high=Decimal(r[2]), low=Decimal(r[3]), + close=Decimal(r[4]), volume=Decimal(r[5])) + for r in rows + ] + + +def median_daily_quote_volume(candles: list[Candle]) -> Decimal: + """The statistic `slippage_for_quote_volume` reads. These are ONE_DAY bars, so the + per-bar median IS the daily median -- no bars-per-day scaling, and none of the unit trap + that bit the triple-barrier work.""" + if not candles: + return Decimal("0") + return Decimal(str(statistics.median(float(c.volume * c.close) for c in candles))) + + +def measure(db: str, product_id: str, asset_class: str) -> dict[str, object] | None: + candles = load(db, product_id) + if len(candles) < 2: + return None + est = corwin_schultz_spread(candles) + naive = corwin_schultz_spread(candles, block_size=None) + if est is None or naive is None: + return None + adv = median_daily_quote_volume(candles) + modelled_slip = slippage_for_quote_volume(adv) + last_close = candles[-1].close + + # The regulatory term, at the size this account trades. Sells only: a BUY pays nothing. + shares = ORDER_USD / last_close if last_close > 0 else Decimal("0") + reg_total, sec31, taf = ( + estimate_regulatory_fees(Side.SELL, shares, ORDER_USD) + if asset_class == "equities" + else (Decimal("0"), Decimal("0"), Decimal("0")) + ) + reg_bp = (reg_total / ORDER_USD * 10000) if ORDER_USD else Decimal("0") + + # What keel charges this product TODAY, round trip: two legs of commission/fee plus two + # legs of slippage. Equities run at `taker_pct: 0.0`; crypto at the 1.2% taker rate. + modelled_fee = Decimal("0") if asset_class == "equities" else TAKER_FEE_PCT + modelled_rt_bp = (2 * modelled_fee + 2 * modelled_slip) * 10000 + + # What it plausibly costs. THE COMMISSION IS NOT AN ESTIMATE -- 1.2%/leg on Coinbase and + # $0 on Alpaca are published rates actually charged, so they belong in the measured total + # beside the estimated spread. Omitting them would compare an equities total that is + # ~all spread against a crypto total that is ~all fee, and understate crypto 5-fold. + commission_bp = (Decimal("0") if asset_class == "equities" else 2 * TAKER_FEE_PCT) * 10000 + measured_rt_bp = commission_bp + 2 * est.half_spread_bp + reg_bp + + return { + "product_id": product_id, + "asset_class": asset_class, + "bars": len(candles), + "pairs": est.pairs, + "negative_pair_share": str(est.negative_pair_share), + "gap_adjusted_pairs": est.gap_adjusted_pairs, + "spread_bp": str(est.spread_bp), + "naive_spread_bp": str(naive.spread_bp), + "raw_spread_bp": str(est.raw_spread_bp), + "blocks": est.blocks, + "half_spread_bp": str(est.half_spread_bp), + "median_daily_quote_volume": str(adv.quantize(Decimal("1"))), + "modelled_slippage_bp": str((modelled_slip * 10000).quantize(Decimal("0.01"))), + "modelled_roundtrip_bp": str(modelled_rt_bp.quantize(Decimal("0.01"))), + "regulatory_bp": str(reg_bp.quantize(Decimal("0.0001"))), + "sec_section_31_usd": str(sec31.quantize(Decimal("0.000001"))), + "taf_usd": str(taf.quantize(Decimal("0.000001"))), + "commission_bp": str(commission_bp.quantize(Decimal("0.01"))), + "measured_roundtrip_bp": str(measured_rt_bp.quantize(Decimal("0.01"))), + } + + +def main() -> None: + rows: list[dict[str, object]] = [] + for pid in EQUITIES: + row = measure(EQUITIES_DB, pid, "equities") + if row: + rows.append(row) + for pid in CRYPTO: + row = measure(CRYPTO_DB, pid, "crypto") + if row: + rows.append(row) + + with JSONL_PATH.open("w") as fh: + for row in rows: + fh.write(json.dumps(row) + "\n") + + hdr = (f"{'product':<12}{'bars':>6}{'spread bp':>11}{'naive bp':>10}{'raw bp':>10}" + f"{'neg share':>11}{'gaps':>7}{'model rt bp':>13}{'measured rt bp':>16}") + for klass in ("equities", "crypto"): + sub = [r for r in rows if r["asset_class"] == klass] + print(f"\n=== {klass} ({len(sub)}) ===") + print(hdr) + for r in sorted(sub, key=lambda r: Decimal(str(r["spread_bp"]))): + print( + f"{r['product_id']:<12}{r['bars']:>6}{r['spread_bp']:>11}{r['naive_spread_bp']:>10}" + f"{r['raw_spread_bp']:>10}" + f"{r['negative_pair_share']:>11}{r['gap_adjusted_pairs']:>7}" + f"{r['modelled_roundtrip_bp']:>13}{r['measured_roundtrip_bp']:>16}" + ) + meas = sorted(Decimal(str(r["measured_roundtrip_bp"])) for r in sub) + modl = sorted(Decimal(str(r["modelled_roundtrip_bp"])) for r in sub) + print(f" median measured round trip: {statistics.median(meas):>10} bp") + print(f" median modelled round trip: {statistics.median(modl):>10} bp") + + eq = [r for r in rows if r["asset_class"] == "equities"] + cr = [r for r in rows if r["asset_class"] == "crypto"] + eq_med = statistics.median(sorted(Decimal(str(r["measured_roundtrip_bp"])) for r in eq)) + cr_med = statistics.median(sorted(Decimal(str(r["measured_roundtrip_bp"])) for r in cr)) + print(f"\ncrypto / equities measured round-trip ratio: {(cr_med / eq_med):.1f}x") + print(f"equities: modelled {statistics.median(sorted(Decimal(str(r['modelled_roundtrip_bp'])) for r in eq))} bp" + f" vs measured {eq_med} bp") + print(f"floor reference: SLIPPAGE_FLOOR_PCT = {SLIPPAGE_FLOOR_PCT * 10000} bp one way") + for klass, sub in (("equities", eq), ("crypto", cr)): + blocked = statistics.median(sorted(Decimal(str(r["spread_bp"])) for r in sub)) + naive = statistics.median(sorted(Decimal(str(r["naive_spread_bp"])) for r in sub)) + print(f"aggregation bias, {klass}: naive {naive} bp vs blocked {blocked} bp" + f" ({naive / blocked:.1f}x)" if blocked else f"{klass}: blocked is zero") + print(f"\nwrote {JSONL_PATH}") + + +if __name__ == "__main__": + main() diff --git a/keel/commands/research.py b/keel/commands/research.py index 3307f938..ace529e6 100644 --- a/keel/commands/research.py +++ b/keel/commands/research.py @@ -298,6 +298,21 @@ class ResearchModuleEntry: ), runs_as="keel research tuning", ), + ResearchModuleEntry( + module="spread.py", + question=( + "What does a round trip cost on a venue whose commission is zero -- the " + "Corwin-Schultz (2012) proportional spread, recovered from daily highs and lows " + "alone, so ONE estimator prices both asset classes from candles already cached." + ), + cannot_answer=( + "Cannot report the spread anyone actually paid: it is an estimator with real " + "dispersion, not a quote feed, and it is noisy for any single series. It also " + "cannot tell a zero spread from an unmeasurable one on its own -- that is what " + "the unfloored `raw_spread_pct` and the negative-pair share are for." + ), + runs_as="docs/experiments/2026-09-02-equities-cost-fidelity.py", + ), ) #: Names accepted by `--module`, in the same declared order as `RESEARCH_INDEX` -- derived diff --git a/keel/research/spread.py b/keel/research/spread.py new file mode 100644 index 00000000..7d42d972 --- /dev/null +++ b/keel/research/spread.py @@ -0,0 +1,231 @@ +"""Proportional effective spread, estimated from daily highs and lows -- issue #371. + +⚠️ **This module measures. It does not gate, size, or price anything on the live path.** +Nothing here is imported by `execution/` or `strategy/`; it exists so a cost claim can be +checked rather than assumed. + +WHY AN ESTIMATOR AT ALL. Phase C of the Alpaca PRD (§6.3, O4) needs one number keel has never +had: what a round trip actually costs on a venue whose commission is zero. Two of the three +components were already exact -- Alpaca's commission is $0 and the sell-side regulatory +pass-throughs are published formulas (`keel_broker_alpaca.fees`). The third, the SPREAD, is +the one that dominates, and keel caches OHLCV bars, never quotes. Buying a quote history to +answer it would make the number unreproducible by anyone reading the write-up. + +Corwin & Schultz (2012), "A Simple Way to Estimate Bid-Ask Spreads from Daily High and Low +Prices", Journal of Finance 67(2), 719-759, recovers the spread from highs and lows alone. +The insight: a day's high-low RANGE contains both the true variance of the security and the +spread, but variance scales with the length of the interval while the spread does not. So two +single-day ranges (two units of variance, two spreads) compared against one two-day range (two +units of variance, ONE spread) isolate the spread. + +THE POINT OF USING IT HERE IS THAT IT APPLIES TO BOTH ASSET CLASSES, from data already on +disk, with ONE estimator and no per-venue tuning. A cost comparison between crypto and +equities measured two different ways would be an artifact of the methods. + +THE OVERNIGHT-GAP ADJUSTMENT IS LOAD-BEARING, NOT A REFINEMENT. Section I.B of the paper +subtracts any overnight jump before estimating, because a gap widens the two-day range without +any spread having been paid. Equities gap every night; crypto trades continuously and barely +gaps at all. An unadjusted estimator would therefore inflate the EQUITIES side of exactly the +comparison this module exists to make -- and inflating equity costs is the direction that +flatters keel's existing crypto-heavy conclusion. A measurement must never be trusted in the +direction of its author's prior, so the adjustment is on by default and the count of adjusted +pairs is reported beside every estimate. + +WHAT IT IS NOT. This is an estimator with real dispersion, not a quote feed. Corwin & Schultz +report it tracking TAQ spreads closely in the cross-section while being noisy for any single +security-month; negative two-day estimates are common, since no venue charges a negative spread. + +**HOW THE NEGATIVES ARE HANDLED DECIDES THE ANSWER, BY A FACTOR OF TWENTY.** This is the single +most consequential choice in the module, and the obvious treatment is the wrong one: + +* *Floor every pair, then average the survivors.* Tempting, and wrong. On a series whose true + spread is small relative to its volatility, the two-day estimate is symmetric noise about + zero; keeping the positive half and discarding the negative half makes the mean converge on + E[max(X,0)] > 0. It reports a spread that is not there, and reports a LARGER one the more + volatile the series is. Measured on keel's own equity candles this reads 41.8bp for MSFT -- + roughly twenty times the penny spread that name actually quotes. +* *Average within a month, then floor the monthly mean* -- Corwin & Schultz's own procedure, + and the default here. The negatives cancel against the positives inside the block, so noise + averages out instead of accumulating. The same MSFT series reads 1.9bp, which is the right + order of magnitude for a mega-cap. + +Both are reachable (`block_size=None` selects the biased one) because the comparison between +them is itself a finding, and `raw_spread_pct` reports the mean with no flooring anywhere -- +the only figure that can come out negative, and therefore the only one able to say "this series +carries no measurable spread" out loud. Read `negative_pairs` as the noise diagnostic it is: a +series that floors half its pairs has not been measured, it has been sampled. + +ONE LIMITATION WORTH KNOWING BEFORE READING ANY NUMBER THIS PRODUCES. The gap adjustment +shifts day 2 to sit exactly ON TOP of day 1, which is the most trend-like geometry two ranges +can have, so a pure gap-and-continue day estimates negative and floors to zero. The estimate +for a gapping series is therefore carried by its OVERLAPPING pairs, and `gap_adjusted_pairs` +is the honest sample-size caveat: a series that gaps constantly rests on fewer effective +observations than `pairs` suggests. This does not bias the estimate up -- gap days contribute +zero rather than noise -- but it does widen its uncertainty on the equities side. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from dataclasses import dataclass +from decimal import Decimal + +from keel_core.types import Candle + +#: 3 - 2*sqrt(2), the constant in Corwin & Schultz eq. (14)/(18). Named because it appears +#: three times and a bare `0.1716` in the arithmetic is unreadable. +_K: float = 3.0 - 2.0 * math.sqrt(2.0) + +#: The estimate's resolution. Ten places keeps sub-basis-point detail (1bp = 1e-4) without +#: pretending to precision the estimator does not have. +_QUANTUM: Decimal = Decimal("1e-10") + +#: Basis-point figures are for the write-up, where two decimals is already generous. +_BP_QUANTUM: Decimal = Decimal("0.01") + +#: Pairs per averaging block. 21 trading days is the month Corwin & Schultz aggregate over. +#: It is a bias/variance dial, not a free parameter: smaller blocks floor more often and drift +#: back toward the per-pair bias; larger ones average across regimes that differ. +MONTHLY_BLOCK: int = 21 + + +@dataclass(frozen=True) +class SpreadEstimate: + """A series' estimated proportional spread, with the diagnostics needed to judge it. + + `spread_pct` is the FULL quoted spread as a fraction of price (0.0012 = 12bp). One leg of + a trade crosses half of it, which is what `half_spread_pct` reports and what maps onto + keel's slippage assumption -- `strategy/backtest.slippage_for_quote_volume` is likewise a + one-way number. + """ + + spread_pct: Decimal + raw_spread_pct: Decimal + pairs: int + negative_pairs: int + gap_adjusted_pairs: int + blocks: int + + @property + def half_spread_pct(self) -> Decimal: + """The one-way cost: crossing from the mid to the touch.""" + return self.spread_pct / 2 + + @property + def raw_spread_bp(self) -> Decimal: + """The unfloored mean, in basis points. Negative means "no measurable spread".""" + return (self.raw_spread_pct * 10000).quantize(_BP_QUANTUM) + + @property + def spread_bp(self) -> Decimal: + return (self.spread_pct * 10000).quantize(_BP_QUANTUM) + + @property + def half_spread_bp(self) -> Decimal: + return (self.half_spread_pct * 10000).quantize(_BP_QUANTUM) + + @property + def negative_pair_share(self) -> Decimal: + """How much of the series floored to zero -- the noise diagnostic. Above roughly half + and the mean is being carried by a minority of pairs.""" + if self.pairs == 0: + return Decimal("0") + return (Decimal(self.negative_pairs) / Decimal(self.pairs)).quantize(Decimal("0.0001")) + + +def _usable(candle: Candle) -> bool: + """A bar the logarithm can take. A zero-range bar (H == L) is excluded deliberately: it + means halted or untraded, not a zero spread, and it collapses both beta and gamma to zero + which would return a spurious 0.0.""" + return candle.low > 0 and candle.high > candle.low + + +def corwin_schultz_pair( + first: Candle, second: Candle, *, adjust_overnight_gap: bool = True +) -> float | None: + """The two-day estimate, Corwin & Schultz eq. (14)-(18). `None` when the pair is unusable. + + Returned as a raw float and NOT floored at zero: the flooring is the aggregator's + decision, and how often a pair goes negative is a diagnostic the caller needs. Float + rather than Decimal because the formula is logarithms and exponentials throughout; the + conversion to Decimal happens once, at the aggregate. + """ + if not _usable(first) or not _usable(second): + return None + + high_1, low_1 = float(first.high), float(first.low) + high_2, low_2 = float(second.high), float(second.low) + + if adjust_overnight_gap: + # Section I.B: a gap moves day 2's whole range without any spread being paid. Shift + # day 2 back onto day 1's range -- the shift cancels inside every log ratio of day 2's + # own bar, so it changes ONLY the two-day range, which is precisely the term the gap + # contaminated. + if low_2 > high_1: + gap = low_2 - high_1 + high_2, low_2 = high_2 - gap, low_2 - gap + elif high_2 < low_1: + gap = low_1 - high_2 + high_2, low_2 = high_2 + gap, low_2 + gap + + beta = math.log(high_1 / low_1) ** 2 + math.log(high_2 / low_2) ** 2 + gamma = math.log(max(high_1, high_2) / min(low_1, low_2)) ** 2 + alpha = (math.sqrt(2.0 * beta) - math.sqrt(beta)) / _K - math.sqrt(gamma / _K) + return 2.0 * (math.exp(alpha) - 1.0) / (1.0 + math.exp(alpha)) + + +def corwin_schultz_spread( + candles: Sequence[Candle], + *, + adjust_overnight_gap: bool = True, + block_size: int | None = MONTHLY_BLOCK, +) -> SpreadEstimate | None: + """Estimate a series' proportional spread from consecutive high-low pairs. + + `block_size` chooses the aggregation, and the choice is worth twenty-fold on a quiet + series -- see the module docstring. The default averages within blocks of that many pairs + and floors each BLOCK mean; `None` floors each PAIR instead, which is the biased variant, + retained so the bias stays demonstrable rather than folklore. + + `None` (the return, not the argument) means NOTHING WAS MEASURABLE -- fewer than two bars, + or no usable pair. That is a different statement from an estimate of zero, and a caller + must not read the two the same way: one says the venue is cheap, the other says the + question was not answered. + """ + estimates: list[float] = [] + negative = 0 + gapped = 0 + for first, second in zip(candles, candles[1:], strict=False): + estimate = corwin_schultz_pair(first, second, adjust_overnight_gap=adjust_overnight_gap) + if estimate is None: + continue + estimates.append(estimate) + if adjust_overnight_gap and (second.low > first.high or second.high < first.low): + gapped += 1 + if estimate < 0: + negative += 1 + if not estimates: + return None + + raw = sum(estimates) / len(estimates) + if block_size is None: + floored = sum(e for e in estimates if e > 0) / len(estimates) + blocks = 0 + else: + chunks = [estimates[i : i + block_size] for i in range(0, len(estimates), block_size)] + means = [sum(chunk) / len(chunk) for chunk in chunks] + floored = sum(max(m, 0.0) for m in means) / len(means) + blocks = len(means) + + return SpreadEstimate( + spread_pct=Decimal(repr(floored)).quantize(_QUANTUM), + raw_spread_pct=Decimal(repr(raw)).quantize(_QUANTUM), + pairs=len(estimates), + negative_pairs=negative, + gap_adjusted_pairs=gapped, + blocks=blocks, + ) + + +__all__ = ["MONTHLY_BLOCK", "SpreadEstimate", "corwin_schultz_pair", "corwin_schultz_spread"] diff --git a/tests/research/test_spread.py b/tests/research/test_spread.py new file mode 100644 index 00000000..ae4f9204 --- /dev/null +++ b/tests/research/test_spread.py @@ -0,0 +1,321 @@ +"""The Corwin-Schultz high-low spread estimator (issue #371). + +Phase C of the Alpaca PRD needs one number keel has never had: **what a round trip actually +costs on a venue whose commission is zero.** The regulatory pass-throughs are exact and already +modelled (`keel_broker_alpaca.fees`); the spread is not, and keel caches OHLCV bars, never +quotes. Corwin-Schultz recovers a proportional effective spread from daily highs and lows +alone, which makes it the one estimator that can be applied to BOTH asset classes from data +already on disk -- the like-for-like comparison the cost-distortion claim rests on. + +The load-bearing test is `test_the_overnight_gap_adjustment_is_applied`: equities gap overnight +and crypto (24/7) essentially does not, so an unadjusted estimator would inflate the equity side +of exactly the comparison this exists to make. The bias would run in the direction that +FLATTERS keel's crypto-heavy prior, which is the direction a measurement must never be trusted +in. +""" + +from __future__ import annotations + +import math +import random +from decimal import Decimal + +import pytest +from keel_core.types import Candle + +from keel.research.spread import ( + MONTHLY_BLOCK, + SpreadEstimate, + corwin_schultz_pair, + corwin_schultz_spread, +) + + +def _c(high: str, low: str, ts: int = 0, close: str | None = None) -> Candle: + """A candle carrying only what the estimator reads: its high and its low.""" + hi, lo = Decimal(high), Decimal(low) + mid = close if close is not None else str((hi + lo) / 2) + return Candle(ts=ts, open=mid, high=hi, low=lo, close=Decimal(mid), volume=Decimal("1000")) + + +# --- the pair estimator: known answers, computed from the published formula ------------------ + + +def test_a_two_day_pair_reproduces_the_published_formula() -> None: + """Corwin & Schultz (2012) eq. (14)-(18), evaluated by hand on a wide-range pair.""" + got = corwin_schultz_pair(_c("101", "99"), _c("101.5", "99.5")) + assert got is not None + assert math.isclose(float(got), 0.0079088925, rel_tol=1e-6) + + +def test_a_tighter_range_estimates_a_tighter_spread() -> None: + wide = corwin_schultz_pair(_c("101", "99"), _c("101.5", "99.5")) + tight = corwin_schultz_pair(_c("100.2", "99.8"), _c("100.3", "99.9")) + assert wide is not None and tight is not None + assert math.isclose(float(tight), 0.0015849918, rel_tol=1e-6) + assert tight < wide + + +def test_a_trending_pair_estimates_a_negative_spread() -> None: + """A strong two-day trend makes the two-day range much wider than the daily ranges, which + drives alpha negative. The estimator does NOT hide this at the pair level -- the flooring + is the aggregator's decision, and counting how often it happens is a diagnostic.""" + got = corwin_schultz_pair(_c("105", "100"), _c("110", "104")) + assert got is not None + assert got < 0 + assert math.isclose(float(got), -0.0506145009, rel_tol=1e-6) + + +def test_a_zero_range_bar_is_not_an_estimate() -> None: + """H == L means no intraday range at all -- a halted or untraded bar, not a zero spread. + log(1) = 0 collapses beta AND gamma, and the formula would return a spurious 0.0.""" + assert corwin_schultz_pair(_c("100", "100"), _c("101", "99")) is None + assert corwin_schultz_pair(_c("101", "99"), _c("100", "100")) is None + + +def test_a_nonpositive_price_is_not_an_estimate() -> None: + """Guards the logarithm rather than raising: one bad cached bar must not end a sweep.""" + assert corwin_schultz_pair(_c("0", "0"), _c("101", "99")) is None + assert corwin_schultz_pair(_c("101", "99"), _c("-1", "-2")) is None + + +def test_a_low_above_its_own_high_is_not_an_estimate() -> None: + """An inverted bar is corrupt data, never a tradeable range.""" + assert corwin_schultz_pair(_c("99", "101"), _c("101", "99")) is None + + +# --- the overnight gap adjustment ------------------------------------------------------------ + + +def test_the_overnight_gap_adjustment_is_applied() -> None: + """Corwin & Schultz section I.B: when day 2's LOW sits above day 1's HIGH, the two-day + range contains an overnight jump that no spread produced. The gap is subtracted from day + 2 before the estimate, or the jump is priced as if it were a spread. + + Day 1 [99, 101], day 2 [110, 112]: a 9-point gap up on a 2-point daily range. Unadjusted, + the two-day range spans 13 points and the estimate is deeply negative; adjusted, day 2 + becomes [101, 103] and the overnight jump is gone from the two-day range.""" + d1, d2 = _c("101", "99"), _c("112", "110") + adjusted = corwin_schultz_pair(d1, d2) + unadjusted = corwin_schultz_pair(d1, d2, adjust_overnight_gap=False) + assert adjusted is not None and unadjusted is not None + # The gap contaminated the estimate downward by an order of magnitude; removing it is + # most of the correction. + assert unadjusted < adjusted + assert float(unadjusted) < 8 * float(adjusted) + # Shifting day 2 down by the gap must give EXACTLY the same answer as adjusting it -- this + # is the assertion that pins the arithmetic, not the sign. + shifted = corwin_schultz_pair(d1, _c("103", "101"), adjust_overnight_gap=False) + assert shifted is not None + assert math.isclose(float(adjusted), float(shifted), rel_tol=1e-12) + + +def test_a_pure_gap_still_floors_to_zero_after_adjustment() -> None: + """An honest limitation, pinned so nobody later reads a gap-heavy series as a cheap one. + + The adjustment shifts day 2 to sit EXACTLY on top of day 1, which is the most + trend-like geometry two ranges can have, so a pure gap-and-continue day estimates negative + and floors to zero. Corwin & Schultz's estimate on a gapping series is therefore carried by + its OVERLAPPING pairs; the gap days contribute nothing rather than contributing noise. That + is why `gap_adjusted_pairs` is reported -- a series that gaps constantly is a series whose + estimate rests on a smaller sample than `pairs` suggests.""" + adjusted = corwin_schultz_pair(_c("101", "99"), _c("112", "110")) + assert adjusted is not None and adjusted < 0 + + +def test_the_gap_adjustment_works_downward_too() -> None: + """A gap DOWN (day 2's high below day 1's low) is the same distortion, mirrored.""" + d1, d2 = _c("101", "99"), _c("90", "88") + adjusted = corwin_schultz_pair(d1, d2) + # The 9-point gap (99 - 90) is added back to day 2, lifting [88, 90] to [97, 99]. + shifted = corwin_schultz_pair(d1, _c("99", "97"), adjust_overnight_gap=False) + assert adjusted is not None and shifted is not None + assert math.isclose(float(adjusted), float(shifted), rel_tol=1e-12) + + +def test_overlapping_days_are_not_gap_adjusted() -> None: + """Ranges that overlap have no overnight jump to remove; touching the estimate there would + be the estimator inventing an adjustment the paper does not make.""" + d1, d2 = _c("101", "99"), _c("101.5", "99.5") + assert corwin_schultz_pair(d1, d2) == corwin_schultz_pair(d1, d2, adjust_overnight_gap=False) + + +# --- the aggregate: WHICH average, and why it is not the obvious one --------------------- + + +def test_the_default_averages_within_blocks_before_flooring() -> None: + """Corwin & Schultz's own procedure: average the two-day estimates within a month, THEN + set a negative MONTHLY average to zero. Not: floor every pair and average the survivors. + + The difference is not cosmetic -- see `test_flooring_each_pair_biases_a_quiet_series_up`. + Here, two pairs in one block whose mean is negative give zero, where per-pair flooring + would have kept the positive one and reported half of it.""" + candles = [_c("101", "99", ts=0), _c("101.5", "99.5", ts=1), _c("110", "104", ts=2)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.pairs == 2 + assert got.blocks == 1 + assert got.spread_pct == Decimal("0") + + +def test_flooring_each_pair_biases_a_quiet_series_up() -> None: + """THE LOAD-BEARING TEST. On a series with NO spread at all -- a pure random walk whose + highs and lows come only from volatility -- the two-day estimate is symmetric noise around + zero. Flooring each pair keeps the positive half and discards the negative half, so its + mean converges on E[max(X,0)] > 0: it reports a spread that does not exist, and reports a + BIGGER one the more volatile the series is. + + That is not a hypothetical. On real mega-cap equities the two aggregations differ by ~20x, + and only the block figure lands anywhere near the penny spreads those names actually quote. + Getting this wrong would have priced a commission-free venue at 20x its true cost.""" + rng = random.Random(20260902) + price, candles = 100.0, [] + for i in range(2000): + price *= math.exp(rng.gauss(0.0, 0.02)) + # High/low from intraday volatility ONLY -- no bid-ask bounce, no spread. + hi = price * math.exp(abs(rng.gauss(0.0, 0.01))) + lo = price * math.exp(-abs(rng.gauss(0.0, 0.01))) + candles.append(_c(f"{hi:.6f}", f"{lo:.6f}", ts=i)) + + naive = corwin_schultz_spread(candles, block_size=None) + block = corwin_schultz_spread(candles) + assert naive is not None and block is not None + assert naive.spread_bp > 5 * block.spread_bp + # And the block estimate is near zero, which is the truth for this series. + assert block.spread_bp < Decimal("10") + + +def test_the_naive_per_pair_flooring_is_still_reachable() -> None: + """Kept, not deleted: the biased variant is what makes the bias demonstrable, and the + write-up quotes both figures. `block_size=None` selects it explicitly.""" + candles = [_c("101", "99", ts=0), _c("101.5", "99.5", ts=1), _c("110", "104", ts=2)] + got = corwin_schultz_spread(candles, block_size=None) + assert got is not None + assert got.pairs == 2 + assert got.negative_pairs == 1 + assert got.blocks == 0 # no blocking was done + assert math.isclose(float(got.spread_pct), 0.0079088925 / 2, rel_tol=1e-6) + + +def _quiet_then_trending() -> list[Candle]: + """21 pairs of tight overlapping ranges (a positive block), then 21 pairs of a 5%-a-bar + ramp whose ranges never overlap (a negative block, floored).""" + candles = [_c("101", "99", ts=i) for i in range(22)] + centre = 100.0 + for i in range(22, 43): + centre *= 1.05 + candles.append(_c(f"{centre * 1.001:.6f}", f"{centre / 1.001:.6f}", ts=i)) + return candles + + +def test_a_floored_block_still_counts_toward_the_average() -> None: + """A negative block contributes ZERO to the mean, not nothing at all -- the denominator is + every block, not just the surviving ones. + + Dropping floored blocks from the denominator would reintroduce the selection bias the + blocking exists to remove, one level up: a series whose spread is unmeasurable in half its + months would report the other half's figure as if it held throughout. Here that is worth + exactly 2x, because one of the two blocks floors.""" + mixed = corwin_schultz_spread(_quiet_then_trending()) + quiet_only = corwin_schultz_spread([_c("101", "99", ts=i) for i in range(22)]) + assert mixed is not None and quiet_only is not None + assert mixed.blocks == 2 and quiet_only.blocks == 1 + # The trending block floored, so the mixed series reports HALF the quiet block's figure. + assert math.isclose( + float(mixed.spread_pct), float(quiet_only.spread_pct) / 2, rel_tol=1e-6 + ) + + +def test_the_block_size_defaults_to_a_trading_month() -> None: + assert MONTHLY_BLOCK == 21 + + +def test_blocks_are_counted_and_a_short_tail_is_still_a_block() -> None: + """43 pairs at 21 to a block is two full blocks and a 1-pair remainder. Dropping the + remainder would silently discard the most recent data, which is the part a cost estimate + most needs.""" + candles = [_c("101", "99", ts=i) for i in range(45)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.pairs == 44 + assert got.blocks == 3 + + +def test_the_raw_unfloored_mean_is_reported_as_a_diagnostic() -> None: + """The only figure with no flooring anywhere. It can be negative, and when it is, that is + the honest statement that the series carries no measurable spread -- a fact both floored + variants are structurally incapable of expressing.""" + candles = [_c("105", "100", ts=0), _c("110", "104", ts=1)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.raw_spread_pct < 0 + assert got.spread_pct == Decimal("0") + + +def test_the_half_spread_is_the_one_way_cost() -> None: + """keel prices ONE leg at a time, so the number that maps onto slippage is half the + quoted spread -- crossing from the mid to the touch.""" + got = corwin_schultz_spread([_c("101", "99", ts=i) for i in range(30)]) + assert got is not None + assert got.half_spread_pct == got.spread_pct / 2 + assert got.spread_pct > 0 + + +def test_a_series_shorter_than_two_bars_has_no_estimate() -> None: + assert corwin_schultz_spread([]) is None + assert corwin_schultz_spread([_c("101", "99")]) is None + + +def test_a_series_whose_every_pair_is_unusable_has_no_estimate() -> None: + """Distinct from "the spread is zero": nothing was measurable, and the caller must be able + to tell those apart rather than reading 0bp as a free venue.""" + assert corwin_schultz_spread([_c("100", "100", ts=0), _c("100", "100", ts=1)]) is None + + +def test_the_gap_adjusted_pair_count_is_reported() -> None: + """The equities-vs-crypto comparison turns on this number: if the equity series adjusts + many pairs and the crypto series adjusts almost none, that asymmetry is the finding.""" + candles = [_c("101", "99", ts=0), _c("112", "110", ts=1), _c("112.5", "110.5", ts=2)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.gap_adjusted_pairs == 1 + + +def test_the_estimate_is_a_decimal_so_it_composes_with_keels_money_types() -> None: + got = corwin_schultz_spread([_c("101", "99", ts=i) for i in range(30)]) + assert got is not None + assert isinstance(got.spread_pct, Decimal) + assert isinstance(got.half_spread_pct, Decimal) + assert isinstance(got.raw_spread_pct, Decimal) + assert got.spread_pct * Decimal("1000") > 0 + + +def test_the_estimate_is_never_negative_in_aggregate() -> None: + """Every pair negative means the block mean is negative and floors to zero -- reported as + zero WITH its pair count, not as None, because the pairs were measurable.""" + got = corwin_schultz_spread([_c("105", "100", ts=0), _c("110", "104", ts=1)]) + assert got is not None + assert got.spread_pct == Decimal("0") + assert got.negative_pairs == 1 + + +@pytest.mark.parametrize("bars", [2, 3, 10, 50]) +def test_pair_count_is_one_less_than_the_usable_bar_count(bars: int) -> None: + candles = [_c("101", "99", ts=i) for i in range(bars)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.pairs == bars - 1 + + +def test_the_estimate_is_reported_as_basis_points_for_the_write_up() -> None: + est = SpreadEstimate( + spread_pct=Decimal("0.0012"), + raw_spread_pct=Decimal("0.0010"), + pairs=100, + negative_pairs=10, + gap_adjusted_pairs=0, + blocks=5, + ) + assert est.spread_bp == Decimal("12.00") + assert est.half_spread_bp == Decimal("6.00") + assert est.raw_spread_bp == Decimal("10.00") From e804324977434c4b90b9165f47699790f17de4b6 Mon Sep 17 00:00:00 2001 From: Elmehdi Aitbrahim Date: Thu, 3 Sep 2026 01:49:03 -0400 Subject: [PATCH 2/2] fix(research): review findings on the cost-fidelity measurement (#371) Five findings from the review of #697, fixed on the branch rather than posted as comments. Two are substantive; the headline (141x, 2.17bp vs 306.31bp) is unchanged by any of them. 1. `corwin_schultz_spread` did not validate `block_size`. `0` reached `range(step=0)` and raised `ValueError: range() arg 3 must not be zero`; negatives built an empty block list and raised `ZeroDivisionError`. Now refused by name, the way `autonomy_on` bounds `--for-hours`. `None` keeps its meaning as the one non-positive-integer value that has one. 2. `negative_pair_share`'s docstring said "how much of the series floored to zero". Under the DEFAULT aggregation that is false -- a negative pair averages against the positives inside its block, and only a negative BLOCK mean floors. The two coincide only under `block_size=None`, which is exactly the conflation that makes that variant biased, so this was the worst possible place to be loose about it. 3. The estimator never reads `ts` and will price a pair spanning an arbitrary hole as a two-day pair. Benign for this measurement (MSFT's largest bar-to-bar gap is 4 days, TON's is 1) but unstated for a public API. The contiguity precondition is now written down, including that blocks are formed over usable PAIRS rather than over calendar time. 4. The driver derived the round trip from `2 * est.half_spread_bp`, which quantises to 0.01bp before doubling, so the published round-trip column disagreed with the spread column it comes from (GOOGL printed 1.29 and used 1.28; AAPL 2.83 and used 2.84). Now `est.spread_bp`. Two cells move by 0.01bp: GOOGL 1.51 -> 1.52, AAPL 3.07 -> 3.06. 5. THE ONE THAT MATTERED. Finding 2 of the document asserted that the cached MSFT volume was "roughly 2% ... approximately IEX's share of US equity volume" and that the understatement was ~50x. Neither was measured, and the 2% was simply wrong -- IEX publishes ~3.8% for Q2 2026. In a document whose entire value is that every number is checkable, that was a recalled figure wearing a measurement's clothes. The mechanism does not need the statistic: a single-venue feed reports a fraction of consolidated volume BY CONSTRUCTION, however faithfully it reports its own executions, and that alone establishes that the liquidity statistic cannot answer the question being asked of it. The section now argues it structurally, quotes IEX's own published share as a dated and attributed order-of-magnitude aside, and says plainly that the size of the understatement is not established here and belongs to #696. #696 is corrected the same way, and gains an acceptance criterion requiring the factor to be measured against a consolidated source rather than asserted. Test written first, red before green: tests/research/test_spread.py::test_a_block_size_that_cannot_form_a_block_is_refused Mutation-verified, 3 mutants on the new guard, all killed: guard removed; guard admitting zero; guard also rejecting `None`. Refs #371 Refs #696 Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01NzuKAe2RVrPt9acVAWjRyL --- .../2026-09-02-equities-cost-fidelity.jsonl | 24 ++++++------ .../2026-09-02-equities-cost-fidelity.md | 39 ++++++++++++------- .../2026-09-02-equities-cost-fidelity.py | 7 +++- keel/research/spread.py | 24 +++++++++++- tests/research/test_spread.py | 12 ++++++ 5 files changed, 77 insertions(+), 29 deletions(-) diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl b/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl index d5b5a5ef..7f791096 100644 --- a/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl @@ -1,29 +1,29 @@ {"product_id": "MSFT-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4984", "gap_adjusted_pairs": 147, "spread_bp": "1.94", "naive_spread_bp": "41.82", "raw_spread_bp": "-32.21", "blocks": 60, "half_spread_bp": "0.97", "median_daily_quote_volume": "186462740", "modelled_slippage_bp": "8.19", "modelled_roundtrip_bp": "16.38", "regulatory_bp": "0.2323", "sec_section_31_usd": "0.022900", "taf_usd": "0.000333", "commission_bp": "0.00", "measured_roundtrip_bp": "2.17"} -{"product_id": "AAPL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4952", "gap_adjusted_pairs": 141, "spread_bp": "2.83", "naive_spread_bp": "43.55", "raw_spread_bp": "-32.96", "blocks": 60, "half_spread_bp": "1.42", "median_daily_quote_volume": "216061742", "modelled_slippage_bp": "7.61", "modelled_roundtrip_bp": "15.21", "regulatory_bp": "0.2341", "sec_section_31_usd": "0.022900", "taf_usd": "0.000511", "commission_bp": "0.00", "measured_roundtrip_bp": "3.07"} -{"product_id": "GOOGL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.5296", "gap_adjusted_pairs": 152, "spread_bp": "1.29", "naive_spread_bp": "44.83", "raw_spread_bp": "-44.21", "blocks": 60, "half_spread_bp": "0.64", "median_daily_quote_volume": "146760641", "modelled_slippage_bp": "9.23", "modelled_roundtrip_bp": "18.46", "regulatory_bp": "0.2339", "sec_section_31_usd": "0.022900", "taf_usd": "0.000494", "commission_bp": "0.00", "measured_roundtrip_bp": "1.51"} +{"product_id": "AAPL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4952", "gap_adjusted_pairs": 141, "spread_bp": "2.83", "naive_spread_bp": "43.55", "raw_spread_bp": "-32.96", "blocks": 60, "half_spread_bp": "1.42", "median_daily_quote_volume": "216061742", "modelled_slippage_bp": "7.61", "modelled_roundtrip_bp": "15.21", "regulatory_bp": "0.2341", "sec_section_31_usd": "0.022900", "taf_usd": "0.000511", "commission_bp": "0.00", "measured_roundtrip_bp": "3.06"} +{"product_id": "GOOGL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.5296", "gap_adjusted_pairs": 152, "spread_bp": "1.29", "naive_spread_bp": "44.83", "raw_spread_bp": "-44.21", "blocks": 60, "half_spread_bp": "0.64", "median_daily_quote_volume": "146760641", "modelled_slippage_bp": "9.23", "modelled_roundtrip_bp": "18.46", "regulatory_bp": "0.2339", "sec_section_31_usd": "0.022900", "taf_usd": "0.000494", "commission_bp": "0.00", "measured_roundtrip_bp": "1.52"} {"product_id": "NVDA-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4952", "gap_adjusted_pairs": 164, "spread_bp": "4.50", "naive_spread_bp": "77.48", "raw_spread_bp": "-63.29", "blocks": 60, "half_spread_bp": "2.25", "median_daily_quote_volume": "300409740", "modelled_slippage_bp": "6.45", "modelled_roundtrip_bp": "12.90", "regulatory_bp": "0.2364", "sec_section_31_usd": "0.022900", "taf_usd": "0.000739", "commission_bp": "0.00", "measured_roundtrip_bp": "4.74"} {"product_id": "COST-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4808", "gap_adjusted_pairs": 141, "spread_bp": "1.36", "naive_spread_bp": "34.81", "raw_spread_bp": "-27.66", "blocks": 60, "half_spread_bp": "0.68", "median_daily_quote_volume": "44370750", "modelled_slippage_bp": "16.78", "modelled_roundtrip_bp": "33.57", "regulatory_bp": "0.2308", "sec_section_31_usd": "0.022900", "taf_usd": "0.000178", "commission_bp": "0.00", "measured_roundtrip_bp": "1.59"} -{"product_id": "BTC-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3289", "gap_adjusted_pairs": 3, "spread_bp": "44.67", "naive_spread_bp": "100.10", "raw_spread_bp": "38.70", "blocks": 90, "half_spread_bp": "22.33", "median_daily_quote_volume": "568492017", "modelled_slippage_bp": "5.00", "modelled_roundtrip_bp": "250.00", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "284.66"} -{"product_id": "ETH-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3364", "gap_adjusted_pairs": 1, "spread_bp": "54.73", "naive_spread_bp": "132.55", "raw_spread_bp": "46.32", "blocks": 90, "half_spread_bp": "27.37", "median_daily_quote_volume": "329719539", "modelled_slippage_bp": "6.16", "modelled_roundtrip_bp": "252.31", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "294.74"} +{"product_id": "BTC-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3289", "gap_adjusted_pairs": 3, "spread_bp": "44.67", "naive_spread_bp": "100.10", "raw_spread_bp": "38.70", "blocks": 90, "half_spread_bp": "22.33", "median_daily_quote_volume": "568492017", "modelled_slippage_bp": "5.00", "modelled_roundtrip_bp": "250.00", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "284.67"} +{"product_id": "ETH-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3364", "gap_adjusted_pairs": 1, "spread_bp": "54.73", "naive_spread_bp": "132.55", "raw_spread_bp": "46.32", "blocks": 90, "half_spread_bp": "27.37", "median_daily_quote_volume": "329719539", "modelled_slippage_bp": "6.16", "modelled_roundtrip_bp": "252.31", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "294.73"} {"product_id": "ADA-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3600", "gap_adjusted_pairs": 0, "spread_bp": "60.54", "naive_spread_bp": "167.60", "raw_spread_bp": "48.67", "blocks": 89, "half_spread_bp": "30.27", "median_daily_quote_volume": "21810985", "modelled_slippage_bp": "23.94", "modelled_roundtrip_bp": "287.88", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "300.54"} {"product_id": "LINK-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3616", "gap_adjusted_pairs": 0, "spread_bp": "69.94", "naive_spread_bp": "180.72", "raw_spread_bp": "61.57", "blocks": 89, "half_spread_bp": "34.97", "median_daily_quote_volume": "21072345", "modelled_slippage_bp": "24.36", "modelled_roundtrip_bp": "288.71", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "309.94"} -{"product_id": "LTC-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3401", "gap_adjusted_pairs": 1, "spread_bp": "65.01", "naive_spread_bp": "153.09", "raw_spread_bp": "61.09", "blocks": 89, "half_spread_bp": "32.51", "median_daily_quote_volume": "14815056", "modelled_slippage_bp": "29.05", "modelled_roundtrip_bp": "298.09", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "305.02"} +{"product_id": "LTC-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3401", "gap_adjusted_pairs": 1, "spread_bp": "65.01", "naive_spread_bp": "153.09", "raw_spread_bp": "61.09", "blocks": 89, "half_spread_bp": "32.51", "median_daily_quote_volume": "14815056", "modelled_slippage_bp": "29.05", "modelled_roundtrip_bp": "298.09", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "305.01"} {"product_id": "SOL-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3718", "gap_adjusted_pairs": 1, "spread_bp": "69.76", "naive_spread_bp": "192.37", "raw_spread_bp": "62.52", "blocks": 89, "half_spread_bp": "34.88", "median_daily_quote_volume": "107673223", "modelled_slippage_bp": "10.77", "modelled_roundtrip_bp": "261.55", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "309.76"} {"product_id": "XLM-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3718", "gap_adjusted_pairs": 0, "spread_bp": "57.44", "naive_spread_bp": "148.24", "raw_spread_bp": "43.95", "blocks": 89, "half_spread_bp": "28.72", "median_daily_quote_volume": "10004921", "modelled_slippage_bp": "35.35", "modelled_roundtrip_bp": "310.69", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "297.44"} {"product_id": "PAXG-USDT", "asset_class": "crypto", "bars": 1826, "pairs": 1825, "negative_pair_share": "0.2833", "gap_adjusted_pairs": 0, "spread_bp": "21.16", "naive_spread_bp": "39.80", "raw_spread_bp": "20.30", "blocks": 87, "half_spread_bp": "10.58", "median_daily_quote_volume": "3381426", "modelled_slippage_bp": "60.80", "modelled_roundtrip_bp": "361.60", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "261.16"} {"product_id": "BCH-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3369", "gap_adjusted_pairs": 1, "spread_bp": "70.20", "naive_spread_bp": "158.66", "raw_spread_bp": "63.15", "blocks": 89, "half_spread_bp": "35.10", "median_daily_quote_volume": "5497843", "modelled_slippage_bp": "47.68", "modelled_roundtrip_bp": "335.36", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "310.20"} -{"product_id": "AAVE-USD", "asset_class": "crypto", "bars": 1819, "pairs": 1818, "negative_pair_share": "0.3614", "gap_adjusted_pairs": 2, "spread_bp": "67.51", "naive_spread_bp": "195.25", "raw_spread_bp": "55.45", "blocks": 87, "half_spread_bp": "33.76", "median_daily_quote_volume": "6365686", "modelled_slippage_bp": "44.31", "modelled_roundtrip_bp": "328.63", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "307.52"} +{"product_id": "AAVE-USD", "asset_class": "crypto", "bars": 1819, "pairs": 1818, "negative_pair_share": "0.3614", "gap_adjusted_pairs": 2, "spread_bp": "67.51", "naive_spread_bp": "195.25", "raw_spread_bp": "55.45", "blocks": 87, "half_spread_bp": "33.76", "median_daily_quote_volume": "6365686", "modelled_slippage_bp": "44.31", "modelled_roundtrip_bp": "328.63", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "307.51"} {"product_id": "DOGE-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3556", "gap_adjusted_pairs": 2, "spread_bp": "65.98", "naive_spread_bp": "173.97", "raw_spread_bp": "58.47", "blocks": 88, "half_spread_bp": "32.99", "median_daily_quote_volume": "29636691", "modelled_slippage_bp": "20.54", "modelled_roundtrip_bp": "281.07", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "305.98"} {"product_id": "DOT-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3638", "gap_adjusted_pairs": 2, "spread_bp": "66.16", "naive_spread_bp": "174.75", "raw_spread_bp": "58.49", "blocks": 88, "half_spread_bp": "33.08", "median_daily_quote_volume": "6286489", "modelled_slippage_bp": "44.59", "modelled_roundtrip_bp": "329.18", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "306.16"} {"product_id": "UNI-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3529", "gap_adjusted_pairs": 1, "spread_bp": "74.46", "naive_spread_bp": "192.42", "raw_spread_bp": "66.66", "blocks": 88, "half_spread_bp": "37.23", "median_daily_quote_volume": "4917765", "modelled_slippage_bp": "50.42", "modelled_roundtrip_bp": "340.83", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "314.46"} {"product_id": "ZEC-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3455", "gap_adjusted_pairs": 2, "spread_bp": "88.94", "naive_spread_bp": "216.30", "raw_spread_bp": "73.56", "blocks": 89, "half_spread_bp": "44.47", "median_daily_quote_volume": "1278710", "modelled_slippage_bp": "98.87", "modelled_roundtrip_bp": "437.74", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "328.94"} -{"product_id": "ALGO-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3660", "gap_adjusted_pairs": 1, "spread_bp": "66.45", "naive_spread_bp": "191.71", "raw_spread_bp": "55.93", "blocks": 88, "half_spread_bp": "33.23", "median_daily_quote_volume": "3699580", "modelled_slippage_bp": "58.13", "modelled_roundtrip_bp": "356.25", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "306.46"} -{"product_id": "FET-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3611", "gap_adjusted_pairs": 2, "spread_bp": "113.21", "naive_spread_bp": "261.64", "raw_spread_bp": "103.98", "blocks": 88, "half_spread_bp": "56.60", "median_daily_quote_volume": "4661785", "modelled_slippage_bp": "51.78", "modelled_roundtrip_bp": "343.56", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "353.20"} +{"product_id": "ALGO-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3660", "gap_adjusted_pairs": 1, "spread_bp": "66.45", "naive_spread_bp": "191.71", "raw_spread_bp": "55.93", "blocks": 88, "half_spread_bp": "33.23", "median_daily_quote_volume": "3699580", "modelled_slippage_bp": "58.13", "modelled_roundtrip_bp": "356.25", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "306.45"} +{"product_id": "FET-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3611", "gap_adjusted_pairs": 2, "spread_bp": "113.21", "naive_spread_bp": "261.64", "raw_spread_bp": "103.98", "blocks": 88, "half_spread_bp": "56.60", "median_daily_quote_volume": "4661785", "modelled_slippage_bp": "51.78", "modelled_roundtrip_bp": "343.56", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "353.21"} {"product_id": "CRV-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3399", "gap_adjusted_pairs": 1, "spread_bp": "97.98", "naive_spread_bp": "227.64", "raw_spread_bp": "87.52", "blocks": 88, "half_spread_bp": "48.99", "median_daily_quote_volume": "3322338", "modelled_slippage_bp": "61.34", "modelled_roundtrip_bp": "362.68", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "337.98"} -{"product_id": "ICP-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3461", "gap_adjusted_pairs": 2, "spread_bp": "84.89", "naive_spread_bp": "212.25", "raw_spread_bp": "79.23", "blocks": 89, "half_spread_bp": "42.45", "median_daily_quote_volume": "4813975", "modelled_slippage_bp": "50.96", "modelled_roundtrip_bp": "341.91", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "324.90"} +{"product_id": "ICP-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3461", "gap_adjusted_pairs": 2, "spread_bp": "84.89", "naive_spread_bp": "212.25", "raw_spread_bp": "79.23", "blocks": 89, "half_spread_bp": "42.45", "median_daily_quote_volume": "4813975", "modelled_slippage_bp": "50.96", "modelled_roundtrip_bp": "341.91", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "324.89"} {"product_id": "AVAX-USD", "asset_class": "crypto", "bars": 1793, "pairs": 1792, "negative_pair_share": "0.3839", "gap_adjusted_pairs": 2, "spread_bp": "57.78", "naive_spread_bp": "181.86", "raw_spread_bp": "39.01", "blocks": 86, "half_spread_bp": "28.89", "median_daily_quote_volume": "13043019", "modelled_slippage_bp": "30.96", "modelled_roundtrip_bp": "301.92", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "297.78"} {"product_id": "NEAR-USD", "asset_class": "crypto", "bars": 1458, "pairs": 1457, "negative_pair_share": "0.3789", "gap_adjusted_pairs": 1, "spread_bp": "68.40", "naive_spread_bp": "193.36", "raw_spread_bp": "55.23", "blocks": 70, "half_spread_bp": "34.20", "median_daily_quote_volume": "3641518", "modelled_slippage_bp": "58.59", "modelled_roundtrip_bp": "357.18", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "308.40"} {"product_id": "XRP-USD", "asset_class": "crypto", "bars": 1104, "pairs": 1103, "negative_pair_share": "0.3509", "gap_adjusted_pairs": 1, "spread_bp": "52.56", "naive_spread_bp": "137.22", "raw_spread_bp": "44.97", "blocks": 53, "half_spread_bp": "26.28", "median_daily_quote_volume": "77210801", "modelled_slippage_bp": "12.72", "modelled_roundtrip_bp": "265.45", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "292.56"} -{"product_id": "PAXG-USD", "asset_class": "crypto", "bars": 482, "pairs": 481, "negative_pair_share": "0.3160", "gap_adjusted_pairs": 1, "spread_bp": "20.25", "naive_spread_bp": "49.45", "raw_spread_bp": "18.68", "blocks": 23, "half_spread_bp": "10.12", "median_daily_quote_volume": "1262880", "modelled_slippage_bp": "99.49", "modelled_roundtrip_bp": "438.98", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "260.24"} -{"product_id": "WLD-USD", "asset_class": "crypto", "bars": 449, "pairs": 448, "negative_pair_share": "0.3795", "gap_adjusted_pairs": 0, "spread_bp": "78.15", "naive_spread_bp": "228.26", "raw_spread_bp": "49.44", "blocks": 22, "half_spread_bp": "39.08", "median_daily_quote_volume": "1245627", "modelled_slippage_bp": "100.18", "modelled_roundtrip_bp": "440.35", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "318.16"} -{"product_id": "TON-USD", "asset_class": "crypto", "bars": 248, "pairs": 247, "negative_pair_share": "0.3522", "gap_adjusted_pairs": 0, "spread_bp": "48.29", "naive_spread_bp": "164.68", "raw_spread_bp": "31.74", "blocks": 12, "half_spread_bp": "24.15", "median_daily_quote_volume": "366376", "modelled_slippage_bp": "183.80", "modelled_roundtrip_bp": "607.60", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "288.30"} +{"product_id": "PAXG-USD", "asset_class": "crypto", "bars": 482, "pairs": 481, "negative_pair_share": "0.3160", "gap_adjusted_pairs": 1, "spread_bp": "20.25", "naive_spread_bp": "49.45", "raw_spread_bp": "18.68", "blocks": 23, "half_spread_bp": "10.12", "median_daily_quote_volume": "1262880", "modelled_slippage_bp": "99.49", "modelled_roundtrip_bp": "438.98", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "260.25"} +{"product_id": "WLD-USD", "asset_class": "crypto", "bars": 449, "pairs": 448, "negative_pair_share": "0.3795", "gap_adjusted_pairs": 0, "spread_bp": "78.15", "naive_spread_bp": "228.26", "raw_spread_bp": "49.44", "blocks": 22, "half_spread_bp": "39.08", "median_daily_quote_volume": "1245627", "modelled_slippage_bp": "100.18", "modelled_roundtrip_bp": "440.35", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "318.15"} +{"product_id": "TON-USD", "asset_class": "crypto", "bars": 248, "pairs": 247, "negative_pair_share": "0.3522", "gap_adjusted_pairs": 0, "spread_bp": "48.29", "naive_spread_bp": "164.68", "raw_spread_bp": "31.74", "blocks": 12, "half_spread_bp": "24.15", "median_daily_quote_volume": "366376", "modelled_slippage_bp": "183.80", "modelled_roundtrip_bp": "607.60", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "288.29"} diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.md b/docs/experiments/2026-09-02-equities-cost-fidelity.md index 6385fd7d..5d483752 100644 --- a/docs/experiments/2026-09-02-equities-cost-fidelity.md +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.md @@ -4,8 +4,8 @@ factor of 141.** keel's cost model is close to right on crypto (it charges 332bp against a measured 306bp, conservative by 8%) and wrong by 7.5× on equities, where it charges 16.4bp against a measured 2.2bp. Both errors are now explained, and the equities one has a specific -cause: the IEX feed reports only IEX's own executions, so keel's liquidity statistic reads -MSFT as a $186M/day product when it trades several billion. +cause: the equities profile runs on a single-venue (IEX) feed, so keel's liquidity statistic +reads MSFT as a $186M/day product and prices it as thinner than the model's reference liquidity. The PRD makes this a precondition — "no strategy claim is believed before it" (§6.3, O4) — and that ordering earned its keep. `config.paper-equities.yaml` ships `taker_pct: 0.0`, honestly @@ -39,10 +39,10 @@ Round trip = commission (both legs) + spread (half per leg) + regulatory (sell o | ticker | bars | spread bp | naive bp | raw bp | reg bp | measured RT bp | keel charges bp | | --- | --- | --- | --- | --- | --- | --- | --- | -| GOOGL | 1253 | 1.29 | 44.83 | -44.21 | 0.2339 | **1.51** | 18.46 | +| GOOGL | 1253 | 1.29 | 44.83 | -44.21 | 0.2339 | **1.52** | 18.46 | | COST | 1253 | 1.36 | 34.81 | -27.66 | 0.2308 | **1.59** | 33.57 | | MSFT | 1253 | 1.94 | 41.82 | -32.21 | 0.2323 | **2.17** | 16.38 | -| AAPL | 1253 | 2.83 | 43.55 | -32.96 | 0.2341 | **3.07** | 15.21 | +| AAPL | 1253 | 2.83 | 43.55 | -32.96 | 0.2341 | **3.06** | 15.21 | | NVDA | 1253 | 4.50 | 77.48 | -63.29 | 0.2364 | **4.74** | 12.90 | | | median spread | median measured RT | median keel charges | model error | @@ -86,7 +86,7 @@ asset class this measurement exists to price. `keel.research.spread` keeps both reachable, because the comparison between them is itself the finding, and the biased one is what makes the bias demonstrable rather than folklore. -## Finding 2 — keel reads MSFT as a thin asset, because IEX is 2% of the market +## Finding 2 — keel reads MSFT as a thin asset, because a single-venue feed is not the market `slippage_for_quote_volume` keys off `median_daily_quote_volume` computed from cached candles, against a $500M/day anchor with a 5bp floor. What it sees: @@ -100,14 +100,25 @@ against a $500M/day anchor with a 5bp floor. What it sees: | COST | $44,370,750 | 16.78bp | Every one sits below the anchor, so every one is priced as *thinner than the model's reference -liquidity*. MSFT's real consolidated volume is several billion dollars a day. The cached figure -is roughly 2% of it — which is approximately IEX's share of US equity volume, and the equities -profile runs on the IEX feed by configuration. - -**The candles are not wrong; the statistic is being asked a question the feed cannot answer.** -IEX reports its own executions faithfully. A model keyed on that volume will misread every US -equity as ~50× thinner than it is, and will do so silently, because nothing in the pipeline -knows the difference between "this product is thin" and "this feed sees 2% of the market". +liquidity* — including MSFT, which is not a thin instrument by any reading. + +The mechanism does not need a market-share statistic to establish, and is stated structurally +here for that reason. **The equities profile runs on Alpaca's IEX feed by configuration, and +IEX is one exchange among the many US venues (plus off-exchange execution) that make up +consolidated volume.** A single-venue feed therefore reports a fraction of consolidated volume +BY CONSTRUCTION, however faithfully it reports its own executions. Any statistic that treats +that fraction as though it were the whole market will understate liquidity by whatever that +venue's share happens to be, on every symbol, silently — because nothing in the pipeline +distinguishes "this product is thin" from "this feed sees part of the market". + +For scale, and flagged as an order-of-magnitude expectation rather than a measurement: IEX +publishes its own overall share as roughly 3.8% for Q2 2026 ([iex.io](https://www.iex.io/news), +the operator's own figure, not independently verified here), and the cached MSFT series is a +low-single-digit percentage of the consolidated volume a mega-cap of that size is generally +understood to trade. **This document does not measure either quantity**, and no number in the +tables above depends on them — the finding is that the statistic is structurally unable to +answer the question, not that it is off by a specific factor. Establishing the actual factor is +part of #696, not of this document. This is the SIP-vs-IEX data-tier implication #371 asked about, and it is more consequential than the fee question it was filed beside. Two consequences worth separating: @@ -116,7 +127,7 @@ the fee question it was filed beside. Two consequences worth separating: anyway, and even the floor is ~2.5× the measured 1.94bp spread. Both the floor and the volume statistic are crypto-shaped. 2. **For anything else keyed on volume** — liquidity screens, admission floors, the asset - scout's `--probe-liquidity` — the same 50× understatement applies, and a genuinely liquid + scout's `--probe-liquidity` — the same understatement applies, and a genuinely liquid equity could be refused admission for thinness it does not have. That direction fails closed, so it is safe — but safe for the wrong reason, and it will misinform any equities universe decision. Filed as **#696**; nothing in this document changes it. diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.py b/docs/experiments/2026-09-02-equities-cost-fidelity.py index eb0c31c1..553d1eb4 100644 --- a/docs/experiments/2026-09-02-equities-cost-fidelity.py +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.py @@ -129,7 +129,12 @@ def measure(db: str, product_id: str, asset_class: str) -> dict[str, object] | N # beside the estimated spread. Omitting them would compare an equities total that is # ~all spread against a crypto total that is ~all fee, and understate crypto 5-fold. commission_bp = (Decimal("0") if asset_class == "equities" else 2 * TAKER_FEE_PCT) * 10000 - measured_rt_bp = commission_bp + 2 * est.half_spread_bp + reg_bp + # `est.spread_bp`, NOT `2 * est.half_spread_bp`: the half is quantised to 0.01bp before + # doubling, so the round trip disagreed with the spread column it is derived from by up to + # 0.01bp in either direction (GOOGL published 1.29 and used 1.28; AAPL 2.83 and used 2.84). + # Immaterial to the conclusion, and a table whose columns do not add up invites the reader + # to distrust the ones that matter. + measured_rt_bp = commission_bp + est.spread_bp + reg_bp return { "product_id": product_id, diff --git a/keel/research/spread.py b/keel/research/spread.py index 7d42d972..9f6e0226 100644 --- a/keel/research/spread.py +++ b/keel/research/spread.py @@ -127,8 +127,14 @@ def half_spread_bp(self) -> Decimal: @property def negative_pair_share(self) -> Decimal: - """How much of the series floored to zero -- the noise diagnostic. Above roughly half - and the mean is being carried by a minority of pairs.""" + """How much of the series estimated a negative spread -- the noise diagnostic. + + NOT "how much floored to zero": under the default block aggregation a negative PAIR + does not floor, it averages against the positives inside its block, and only a + negative BLOCK mean floors. (Under `block_size=None` the two coincide, which is + exactly the conflation that makes that variant biased.) Read it as signal-to-noise: + approaching half means the estimator cannot resolve a spread in this series at this + frequency, whichever aggregation is then applied.""" if self.pairs == 0: return Decimal("0") return (Decimal(self.negative_pairs) / Decimal(self.pairs)).quantize(Decimal("0.0001")) @@ -192,7 +198,21 @@ def corwin_schultz_spread( or no usable pair. That is a different statement from an estimate of zero, and a caller must not read the two the same way: one says the venue is cheap, the other says the question was not answered. + + **PRECONDITION, UNCHECKED: `candles` must be CONSECUTIVE bars of one series.** `ts` is + never read. Every adjacent pair is treated as a two-day pair, so a series with a hole in + it -- a fetch gap, two series concatenated, a filtered subset -- prices that hole as an + overnight move and returns a number with no warning attached. Blocks are likewise formed + over usable PAIRS, not over calendar time, so a gappy series' "month" can span far more + than a month. Checking adjacency here would need a granularity this function is not given; + the caller has it, and owns this. """ + if block_size is not None and block_size < 1: + # Caught by name rather than left to the slice arithmetic: 0 reached `range(step=0)` + # and negatives produced an empty block list to divide by. `None` is the one + # non-positive-integer value with a meaning, and it keeps it. + raise ValueError(f"block_size must be a positive integer or None; got {block_size!r}") + estimates: list[float] = [] negative = 0 gapped = 0 diff --git a/tests/research/test_spread.py b/tests/research/test_spread.py index ae4f9204..0b954926 100644 --- a/tests/research/test_spread.py +++ b/tests/research/test_spread.py @@ -226,6 +226,18 @@ def test_a_floored_block_still_counts_toward_the_average() -> None: ) +def test_a_block_size_that_cannot_form_a_block_is_refused() -> None: + """Zero and negative block sizes are rejected by name rather than by whatever the slice + arithmetic happens to raise -- `block_size=0` reached `range(step=0)` and `-1` divided by + an empty block list. `None` is the one non-positive-integer value with a meaning, and it + keeps it.""" + candles = [_c("101", "99", ts=i) for i in range(30)] + for bad in (0, -1, -21): + with pytest.raises(ValueError, match="block_size"): + corwin_schultz_spread(candles, block_size=bad) + assert corwin_schultz_spread(candles, block_size=None) is not None + + def test_the_block_size_defaults_to_a_trading_month() -> None: assert MONTHLY_BLOCK == 21