diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl b/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl new file mode 100644 index 00000000..7f791096 --- /dev/null +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.jsonl @@ -0,0 +1,29 @@ +{"product_id": "MSFT-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4984", "gap_adjusted_pairs": 147, "spread_bp": "1.94", "naive_spread_bp": "41.82", "raw_spread_bp": "-32.21", "blocks": 60, "half_spread_bp": "0.97", "median_daily_quote_volume": "186462740", "modelled_slippage_bp": "8.19", "modelled_roundtrip_bp": "16.38", "regulatory_bp": "0.2323", "sec_section_31_usd": "0.022900", "taf_usd": "0.000333", "commission_bp": "0.00", "measured_roundtrip_bp": "2.17"} +{"product_id": "AAPL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4952", "gap_adjusted_pairs": 141, "spread_bp": "2.83", "naive_spread_bp": "43.55", "raw_spread_bp": "-32.96", "blocks": 60, "half_spread_bp": "1.42", "median_daily_quote_volume": "216061742", "modelled_slippage_bp": "7.61", "modelled_roundtrip_bp": "15.21", "regulatory_bp": "0.2341", "sec_section_31_usd": "0.022900", "taf_usd": "0.000511", "commission_bp": "0.00", "measured_roundtrip_bp": "3.06"} +{"product_id": "GOOGL-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.5296", "gap_adjusted_pairs": 152, "spread_bp": "1.29", "naive_spread_bp": "44.83", "raw_spread_bp": "-44.21", "blocks": 60, "half_spread_bp": "0.64", "median_daily_quote_volume": "146760641", "modelled_slippage_bp": "9.23", "modelled_roundtrip_bp": "18.46", "regulatory_bp": "0.2339", "sec_section_31_usd": "0.022900", "taf_usd": "0.000494", "commission_bp": "0.00", "measured_roundtrip_bp": "1.52"} +{"product_id": "NVDA-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4952", "gap_adjusted_pairs": 164, "spread_bp": "4.50", "naive_spread_bp": "77.48", "raw_spread_bp": "-63.29", "blocks": 60, "half_spread_bp": "2.25", "median_daily_quote_volume": "300409740", "modelled_slippage_bp": "6.45", "modelled_roundtrip_bp": "12.90", "regulatory_bp": "0.2364", "sec_section_31_usd": "0.022900", "taf_usd": "0.000739", "commission_bp": "0.00", "measured_roundtrip_bp": "4.74"} +{"product_id": "COST-USD", "asset_class": "equities", "bars": 1253, "pairs": 1252, "negative_pair_share": "0.4808", "gap_adjusted_pairs": 141, "spread_bp": "1.36", "naive_spread_bp": "34.81", "raw_spread_bp": "-27.66", "blocks": 60, "half_spread_bp": "0.68", "median_daily_quote_volume": "44370750", "modelled_slippage_bp": "16.78", "modelled_roundtrip_bp": "33.57", "regulatory_bp": "0.2308", "sec_section_31_usd": "0.022900", "taf_usd": "0.000178", "commission_bp": "0.00", "measured_roundtrip_bp": "1.59"} +{"product_id": "BTC-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3289", "gap_adjusted_pairs": 3, "spread_bp": "44.67", "naive_spread_bp": "100.10", "raw_spread_bp": "38.70", "blocks": 90, "half_spread_bp": "22.33", "median_daily_quote_volume": "568492017", "modelled_slippage_bp": "5.00", "modelled_roundtrip_bp": "250.00", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "284.67"} +{"product_id": "ETH-USD", "asset_class": "crypto", "bars": 1871, "pairs": 1870, "negative_pair_share": "0.3364", "gap_adjusted_pairs": 1, "spread_bp": "54.73", "naive_spread_bp": "132.55", "raw_spread_bp": "46.32", "blocks": 90, "half_spread_bp": "27.37", "median_daily_quote_volume": "329719539", "modelled_slippage_bp": "6.16", "modelled_roundtrip_bp": "252.31", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "294.73"} +{"product_id": "ADA-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3600", "gap_adjusted_pairs": 0, "spread_bp": "60.54", "naive_spread_bp": "167.60", "raw_spread_bp": "48.67", "blocks": 89, "half_spread_bp": "30.27", "median_daily_quote_volume": "21810985", "modelled_slippage_bp": "23.94", "modelled_roundtrip_bp": "287.88", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "300.54"} +{"product_id": "LINK-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3616", "gap_adjusted_pairs": 0, "spread_bp": "69.94", "naive_spread_bp": "180.72", "raw_spread_bp": "61.57", "blocks": 89, "half_spread_bp": "34.97", "median_daily_quote_volume": "21072345", "modelled_slippage_bp": "24.36", "modelled_roundtrip_bp": "288.71", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "309.94"} +{"product_id": "LTC-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3401", "gap_adjusted_pairs": 1, "spread_bp": "65.01", "naive_spread_bp": "153.09", "raw_spread_bp": "61.09", "blocks": 89, "half_spread_bp": "32.51", "median_daily_quote_volume": "14815056", "modelled_slippage_bp": "29.05", "modelled_roundtrip_bp": "298.09", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "305.01"} +{"product_id": "SOL-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3718", "gap_adjusted_pairs": 1, "spread_bp": "69.76", "naive_spread_bp": "192.37", "raw_spread_bp": "62.52", "blocks": 89, "half_spread_bp": "34.88", "median_daily_quote_volume": "107673223", "modelled_slippage_bp": "10.77", "modelled_roundtrip_bp": "261.55", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "309.76"} +{"product_id": "XLM-USD", "asset_class": "crypto", "bars": 1865, "pairs": 1864, "negative_pair_share": "0.3718", "gap_adjusted_pairs": 0, "spread_bp": "57.44", "naive_spread_bp": "148.24", "raw_spread_bp": "43.95", "blocks": 89, "half_spread_bp": "28.72", "median_daily_quote_volume": "10004921", "modelled_slippage_bp": "35.35", "modelled_roundtrip_bp": "310.69", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "297.44"} +{"product_id": "PAXG-USDT", "asset_class": "crypto", "bars": 1826, "pairs": 1825, "negative_pair_share": "0.2833", "gap_adjusted_pairs": 0, "spread_bp": "21.16", "naive_spread_bp": "39.80", "raw_spread_bp": "20.30", "blocks": 87, "half_spread_bp": "10.58", "median_daily_quote_volume": "3381426", "modelled_slippage_bp": "60.80", "modelled_roundtrip_bp": "361.60", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "261.16"} +{"product_id": "BCH-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3369", "gap_adjusted_pairs": 1, "spread_bp": "70.20", "naive_spread_bp": "158.66", "raw_spread_bp": "63.15", "blocks": 89, "half_spread_bp": "35.10", "median_daily_quote_volume": "5497843", "modelled_slippage_bp": "47.68", "modelled_roundtrip_bp": "335.36", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "310.20"} +{"product_id": "AAVE-USD", "asset_class": "crypto", "bars": 1819, "pairs": 1818, "negative_pair_share": "0.3614", "gap_adjusted_pairs": 2, "spread_bp": "67.51", "naive_spread_bp": "195.25", "raw_spread_bp": "55.45", "blocks": 87, "half_spread_bp": "33.76", "median_daily_quote_volume": "6365686", "modelled_slippage_bp": "44.31", "modelled_roundtrip_bp": "328.63", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "307.51"} +{"product_id": "DOGE-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3556", "gap_adjusted_pairs": 2, "spread_bp": "65.98", "naive_spread_bp": "173.97", "raw_spread_bp": "58.47", "blocks": 88, "half_spread_bp": "32.99", "median_daily_quote_volume": "29636691", "modelled_slippage_bp": "20.54", "modelled_roundtrip_bp": "281.07", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "305.98"} +{"product_id": "DOT-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3638", "gap_adjusted_pairs": 2, "spread_bp": "66.16", "naive_spread_bp": "174.75", "raw_spread_bp": "58.49", "blocks": 88, "half_spread_bp": "33.08", "median_daily_quote_volume": "6286489", "modelled_slippage_bp": "44.59", "modelled_roundtrip_bp": "329.18", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "306.16"} +{"product_id": "UNI-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3529", "gap_adjusted_pairs": 1, "spread_bp": "74.46", "naive_spread_bp": "192.42", "raw_spread_bp": "66.66", "blocks": 88, "half_spread_bp": "37.23", "median_daily_quote_volume": "4917765", "modelled_slippage_bp": "50.42", "modelled_roundtrip_bp": "340.83", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "314.46"} +{"product_id": "ZEC-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3455", "gap_adjusted_pairs": 2, "spread_bp": "88.94", "naive_spread_bp": "216.30", "raw_spread_bp": "73.56", "blocks": 89, "half_spread_bp": "44.47", "median_daily_quote_volume": "1278710", "modelled_slippage_bp": "98.87", "modelled_roundtrip_bp": "437.74", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "328.94"} +{"product_id": "ALGO-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3660", "gap_adjusted_pairs": 1, "spread_bp": "66.45", "naive_spread_bp": "191.71", "raw_spread_bp": "55.93", "blocks": 88, "half_spread_bp": "33.23", "median_daily_quote_volume": "3699580", "modelled_slippage_bp": "58.13", "modelled_roundtrip_bp": "356.25", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "306.45"} +{"product_id": "FET-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3611", "gap_adjusted_pairs": 2, "spread_bp": "113.21", "naive_spread_bp": "261.64", "raw_spread_bp": "103.98", "blocks": 88, "half_spread_bp": "56.60", "median_daily_quote_volume": "4661785", "modelled_slippage_bp": "51.78", "modelled_roundtrip_bp": "343.56", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "353.21"} +{"product_id": "CRV-USD", "asset_class": "crypto", "bars": 1840, "pairs": 1839, "negative_pair_share": "0.3399", "gap_adjusted_pairs": 1, "spread_bp": "97.98", "naive_spread_bp": "227.64", "raw_spread_bp": "87.52", "blocks": 88, "half_spread_bp": "48.99", "median_daily_quote_volume": "3322338", "modelled_slippage_bp": "61.34", "modelled_roundtrip_bp": "362.68", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "337.98"} +{"product_id": "ICP-USD", "asset_class": "crypto", "bars": 1859, "pairs": 1858, "negative_pair_share": "0.3461", "gap_adjusted_pairs": 2, "spread_bp": "84.89", "naive_spread_bp": "212.25", "raw_spread_bp": "79.23", "blocks": 89, "half_spread_bp": "42.45", "median_daily_quote_volume": "4813975", "modelled_slippage_bp": "50.96", "modelled_roundtrip_bp": "341.91", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "324.89"} +{"product_id": "AVAX-USD", "asset_class": "crypto", "bars": 1793, "pairs": 1792, "negative_pair_share": "0.3839", "gap_adjusted_pairs": 2, "spread_bp": "57.78", "naive_spread_bp": "181.86", "raw_spread_bp": "39.01", "blocks": 86, "half_spread_bp": "28.89", "median_daily_quote_volume": "13043019", "modelled_slippage_bp": "30.96", "modelled_roundtrip_bp": "301.92", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "297.78"} +{"product_id": "NEAR-USD", "asset_class": "crypto", "bars": 1458, "pairs": 1457, "negative_pair_share": "0.3789", "gap_adjusted_pairs": 1, "spread_bp": "68.40", "naive_spread_bp": "193.36", "raw_spread_bp": "55.23", "blocks": 70, "half_spread_bp": "34.20", "median_daily_quote_volume": "3641518", "modelled_slippage_bp": "58.59", "modelled_roundtrip_bp": "357.18", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "308.40"} +{"product_id": "XRP-USD", "asset_class": "crypto", "bars": 1104, "pairs": 1103, "negative_pair_share": "0.3509", "gap_adjusted_pairs": 1, "spread_bp": "52.56", "naive_spread_bp": "137.22", "raw_spread_bp": "44.97", "blocks": 53, "half_spread_bp": "26.28", "median_daily_quote_volume": "77210801", "modelled_slippage_bp": "12.72", "modelled_roundtrip_bp": "265.45", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "292.56"} +{"product_id": "PAXG-USD", "asset_class": "crypto", "bars": 482, "pairs": 481, "negative_pair_share": "0.3160", "gap_adjusted_pairs": 1, "spread_bp": "20.25", "naive_spread_bp": "49.45", "raw_spread_bp": "18.68", "blocks": 23, "half_spread_bp": "10.12", "median_daily_quote_volume": "1262880", "modelled_slippage_bp": "99.49", "modelled_roundtrip_bp": "438.98", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "260.25"} +{"product_id": "WLD-USD", "asset_class": "crypto", "bars": 449, "pairs": 448, "negative_pair_share": "0.3795", "gap_adjusted_pairs": 0, "spread_bp": "78.15", "naive_spread_bp": "228.26", "raw_spread_bp": "49.44", "blocks": 22, "half_spread_bp": "39.08", "median_daily_quote_volume": "1245627", "modelled_slippage_bp": "100.18", "modelled_roundtrip_bp": "440.35", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "318.15"} +{"product_id": "TON-USD", "asset_class": "crypto", "bars": 248, "pairs": 247, "negative_pair_share": "0.3522", "gap_adjusted_pairs": 0, "spread_bp": "48.29", "naive_spread_bp": "164.68", "raw_spread_bp": "31.74", "blocks": 12, "half_spread_bp": "24.15", "median_daily_quote_volume": "366376", "modelled_slippage_bp": "183.80", "modelled_roundtrip_bp": "607.60", "regulatory_bp": "0.0000", "sec_section_31_usd": "0.000000", "taf_usd": "0.000000", "commission_bp": "240.00", "measured_roundtrip_bp": "288.29"} diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.md b/docs/experiments/2026-09-02-equities-cost-fidelity.md new file mode 100644 index 00000000..5d483752 --- /dev/null +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.md @@ -0,0 +1,167 @@ +# 2026-09-02: What a round trip costs on a commission-free venue (#371, Alpaca Phase C) + +**Headline: a round trip costs ~2.2bp on Alpaca equities and ~306bp on Coinbase spot — a +factor of 141.** keel's cost model is close to right on crypto (it charges 332bp against a +measured 306bp, conservative by 8%) and wrong by 7.5× on equities, where it charges 16.4bp +against a measured 2.2bp. Both errors are now explained, and the equities one has a specific +cause: the equities profile runs on a single-venue (IEX) feed, so keel's liquidity statistic +reads MSFT as a $186M/day product and prices it as thinner than the model's reference liquidity. + +The PRD makes this a precondition — "no strategy claim is believed before it" (§6.3, O4) — and +that ordering earned its keep. `config.paper-equities.yaml` ships `taker_pct: 0.0`, honestly +labelled as awaiting this work. Priced literally, that models a free venue, and a free venue +makes everything look profitable. + +**This measures COST ONLY.** It makes no claim about edge on either asset class. The crypto +intersection is empty; the equity rules have never been measured at all. Nothing below changes +either fact, and a cheaper venue is not an edge. + +## What was measured, on what + +- Data: the deployment's own cached ONE_DAY candles — `keel-equities.db` (5 tickers × 1,253 + bars, Alpaca IEX feed) and `keel.db` (24 assets, 248–1,871 bars, Coinbase). +- Spread: `keel.research.spread.corwin_schultz_spread` — Corwin & Schultz (2012), *"A Simple + Way to Estimate Bid-Ask Spreads from Daily High and Low Prices"*, Journal of Finance 67(2), + 719–759. **One estimator, both asset classes, from data already on disk.** A comparison + measured two different ways would be an artifact of the methods rather than a finding about + the venues, and buying a quote history would make the number unreproducible by a reader. +- Overnight-gap adjustment ON (§I.B of the paper). Equities gap nightly; crypto trades + continuously. Leaving it off inflates the equities side of exactly this comparison — the + direction that flatters keel's existing crypto-heavy prior. +- Commission and regulatory fees are **computed, not estimated**: 1.2%/leg is Coinbase's + published taker rate, $0 is Alpaca's published commission, and the sell-side pass-throughs + come from `keel_broker_alpaca.fees`, whose constants carry their own provenance. +- Driver: [`2026-09-02-equities-cost-fidelity.py`](2026-09-02-equities-cost-fidelity.py). + +## The result + +Round trip = commission (both legs) + spread (half per leg) + regulatory (sell only). + +| ticker | bars | spread bp | naive bp | raw bp | reg bp | measured RT bp | keel charges bp | +| --- | --- | --- | --- | --- | --- | --- | --- | +| GOOGL | 1253 | 1.29 | 44.83 | -44.21 | 0.2339 | **1.52** | 18.46 | +| COST | 1253 | 1.36 | 34.81 | -27.66 | 0.2308 | **1.59** | 33.57 | +| MSFT | 1253 | 1.94 | 41.82 | -32.21 | 0.2323 | **2.17** | 16.38 | +| AAPL | 1253 | 2.83 | 43.55 | -32.96 | 0.2341 | **3.06** | 15.21 | +| NVDA | 1253 | 4.50 | 77.48 | -63.29 | 0.2364 | **4.74** | 12.90 | + +| | median spread | median measured RT | median keel charges | model error | +| --- | --- | --- | --- | --- | +| equities (5) | 1.94bp | **2.17bp** | 16.38bp | **7.5× too high** | +| crypto (24) | 66.31bp | **306.31bp** | 332.27bp | 1.08× (conservative) | + +**Ratio: 141×.** The pre-declared expectation was "one to two orders of magnitude"; the +measurement landed inside it. The refutation condition was an equities spread at or above 50bp +or a material regulatory term — neither occurred. The regulatory term is real but negligible at +this account's size: on a $1,000 sell, SEC Section 31 is $0.0229 and FINRA TAF is $0.000333, +together **0.23bp**, about a tenth of the spread. + +## Finding 1 — the aggregation choice is worth 22×, and the obvious one is wrong + +This is the part that nearly produced a wrong document. The first run of this measurement +reported equities at 43.6bp and a ratio of 4.1×, and it was the *plausibility* of MSFT quoting +41.8bp — roughly twenty times the penny it actually quotes — that exposed the error rather than +any test. + +Corwin-Schultz two-day estimates go negative often, and no venue charges a negative spread, so +they must be handled. The tempting treatment is to floor every pair at zero and average the +survivors. On a series whose true spread is small relative to its volatility, the estimate is +symmetric noise about zero, and keeping the positive half while discarding the negative half +makes the mean converge on E[max(X,0)] > 0. **It reports a spread that is not there, and +reports a larger one the more volatile the series is.** The `raw bp` column is the giveaway: +every equity's unfloored mean is strongly NEGATIVE, which is the honest statement that the +series carries no spread the estimator can resolve at daily frequency. + +Corwin & Schultz's own procedure averages within a month and floors the *monthly* mean, so the +negatives cancel against the positives inside the block instead of being discarded. Same data, +same estimator: + +| | naive (floor each pair) | blocked (floor each month) | ratio | +| --- | --- | --- | --- | +| equities | 43.55bp | 1.94bp | **22.4×** | +| crypto | 177.74bp | 66.31bp | 2.7× | + +The bias is worst exactly where the true spread is smallest — which is to say, exactly on the +asset class this measurement exists to price. `keel.research.spread` keeps both aggregations +reachable, because the comparison between them is itself the finding, and the biased one is +what makes the bias demonstrable rather than folklore. + +## Finding 2 — keel reads MSFT as a thin asset, because a single-venue feed is not the market + +`slippage_for_quote_volume` keys off `median_daily_quote_volume` computed from cached candles, +against a $500M/day anchor with a 5bp floor. What it sees: + +| ticker | cached median daily quote volume | modelled slippage | +| --- | --- | --- | +| MSFT | $186,462,740 | 8.19bp | +| AAPL | $216,061,742 | 7.61bp | +| GOOGL | $146,760,641 | 9.23bp | +| NVDA | $300,409,740 | 6.45bp | +| COST | $44,370,750 | 16.78bp | + +Every one sits below the anchor, so every one is priced as *thinner than the model's reference +liquidity* — including MSFT, which is not a thin instrument by any reading. + +The mechanism does not need a market-share statistic to establish, and is stated structurally +here for that reason. **The equities profile runs on Alpaca's IEX feed by configuration, and +IEX is one exchange among the many US venues (plus off-exchange execution) that make up +consolidated volume.** A single-venue feed therefore reports a fraction of consolidated volume +BY CONSTRUCTION, however faithfully it reports its own executions. Any statistic that treats +that fraction as though it were the whole market will understate liquidity by whatever that +venue's share happens to be, on every symbol, silently — because nothing in the pipeline +distinguishes "this product is thin" from "this feed sees part of the market". + +For scale, and flagged as an order-of-magnitude expectation rather than a measurement: IEX +publishes its own overall share as roughly 3.8% for Q2 2026 ([iex.io](https://www.iex.io/news), +the operator's own figure, not independently verified here), and the cached MSFT series is a +low-single-digit percentage of the consolidated volume a mega-cap of that size is generally +understood to trade. **This document does not measure either quantity**, and no number in the +tables above depends on them — the finding is that the statistic is structurally unable to +answer the question, not that it is off by a specific factor. Establishing the actual factor is +part of #696, not of this document. + +This is the SIP-vs-IEX data-tier implication #371 asked about, and it is more consequential than +the fee question it was filed beside. Two consequences worth separating: + +1. **For cost modelling** (this document): the 5bp floor is the binding term for mega-caps + anyway, and even the floor is ~2.5× the measured 1.94bp spread. Both the floor and the + volume statistic are crypto-shaped. +2. **For anything else keyed on volume** — liquidity screens, admission floors, the asset + scout's `--probe-liquidity` — the same understatement applies, and a genuinely liquid + equity could be refused admission for thinness it does not have. That direction fails + closed, so it is safe — but safe for the wrong reason, and it will misinform any equities + universe decision. Filed as **#696**; nothing in this document changes it. + +## Finding 3 — the crypto model is validated, in passing + +332.27bp modelled against 306.31bp measured, conservative by 8%, using an estimator with no +knowledge of Coinbase's fee schedule. The two agree because the fee dominates: 240bp of the +306bp is published commission, and the spread estimate only has to be roughly right for the +total to be. This is the first independent check on keel's crypto cost assumption that does not +reuse keel's own fee constant, and it passes. + +Read the crypto spread column on its own terms, though: PAXG at ~20bp and FET at ~113bp is a +5.6× dispersion that a single global slippage figure cannot express. That is the same argument +[the per-product restatement](2026-09-01-per-product-slippage-restatement.md) made from the +volume side, arriving independently from the price side. + +## What this does and does not license + +It licenses one sentence: **the cost regime that killed every crypto result is not present on +equities.** 2.2bp is not 306bp, and a strategy needing 300bp of gross edge to break even is +playing a different game from one needing 3bp. + +It licenses nothing about edge. Every measured rule is negative on crypto for three unrelated +reasons, and only one of them — `turtle_breakout`'s real gross edge destroyed by cost — is even +addressable by a cheaper venue. `pullback_continuation` has essentially no gross edge (median +0.77) and will not acquire one on a different asset class; `rsi_meanrev` fails on sample size. +A 141× cost reduction is a necessary condition for those first-family results, not a sufficient +one, and the equity rules remain entirely unmeasured. The DCA benchmark on these same tickers is +the next step and is deliberately not in this document. + +One caveat on the equity numbers specifically: ~150 of each ticker's ~1,250 pairs are +gap-adjusted, and a gap-adjusted pair floors to zero by construction (the adjustment leaves day +2 sitting exactly atop day 1, the most trend-like geometry two ranges can have). The equity +estimates therefore rest on ~1,100 effective pairs, not 1,250. This does not bias them upward — +gap days contribute zero rather than noise — but it widens their uncertainty relative to the +crypto series, which barely gap at all (0–3 pairs each). diff --git a/docs/experiments/2026-09-02-equities-cost-fidelity.py b/docs/experiments/2026-09-02-equities-cost-fidelity.py new file mode 100644 index 00000000..553d1eb4 --- /dev/null +++ b/docs/experiments/2026-09-02-equities-cost-fidelity.py @@ -0,0 +1,212 @@ +"""What a round trip actually costs on a commission-free venue -- issue #371 (Alpaca Phase C). + +**The question.** Every cost claim keel has ever made is about Coinbase, where the taker fee +(1.2%/leg) dwarfs everything else and the spread is a rounding error beside it. Alpaca charges +NO commission on US equities, so the entire cost is the two terms keel has never had to measure +carefully: the sell-side regulatory pass-throughs, and the spread. The PRD (§6.3, O4) makes +this measurement a PRECONDITION -- "no strategy claim is believed before it" -- because the +failure mode is obvious and seductive: an equities backtest priced at `taker_pct: 0.0` (which is +what `config.paper-equities.yaml` ships today, honestly labelled as awaiting this work) models a +FREE venue, and a free venue makes everything look profitable. + +DECLARED BEFORE THE RUN: + +* **One estimator, both asset classes, same data type.** Corwin-Schultz (2012) on daily + highs/lows, from candles already cached. A comparison measured two different ways would be an + artifact of the methods, not a finding about the venues. Overnight-gap adjustment ON (equities + gap, crypto does not; leaving it off would inflate the equities side of exactly this + comparison -- the direction that flatters keel's existing prior). +* **The regulatory term is computed, not estimated**: `keel_broker_alpaca.fees` already encodes + SEC Section 31 and FINRA TAF from published formulas, with provenance. +* **Primary quantity: total round-trip friction in basis points**, equities vs crypto, and + against what keel's model charges each today. +* **Pre-declared expectation**: equities friction lands one to two ORDERS OF MAGNITUDE below + crypto's ~250bp, and keel's current equities pricing is wrong in BOTH directions at once -- + under-charging fees (0 instead of the pass-throughs) and over-charging slippage (the 5bp + floor of a crypto-anchored model, on books far deeper than its $500M/day anchor). +* **What would refute it**: an equities spread estimate at or above 50bp, or a regulatory term + material against the spread. Either would mean commission-free is a marketing claim rather + than a cost structure, and the equities profile would need the same null treatment as crypto. + +This measures COST ONLY. It makes no claim about edge on either asset class, and nothing here +should be read as one -- the crypto intersection is empty, and the equity rules have never been +measured at all. + +Re-run: + KEEL_EQUITIES_DB=~/keel/keel-equities.db KEEL_CRYPTO_DB=~/keel/keel.db \ + python docs/experiments/2026-09-02-equities-cost-fidelity.py +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import statistics +from decimal import Decimal +from pathlib import Path + +from keel_broker_alpaca.fees import estimate_regulatory_fees +from keel_core.types import Candle, Side + +from keel.research.spread import corwin_schultz_spread +from keel.strategy.backtest import SLIPPAGE_FLOOR_PCT, TAKER_FEE_PCT, slippage_for_quote_volume + +EQUITIES_DB = os.environ.get("KEEL_EQUITIES_DB") or str(Path.home() / "keel" / "keel-equities.db") +CRYPTO_DB = os.environ.get("KEEL_CRYPTO_DB") or str(Path.home() / "keel" / "keel.db") +OUT_DIR = Path(os.environ.get("KEEL_EXPERIMENT_OUT") or Path(__file__).resolve().parent / "_out") +OUT_DIR.mkdir(parents=True, exist_ok=True) +JSONL_PATH = OUT_DIR / "equities_cost_fidelity.jsonl" + +EQUITIES = ["MSFT-USD", "AAPL-USD", "GOOGL-USD", "NVDA-USD", "COST-USD"] +CRYPTO = [ + "BTC-USD", "ETH-USD", "ADA-USD", "LINK-USD", "LTC-USD", "SOL-USD", + "XLM-USD", "PAXG-USDT", "BCH-USD", "AAVE-USD", "DOGE-USD", "DOT-USD", + "UNI-USD", "ZEC-USD", "ALGO-USD", "FET-USD", "CRV-USD", "ICP-USD", + "AVAX-USD", "NEAR-USD", "XRP-USD", "PAXG-USD", "WLD-USD", "TON-USD", +] + +#: The order size the regulatory term is expressed at. keel's equities profile caps exposure at +#: $5,000 over five names at flat 20% weights, so ~$1,000 is the position this account actually +#: takes -- and the per-share TAF makes the answer size-dependent, so a size must be named. +ORDER_USD = Decimal("1000") + + +def load(db: str, product_id: str) -> list[Candle]: + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + try: + rows = con.execute( + "SELECT ts, o, h, l, c, v FROM candles WHERE product_id=? AND granularity='ONE_DAY'" + " ORDER BY ts", + (product_id,), + ).fetchall() + finally: + con.close() + return [ + Candle(ts=r[0], open=Decimal(r[1]), high=Decimal(r[2]), low=Decimal(r[3]), + close=Decimal(r[4]), volume=Decimal(r[5])) + for r in rows + ] + + +def median_daily_quote_volume(candles: list[Candle]) -> Decimal: + """The statistic `slippage_for_quote_volume` reads. These are ONE_DAY bars, so the + per-bar median IS the daily median -- no bars-per-day scaling, and none of the unit trap + that bit the triple-barrier work.""" + if not candles: + return Decimal("0") + return Decimal(str(statistics.median(float(c.volume * c.close) for c in candles))) + + +def measure(db: str, product_id: str, asset_class: str) -> dict[str, object] | None: + candles = load(db, product_id) + if len(candles) < 2: + return None + est = corwin_schultz_spread(candles) + naive = corwin_schultz_spread(candles, block_size=None) + if est is None or naive is None: + return None + adv = median_daily_quote_volume(candles) + modelled_slip = slippage_for_quote_volume(adv) + last_close = candles[-1].close + + # The regulatory term, at the size this account trades. Sells only: a BUY pays nothing. + shares = ORDER_USD / last_close if last_close > 0 else Decimal("0") + reg_total, sec31, taf = ( + estimate_regulatory_fees(Side.SELL, shares, ORDER_USD) + if asset_class == "equities" + else (Decimal("0"), Decimal("0"), Decimal("0")) + ) + reg_bp = (reg_total / ORDER_USD * 10000) if ORDER_USD else Decimal("0") + + # What keel charges this product TODAY, round trip: two legs of commission/fee plus two + # legs of slippage. Equities run at `taker_pct: 0.0`; crypto at the 1.2% taker rate. + modelled_fee = Decimal("0") if asset_class == "equities" else TAKER_FEE_PCT + modelled_rt_bp = (2 * modelled_fee + 2 * modelled_slip) * 10000 + + # What it plausibly costs. THE COMMISSION IS NOT AN ESTIMATE -- 1.2%/leg on Coinbase and + # $0 on Alpaca are published rates actually charged, so they belong in the measured total + # beside the estimated spread. Omitting them would compare an equities total that is + # ~all spread against a crypto total that is ~all fee, and understate crypto 5-fold. + commission_bp = (Decimal("0") if asset_class == "equities" else 2 * TAKER_FEE_PCT) * 10000 + # `est.spread_bp`, NOT `2 * est.half_spread_bp`: the half is quantised to 0.01bp before + # doubling, so the round trip disagreed with the spread column it is derived from by up to + # 0.01bp in either direction (GOOGL published 1.29 and used 1.28; AAPL 2.83 and used 2.84). + # Immaterial to the conclusion, and a table whose columns do not add up invites the reader + # to distrust the ones that matter. + measured_rt_bp = commission_bp + est.spread_bp + reg_bp + + return { + "product_id": product_id, + "asset_class": asset_class, + "bars": len(candles), + "pairs": est.pairs, + "negative_pair_share": str(est.negative_pair_share), + "gap_adjusted_pairs": est.gap_adjusted_pairs, + "spread_bp": str(est.spread_bp), + "naive_spread_bp": str(naive.spread_bp), + "raw_spread_bp": str(est.raw_spread_bp), + "blocks": est.blocks, + "half_spread_bp": str(est.half_spread_bp), + "median_daily_quote_volume": str(adv.quantize(Decimal("1"))), + "modelled_slippage_bp": str((modelled_slip * 10000).quantize(Decimal("0.01"))), + "modelled_roundtrip_bp": str(modelled_rt_bp.quantize(Decimal("0.01"))), + "regulatory_bp": str(reg_bp.quantize(Decimal("0.0001"))), + "sec_section_31_usd": str(sec31.quantize(Decimal("0.000001"))), + "taf_usd": str(taf.quantize(Decimal("0.000001"))), + "commission_bp": str(commission_bp.quantize(Decimal("0.01"))), + "measured_roundtrip_bp": str(measured_rt_bp.quantize(Decimal("0.01"))), + } + + +def main() -> None: + rows: list[dict[str, object]] = [] + for pid in EQUITIES: + row = measure(EQUITIES_DB, pid, "equities") + if row: + rows.append(row) + for pid in CRYPTO: + row = measure(CRYPTO_DB, pid, "crypto") + if row: + rows.append(row) + + with JSONL_PATH.open("w") as fh: + for row in rows: + fh.write(json.dumps(row) + "\n") + + hdr = (f"{'product':<12}{'bars':>6}{'spread bp':>11}{'naive bp':>10}{'raw bp':>10}" + f"{'neg share':>11}{'gaps':>7}{'model rt bp':>13}{'measured rt bp':>16}") + for klass in ("equities", "crypto"): + sub = [r for r in rows if r["asset_class"] == klass] + print(f"\n=== {klass} ({len(sub)}) ===") + print(hdr) + for r in sorted(sub, key=lambda r: Decimal(str(r["spread_bp"]))): + print( + f"{r['product_id']:<12}{r['bars']:>6}{r['spread_bp']:>11}{r['naive_spread_bp']:>10}" + f"{r['raw_spread_bp']:>10}" + f"{r['negative_pair_share']:>11}{r['gap_adjusted_pairs']:>7}" + f"{r['modelled_roundtrip_bp']:>13}{r['measured_roundtrip_bp']:>16}" + ) + meas = sorted(Decimal(str(r["measured_roundtrip_bp"])) for r in sub) + modl = sorted(Decimal(str(r["modelled_roundtrip_bp"])) for r in sub) + print(f" median measured round trip: {statistics.median(meas):>10} bp") + print(f" median modelled round trip: {statistics.median(modl):>10} bp") + + eq = [r for r in rows if r["asset_class"] == "equities"] + cr = [r for r in rows if r["asset_class"] == "crypto"] + eq_med = statistics.median(sorted(Decimal(str(r["measured_roundtrip_bp"])) for r in eq)) + cr_med = statistics.median(sorted(Decimal(str(r["measured_roundtrip_bp"])) for r in cr)) + print(f"\ncrypto / equities measured round-trip ratio: {(cr_med / eq_med):.1f}x") + print(f"equities: modelled {statistics.median(sorted(Decimal(str(r['modelled_roundtrip_bp'])) for r in eq))} bp" + f" vs measured {eq_med} bp") + print(f"floor reference: SLIPPAGE_FLOOR_PCT = {SLIPPAGE_FLOOR_PCT * 10000} bp one way") + for klass, sub in (("equities", eq), ("crypto", cr)): + blocked = statistics.median(sorted(Decimal(str(r["spread_bp"])) for r in sub)) + naive = statistics.median(sorted(Decimal(str(r["naive_spread_bp"])) for r in sub)) + print(f"aggregation bias, {klass}: naive {naive} bp vs blocked {blocked} bp" + f" ({naive / blocked:.1f}x)" if blocked else f"{klass}: blocked is zero") + print(f"\nwrote {JSONL_PATH}") + + +if __name__ == "__main__": + main() diff --git a/keel/commands/research.py b/keel/commands/research.py index 3307f938..ace529e6 100644 --- a/keel/commands/research.py +++ b/keel/commands/research.py @@ -298,6 +298,21 @@ class ResearchModuleEntry: ), runs_as="keel research tuning", ), + ResearchModuleEntry( + module="spread.py", + question=( + "What does a round trip cost on a venue whose commission is zero -- the " + "Corwin-Schultz (2012) proportional spread, recovered from daily highs and lows " + "alone, so ONE estimator prices both asset classes from candles already cached." + ), + cannot_answer=( + "Cannot report the spread anyone actually paid: it is an estimator with real " + "dispersion, not a quote feed, and it is noisy for any single series. It also " + "cannot tell a zero spread from an unmeasurable one on its own -- that is what " + "the unfloored `raw_spread_pct` and the negative-pair share are for." + ), + runs_as="docs/experiments/2026-09-02-equities-cost-fidelity.py", + ), ) #: Names accepted by `--module`, in the same declared order as `RESEARCH_INDEX` -- derived diff --git a/keel/research/spread.py b/keel/research/spread.py new file mode 100644 index 00000000..9f6e0226 --- /dev/null +++ b/keel/research/spread.py @@ -0,0 +1,251 @@ +"""Proportional effective spread, estimated from daily highs and lows -- issue #371. + +⚠️ **This module measures. It does not gate, size, or price anything on the live path.** +Nothing here is imported by `execution/` or `strategy/`; it exists so a cost claim can be +checked rather than assumed. + +WHY AN ESTIMATOR AT ALL. Phase C of the Alpaca PRD (§6.3, O4) needs one number keel has never +had: what a round trip actually costs on a venue whose commission is zero. Two of the three +components were already exact -- Alpaca's commission is $0 and the sell-side regulatory +pass-throughs are published formulas (`keel_broker_alpaca.fees`). The third, the SPREAD, is +the one that dominates, and keel caches OHLCV bars, never quotes. Buying a quote history to +answer it would make the number unreproducible by anyone reading the write-up. + +Corwin & Schultz (2012), "A Simple Way to Estimate Bid-Ask Spreads from Daily High and Low +Prices", Journal of Finance 67(2), 719-759, recovers the spread from highs and lows alone. +The insight: a day's high-low RANGE contains both the true variance of the security and the +spread, but variance scales with the length of the interval while the spread does not. So two +single-day ranges (two units of variance, two spreads) compared against one two-day range (two +units of variance, ONE spread) isolate the spread. + +THE POINT OF USING IT HERE IS THAT IT APPLIES TO BOTH ASSET CLASSES, from data already on +disk, with ONE estimator and no per-venue tuning. A cost comparison between crypto and +equities measured two different ways would be an artifact of the methods. + +THE OVERNIGHT-GAP ADJUSTMENT IS LOAD-BEARING, NOT A REFINEMENT. Section I.B of the paper +subtracts any overnight jump before estimating, because a gap widens the two-day range without +any spread having been paid. Equities gap every night; crypto trades continuously and barely +gaps at all. An unadjusted estimator would therefore inflate the EQUITIES side of exactly the +comparison this module exists to make -- and inflating equity costs is the direction that +flatters keel's existing crypto-heavy conclusion. A measurement must never be trusted in the +direction of its author's prior, so the adjustment is on by default and the count of adjusted +pairs is reported beside every estimate. + +WHAT IT IS NOT. This is an estimator with real dispersion, not a quote feed. Corwin & Schultz +report it tracking TAQ spreads closely in the cross-section while being noisy for any single +security-month; negative two-day estimates are common, since no venue charges a negative spread. + +**HOW THE NEGATIVES ARE HANDLED DECIDES THE ANSWER, BY A FACTOR OF TWENTY.** This is the single +most consequential choice in the module, and the obvious treatment is the wrong one: + +* *Floor every pair, then average the survivors.* Tempting, and wrong. On a series whose true + spread is small relative to its volatility, the two-day estimate is symmetric noise about + zero; keeping the positive half and discarding the negative half makes the mean converge on + E[max(X,0)] > 0. It reports a spread that is not there, and reports a LARGER one the more + volatile the series is. Measured on keel's own equity candles this reads 41.8bp for MSFT -- + roughly twenty times the penny spread that name actually quotes. +* *Average within a month, then floor the monthly mean* -- Corwin & Schultz's own procedure, + and the default here. The negatives cancel against the positives inside the block, so noise + averages out instead of accumulating. The same MSFT series reads 1.9bp, which is the right + order of magnitude for a mega-cap. + +Both are reachable (`block_size=None` selects the biased one) because the comparison between +them is itself a finding, and `raw_spread_pct` reports the mean with no flooring anywhere -- +the only figure that can come out negative, and therefore the only one able to say "this series +carries no measurable spread" out loud. Read `negative_pairs` as the noise diagnostic it is: a +series that floors half its pairs has not been measured, it has been sampled. + +ONE LIMITATION WORTH KNOWING BEFORE READING ANY NUMBER THIS PRODUCES. The gap adjustment +shifts day 2 to sit exactly ON TOP of day 1, which is the most trend-like geometry two ranges +can have, so a pure gap-and-continue day estimates negative and floors to zero. The estimate +for a gapping series is therefore carried by its OVERLAPPING pairs, and `gap_adjusted_pairs` +is the honest sample-size caveat: a series that gaps constantly rests on fewer effective +observations than `pairs` suggests. This does not bias the estimate up -- gap days contribute +zero rather than noise -- but it does widen its uncertainty on the equities side. +""" + +from __future__ import annotations + +import math +from collections.abc import Sequence +from dataclasses import dataclass +from decimal import Decimal + +from keel_core.types import Candle + +#: 3 - 2*sqrt(2), the constant in Corwin & Schultz eq. (14)/(18). Named because it appears +#: three times and a bare `0.1716` in the arithmetic is unreadable. +_K: float = 3.0 - 2.0 * math.sqrt(2.0) + +#: The estimate's resolution. Ten places keeps sub-basis-point detail (1bp = 1e-4) without +#: pretending to precision the estimator does not have. +_QUANTUM: Decimal = Decimal("1e-10") + +#: Basis-point figures are for the write-up, where two decimals is already generous. +_BP_QUANTUM: Decimal = Decimal("0.01") + +#: Pairs per averaging block. 21 trading days is the month Corwin & Schultz aggregate over. +#: It is a bias/variance dial, not a free parameter: smaller blocks floor more often and drift +#: back toward the per-pair bias; larger ones average across regimes that differ. +MONTHLY_BLOCK: int = 21 + + +@dataclass(frozen=True) +class SpreadEstimate: + """A series' estimated proportional spread, with the diagnostics needed to judge it. + + `spread_pct` is the FULL quoted spread as a fraction of price (0.0012 = 12bp). One leg of + a trade crosses half of it, which is what `half_spread_pct` reports and what maps onto + keel's slippage assumption -- `strategy/backtest.slippage_for_quote_volume` is likewise a + one-way number. + """ + + spread_pct: Decimal + raw_spread_pct: Decimal + pairs: int + negative_pairs: int + gap_adjusted_pairs: int + blocks: int + + @property + def half_spread_pct(self) -> Decimal: + """The one-way cost: crossing from the mid to the touch.""" + return self.spread_pct / 2 + + @property + def raw_spread_bp(self) -> Decimal: + """The unfloored mean, in basis points. Negative means "no measurable spread".""" + return (self.raw_spread_pct * 10000).quantize(_BP_QUANTUM) + + @property + def spread_bp(self) -> Decimal: + return (self.spread_pct * 10000).quantize(_BP_QUANTUM) + + @property + def half_spread_bp(self) -> Decimal: + return (self.half_spread_pct * 10000).quantize(_BP_QUANTUM) + + @property + def negative_pair_share(self) -> Decimal: + """How much of the series estimated a negative spread -- the noise diagnostic. + + NOT "how much floored to zero": under the default block aggregation a negative PAIR + does not floor, it averages against the positives inside its block, and only a + negative BLOCK mean floors. (Under `block_size=None` the two coincide, which is + exactly the conflation that makes that variant biased.) Read it as signal-to-noise: + approaching half means the estimator cannot resolve a spread in this series at this + frequency, whichever aggregation is then applied.""" + if self.pairs == 0: + return Decimal("0") + return (Decimal(self.negative_pairs) / Decimal(self.pairs)).quantize(Decimal("0.0001")) + + +def _usable(candle: Candle) -> bool: + """A bar the logarithm can take. A zero-range bar (H == L) is excluded deliberately: it + means halted or untraded, not a zero spread, and it collapses both beta and gamma to zero + which would return a spurious 0.0.""" + return candle.low > 0 and candle.high > candle.low + + +def corwin_schultz_pair( + first: Candle, second: Candle, *, adjust_overnight_gap: bool = True +) -> float | None: + """The two-day estimate, Corwin & Schultz eq. (14)-(18). `None` when the pair is unusable. + + Returned as a raw float and NOT floored at zero: the flooring is the aggregator's + decision, and how often a pair goes negative is a diagnostic the caller needs. Float + rather than Decimal because the formula is logarithms and exponentials throughout; the + conversion to Decimal happens once, at the aggregate. + """ + if not _usable(first) or not _usable(second): + return None + + high_1, low_1 = float(first.high), float(first.low) + high_2, low_2 = float(second.high), float(second.low) + + if adjust_overnight_gap: + # Section I.B: a gap moves day 2's whole range without any spread being paid. Shift + # day 2 back onto day 1's range -- the shift cancels inside every log ratio of day 2's + # own bar, so it changes ONLY the two-day range, which is precisely the term the gap + # contaminated. + if low_2 > high_1: + gap = low_2 - high_1 + high_2, low_2 = high_2 - gap, low_2 - gap + elif high_2 < low_1: + gap = low_1 - high_2 + high_2, low_2 = high_2 + gap, low_2 + gap + + beta = math.log(high_1 / low_1) ** 2 + math.log(high_2 / low_2) ** 2 + gamma = math.log(max(high_1, high_2) / min(low_1, low_2)) ** 2 + alpha = (math.sqrt(2.0 * beta) - math.sqrt(beta)) / _K - math.sqrt(gamma / _K) + return 2.0 * (math.exp(alpha) - 1.0) / (1.0 + math.exp(alpha)) + + +def corwin_schultz_spread( + candles: Sequence[Candle], + *, + adjust_overnight_gap: bool = True, + block_size: int | None = MONTHLY_BLOCK, +) -> SpreadEstimate | None: + """Estimate a series' proportional spread from consecutive high-low pairs. + + `block_size` chooses the aggregation, and the choice is worth twenty-fold on a quiet + series -- see the module docstring. The default averages within blocks of that many pairs + and floors each BLOCK mean; `None` floors each PAIR instead, which is the biased variant, + retained so the bias stays demonstrable rather than folklore. + + `None` (the return, not the argument) means NOTHING WAS MEASURABLE -- fewer than two bars, + or no usable pair. That is a different statement from an estimate of zero, and a caller + must not read the two the same way: one says the venue is cheap, the other says the + question was not answered. + + **PRECONDITION, UNCHECKED: `candles` must be CONSECUTIVE bars of one series.** `ts` is + never read. Every adjacent pair is treated as a two-day pair, so a series with a hole in + it -- a fetch gap, two series concatenated, a filtered subset -- prices that hole as an + overnight move and returns a number with no warning attached. Blocks are likewise formed + over usable PAIRS, not over calendar time, so a gappy series' "month" can span far more + than a month. Checking adjacency here would need a granularity this function is not given; + the caller has it, and owns this. + """ + if block_size is not None and block_size < 1: + # Caught by name rather than left to the slice arithmetic: 0 reached `range(step=0)` + # and negatives produced an empty block list to divide by. `None` is the one + # non-positive-integer value with a meaning, and it keeps it. + raise ValueError(f"block_size must be a positive integer or None; got {block_size!r}") + + estimates: list[float] = [] + negative = 0 + gapped = 0 + for first, second in zip(candles, candles[1:], strict=False): + estimate = corwin_schultz_pair(first, second, adjust_overnight_gap=adjust_overnight_gap) + if estimate is None: + continue + estimates.append(estimate) + if adjust_overnight_gap and (second.low > first.high or second.high < first.low): + gapped += 1 + if estimate < 0: + negative += 1 + if not estimates: + return None + + raw = sum(estimates) / len(estimates) + if block_size is None: + floored = sum(e for e in estimates if e > 0) / len(estimates) + blocks = 0 + else: + chunks = [estimates[i : i + block_size] for i in range(0, len(estimates), block_size)] + means = [sum(chunk) / len(chunk) for chunk in chunks] + floored = sum(max(m, 0.0) for m in means) / len(means) + blocks = len(means) + + return SpreadEstimate( + spread_pct=Decimal(repr(floored)).quantize(_QUANTUM), + raw_spread_pct=Decimal(repr(raw)).quantize(_QUANTUM), + pairs=len(estimates), + negative_pairs=negative, + gap_adjusted_pairs=gapped, + blocks=blocks, + ) + + +__all__ = ["MONTHLY_BLOCK", "SpreadEstimate", "corwin_schultz_pair", "corwin_schultz_spread"] diff --git a/tests/research/test_spread.py b/tests/research/test_spread.py new file mode 100644 index 00000000..0b954926 --- /dev/null +++ b/tests/research/test_spread.py @@ -0,0 +1,333 @@ +"""The Corwin-Schultz high-low spread estimator (issue #371). + +Phase C of the Alpaca PRD needs one number keel has never had: **what a round trip actually +costs on a venue whose commission is zero.** The regulatory pass-throughs are exact and already +modelled (`keel_broker_alpaca.fees`); the spread is not, and keel caches OHLCV bars, never +quotes. Corwin-Schultz recovers a proportional effective spread from daily highs and lows +alone, which makes it the one estimator that can be applied to BOTH asset classes from data +already on disk -- the like-for-like comparison the cost-distortion claim rests on. + +The load-bearing test is `test_the_overnight_gap_adjustment_is_applied`: equities gap overnight +and crypto (24/7) essentially does not, so an unadjusted estimator would inflate the equity side +of exactly the comparison this exists to make. The bias would run in the direction that +FLATTERS keel's crypto-heavy prior, which is the direction a measurement must never be trusted +in. +""" + +from __future__ import annotations + +import math +import random +from decimal import Decimal + +import pytest +from keel_core.types import Candle + +from keel.research.spread import ( + MONTHLY_BLOCK, + SpreadEstimate, + corwin_schultz_pair, + corwin_schultz_spread, +) + + +def _c(high: str, low: str, ts: int = 0, close: str | None = None) -> Candle: + """A candle carrying only what the estimator reads: its high and its low.""" + hi, lo = Decimal(high), Decimal(low) + mid = close if close is not None else str((hi + lo) / 2) + return Candle(ts=ts, open=mid, high=hi, low=lo, close=Decimal(mid), volume=Decimal("1000")) + + +# --- the pair estimator: known answers, computed from the published formula ------------------ + + +def test_a_two_day_pair_reproduces_the_published_formula() -> None: + """Corwin & Schultz (2012) eq. (14)-(18), evaluated by hand on a wide-range pair.""" + got = corwin_schultz_pair(_c("101", "99"), _c("101.5", "99.5")) + assert got is not None + assert math.isclose(float(got), 0.0079088925, rel_tol=1e-6) + + +def test_a_tighter_range_estimates_a_tighter_spread() -> None: + wide = corwin_schultz_pair(_c("101", "99"), _c("101.5", "99.5")) + tight = corwin_schultz_pair(_c("100.2", "99.8"), _c("100.3", "99.9")) + assert wide is not None and tight is not None + assert math.isclose(float(tight), 0.0015849918, rel_tol=1e-6) + assert tight < wide + + +def test_a_trending_pair_estimates_a_negative_spread() -> None: + """A strong two-day trend makes the two-day range much wider than the daily ranges, which + drives alpha negative. The estimator does NOT hide this at the pair level -- the flooring + is the aggregator's decision, and counting how often it happens is a diagnostic.""" + got = corwin_schultz_pair(_c("105", "100"), _c("110", "104")) + assert got is not None + assert got < 0 + assert math.isclose(float(got), -0.0506145009, rel_tol=1e-6) + + +def test_a_zero_range_bar_is_not_an_estimate() -> None: + """H == L means no intraday range at all -- a halted or untraded bar, not a zero spread. + log(1) = 0 collapses beta AND gamma, and the formula would return a spurious 0.0.""" + assert corwin_schultz_pair(_c("100", "100"), _c("101", "99")) is None + assert corwin_schultz_pair(_c("101", "99"), _c("100", "100")) is None + + +def test_a_nonpositive_price_is_not_an_estimate() -> None: + """Guards the logarithm rather than raising: one bad cached bar must not end a sweep.""" + assert corwin_schultz_pair(_c("0", "0"), _c("101", "99")) is None + assert corwin_schultz_pair(_c("101", "99"), _c("-1", "-2")) is None + + +def test_a_low_above_its_own_high_is_not_an_estimate() -> None: + """An inverted bar is corrupt data, never a tradeable range.""" + assert corwin_schultz_pair(_c("99", "101"), _c("101", "99")) is None + + +# --- the overnight gap adjustment ------------------------------------------------------------ + + +def test_the_overnight_gap_adjustment_is_applied() -> None: + """Corwin & Schultz section I.B: when day 2's LOW sits above day 1's HIGH, the two-day + range contains an overnight jump that no spread produced. The gap is subtracted from day + 2 before the estimate, or the jump is priced as if it were a spread. + + Day 1 [99, 101], day 2 [110, 112]: a 9-point gap up on a 2-point daily range. Unadjusted, + the two-day range spans 13 points and the estimate is deeply negative; adjusted, day 2 + becomes [101, 103] and the overnight jump is gone from the two-day range.""" + d1, d2 = _c("101", "99"), _c("112", "110") + adjusted = corwin_schultz_pair(d1, d2) + unadjusted = corwin_schultz_pair(d1, d2, adjust_overnight_gap=False) + assert adjusted is not None and unadjusted is not None + # The gap contaminated the estimate downward by an order of magnitude; removing it is + # most of the correction. + assert unadjusted < adjusted + assert float(unadjusted) < 8 * float(adjusted) + # Shifting day 2 down by the gap must give EXACTLY the same answer as adjusting it -- this + # is the assertion that pins the arithmetic, not the sign. + shifted = corwin_schultz_pair(d1, _c("103", "101"), adjust_overnight_gap=False) + assert shifted is not None + assert math.isclose(float(adjusted), float(shifted), rel_tol=1e-12) + + +def test_a_pure_gap_still_floors_to_zero_after_adjustment() -> None: + """An honest limitation, pinned so nobody later reads a gap-heavy series as a cheap one. + + The adjustment shifts day 2 to sit EXACTLY on top of day 1, which is the most + trend-like geometry two ranges can have, so a pure gap-and-continue day estimates negative + and floors to zero. Corwin & Schultz's estimate on a gapping series is therefore carried by + its OVERLAPPING pairs; the gap days contribute nothing rather than contributing noise. That + is why `gap_adjusted_pairs` is reported -- a series that gaps constantly is a series whose + estimate rests on a smaller sample than `pairs` suggests.""" + adjusted = corwin_schultz_pair(_c("101", "99"), _c("112", "110")) + assert adjusted is not None and adjusted < 0 + + +def test_the_gap_adjustment_works_downward_too() -> None: + """A gap DOWN (day 2's high below day 1's low) is the same distortion, mirrored.""" + d1, d2 = _c("101", "99"), _c("90", "88") + adjusted = corwin_schultz_pair(d1, d2) + # The 9-point gap (99 - 90) is added back to day 2, lifting [88, 90] to [97, 99]. + shifted = corwin_schultz_pair(d1, _c("99", "97"), adjust_overnight_gap=False) + assert adjusted is not None and shifted is not None + assert math.isclose(float(adjusted), float(shifted), rel_tol=1e-12) + + +def test_overlapping_days_are_not_gap_adjusted() -> None: + """Ranges that overlap have no overnight jump to remove; touching the estimate there would + be the estimator inventing an adjustment the paper does not make.""" + d1, d2 = _c("101", "99"), _c("101.5", "99.5") + assert corwin_schultz_pair(d1, d2) == corwin_schultz_pair(d1, d2, adjust_overnight_gap=False) + + +# --- the aggregate: WHICH average, and why it is not the obvious one --------------------- + + +def test_the_default_averages_within_blocks_before_flooring() -> None: + """Corwin & Schultz's own procedure: average the two-day estimates within a month, THEN + set a negative MONTHLY average to zero. Not: floor every pair and average the survivors. + + The difference is not cosmetic -- see `test_flooring_each_pair_biases_a_quiet_series_up`. + Here, two pairs in one block whose mean is negative give zero, where per-pair flooring + would have kept the positive one and reported half of it.""" + candles = [_c("101", "99", ts=0), _c("101.5", "99.5", ts=1), _c("110", "104", ts=2)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.pairs == 2 + assert got.blocks == 1 + assert got.spread_pct == Decimal("0") + + +def test_flooring_each_pair_biases_a_quiet_series_up() -> None: + """THE LOAD-BEARING TEST. On a series with NO spread at all -- a pure random walk whose + highs and lows come only from volatility -- the two-day estimate is symmetric noise around + zero. Flooring each pair keeps the positive half and discards the negative half, so its + mean converges on E[max(X,0)] > 0: it reports a spread that does not exist, and reports a + BIGGER one the more volatile the series is. + + That is not a hypothetical. On real mega-cap equities the two aggregations differ by ~20x, + and only the block figure lands anywhere near the penny spreads those names actually quote. + Getting this wrong would have priced a commission-free venue at 20x its true cost.""" + rng = random.Random(20260902) + price, candles = 100.0, [] + for i in range(2000): + price *= math.exp(rng.gauss(0.0, 0.02)) + # High/low from intraday volatility ONLY -- no bid-ask bounce, no spread. + hi = price * math.exp(abs(rng.gauss(0.0, 0.01))) + lo = price * math.exp(-abs(rng.gauss(0.0, 0.01))) + candles.append(_c(f"{hi:.6f}", f"{lo:.6f}", ts=i)) + + naive = corwin_schultz_spread(candles, block_size=None) + block = corwin_schultz_spread(candles) + assert naive is not None and block is not None + assert naive.spread_bp > 5 * block.spread_bp + # And the block estimate is near zero, which is the truth for this series. + assert block.spread_bp < Decimal("10") + + +def test_the_naive_per_pair_flooring_is_still_reachable() -> None: + """Kept, not deleted: the biased variant is what makes the bias demonstrable, and the + write-up quotes both figures. `block_size=None` selects it explicitly.""" + candles = [_c("101", "99", ts=0), _c("101.5", "99.5", ts=1), _c("110", "104", ts=2)] + got = corwin_schultz_spread(candles, block_size=None) + assert got is not None + assert got.pairs == 2 + assert got.negative_pairs == 1 + assert got.blocks == 0 # no blocking was done + assert math.isclose(float(got.spread_pct), 0.0079088925 / 2, rel_tol=1e-6) + + +def _quiet_then_trending() -> list[Candle]: + """21 pairs of tight overlapping ranges (a positive block), then 21 pairs of a 5%-a-bar + ramp whose ranges never overlap (a negative block, floored).""" + candles = [_c("101", "99", ts=i) for i in range(22)] + centre = 100.0 + for i in range(22, 43): + centre *= 1.05 + candles.append(_c(f"{centre * 1.001:.6f}", f"{centre / 1.001:.6f}", ts=i)) + return candles + + +def test_a_floored_block_still_counts_toward_the_average() -> None: + """A negative block contributes ZERO to the mean, not nothing at all -- the denominator is + every block, not just the surviving ones. + + Dropping floored blocks from the denominator would reintroduce the selection bias the + blocking exists to remove, one level up: a series whose spread is unmeasurable in half its + months would report the other half's figure as if it held throughout. Here that is worth + exactly 2x, because one of the two blocks floors.""" + mixed = corwin_schultz_spread(_quiet_then_trending()) + quiet_only = corwin_schultz_spread([_c("101", "99", ts=i) for i in range(22)]) + assert mixed is not None and quiet_only is not None + assert mixed.blocks == 2 and quiet_only.blocks == 1 + # The trending block floored, so the mixed series reports HALF the quiet block's figure. + assert math.isclose( + float(mixed.spread_pct), float(quiet_only.spread_pct) / 2, rel_tol=1e-6 + ) + + +def test_a_block_size_that_cannot_form_a_block_is_refused() -> None: + """Zero and negative block sizes are rejected by name rather than by whatever the slice + arithmetic happens to raise -- `block_size=0` reached `range(step=0)` and `-1` divided by + an empty block list. `None` is the one non-positive-integer value with a meaning, and it + keeps it.""" + candles = [_c("101", "99", ts=i) for i in range(30)] + for bad in (0, -1, -21): + with pytest.raises(ValueError, match="block_size"): + corwin_schultz_spread(candles, block_size=bad) + assert corwin_schultz_spread(candles, block_size=None) is not None + + +def test_the_block_size_defaults_to_a_trading_month() -> None: + assert MONTHLY_BLOCK == 21 + + +def test_blocks_are_counted_and_a_short_tail_is_still_a_block() -> None: + """43 pairs at 21 to a block is two full blocks and a 1-pair remainder. Dropping the + remainder would silently discard the most recent data, which is the part a cost estimate + most needs.""" + candles = [_c("101", "99", ts=i) for i in range(45)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.pairs == 44 + assert got.blocks == 3 + + +def test_the_raw_unfloored_mean_is_reported_as_a_diagnostic() -> None: + """The only figure with no flooring anywhere. It can be negative, and when it is, that is + the honest statement that the series carries no measurable spread -- a fact both floored + variants are structurally incapable of expressing.""" + candles = [_c("105", "100", ts=0), _c("110", "104", ts=1)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.raw_spread_pct < 0 + assert got.spread_pct == Decimal("0") + + +def test_the_half_spread_is_the_one_way_cost() -> None: + """keel prices ONE leg at a time, so the number that maps onto slippage is half the + quoted spread -- crossing from the mid to the touch.""" + got = corwin_schultz_spread([_c("101", "99", ts=i) for i in range(30)]) + assert got is not None + assert got.half_spread_pct == got.spread_pct / 2 + assert got.spread_pct > 0 + + +def test_a_series_shorter_than_two_bars_has_no_estimate() -> None: + assert corwin_schultz_spread([]) is None + assert corwin_schultz_spread([_c("101", "99")]) is None + + +def test_a_series_whose_every_pair_is_unusable_has_no_estimate() -> None: + """Distinct from "the spread is zero": nothing was measurable, and the caller must be able + to tell those apart rather than reading 0bp as a free venue.""" + assert corwin_schultz_spread([_c("100", "100", ts=0), _c("100", "100", ts=1)]) is None + + +def test_the_gap_adjusted_pair_count_is_reported() -> None: + """The equities-vs-crypto comparison turns on this number: if the equity series adjusts + many pairs and the crypto series adjusts almost none, that asymmetry is the finding.""" + candles = [_c("101", "99", ts=0), _c("112", "110", ts=1), _c("112.5", "110.5", ts=2)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.gap_adjusted_pairs == 1 + + +def test_the_estimate_is_a_decimal_so_it_composes_with_keels_money_types() -> None: + got = corwin_schultz_spread([_c("101", "99", ts=i) for i in range(30)]) + assert got is not None + assert isinstance(got.spread_pct, Decimal) + assert isinstance(got.half_spread_pct, Decimal) + assert isinstance(got.raw_spread_pct, Decimal) + assert got.spread_pct * Decimal("1000") > 0 + + +def test_the_estimate_is_never_negative_in_aggregate() -> None: + """Every pair negative means the block mean is negative and floors to zero -- reported as + zero WITH its pair count, not as None, because the pairs were measurable.""" + got = corwin_schultz_spread([_c("105", "100", ts=0), _c("110", "104", ts=1)]) + assert got is not None + assert got.spread_pct == Decimal("0") + assert got.negative_pairs == 1 + + +@pytest.mark.parametrize("bars", [2, 3, 10, 50]) +def test_pair_count_is_one_less_than_the_usable_bar_count(bars: int) -> None: + candles = [_c("101", "99", ts=i) for i in range(bars)] + got = corwin_schultz_spread(candles) + assert got is not None + assert got.pairs == bars - 1 + + +def test_the_estimate_is_reported_as_basis_points_for_the_write_up() -> None: + est = SpreadEstimate( + spread_pct=Decimal("0.0012"), + raw_spread_pct=Decimal("0.0010"), + pairs=100, + negative_pairs=10, + gap_adjusted_pairs=0, + blocks=5, + ) + assert est.spread_bp == Decimal("12.00") + assert est.half_spread_bp == Decimal("6.00") + assert est.raw_spread_bp == Decimal("10.00")