diff --git a/.github/workflows/ecosystem.yml b/.github/workflows/ecosystem.yml index fe84b47..4ae81f2 100644 --- a/.github/workflows/ecosystem.yml +++ b/.github/workflows/ecosystem.yml @@ -17,7 +17,7 @@ jobs: qualification: permissions: contents: read - uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@a3080f8e09b197098c9d796240e746aca516492e # 2026-09-23 policy snapshot + uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@066dc737a879faea51a5b8a900315441712ea89a # 2026-09-25 policy snapshot with: import-package: ml4t.data prerelease-exception: python-315-polars diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2bca84..e3a45bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: needs: preflight permissions: contents: read - uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@a3080f8e09b197098c9d796240e746aca516492e # 2026-09-23 policy snapshot + uses: ml4t/ecosystem/.github/workflows/qualify-library.yml@066dc737a879faea51a5b8a900315441712ea89a # 2026-09-25 policy snapshot with: import-package: ml4t.data prerelease-exception: python-315-polars diff --git a/docs/getting-started/provider-selection.md b/docs/getting-started/provider-selection.md index 6c1b4f4..40f8c31 100644 --- a/docs/getting-started/provider-selection.md +++ b/docs/getting-started/provider-selection.md @@ -24,9 +24,9 @@ data = SyntheticProvider(seed=42).fetch_ohlcv( | Equity bars | Yahoo Finance, Alpaca, EODHD, Tiingo, Twelve Data, Massive, Finnhub | Some adapters require credentials, accounts, or paid history | | Foreign exchange | OANDA, Twelve Data, FXMacroData | OANDA and Twelve Data require credentials | | Futures and options | Databento, Binance, OKX | Historical or licensed data may be metered | -| Macroeconomic series | FRED, FXMacroData | FRED requires an API key for normal use | -| Research factors | Fama-French, AQR | Network access and source-specific terms | -| Prediction markets | Kalshi, Polymarket | Network access and changing public endpoints | +| [Macroeconomic series](../providers/macro.md) | FRED, FXMacroData | FRED requires an API key for normal use | +| [Research factors](../providers/factors.md) | Fama-French, AQR | Network access and source-specific terms | +| [Prediction markets](../providers/prediction_markets.md) | Kalshi, Polymarket | Network access and changing public endpoints | | Frozen equity history | Wiki Prices | Local historical dataset ending in 2018 | | CFTC positioning | COT | Install the `cot` extra | diff --git a/docs/providers/alpaca.md b/docs/providers/alpaca.md index a578230..006fda2 100644 --- a/docs/providers/alpaca.md +++ b/docs/providers/alpaca.md @@ -146,5 +146,6 @@ headers, and retries transient failures per pagination page. ## See Also +- [Equity](equities.md) and [ETF](etfs.md) source references - [Alpaca Market Data docs](https://docs.alpaca.markets/us/docs/about-market-data-api) - [Provider README](index.md) diff --git a/docs/providers/alternative_data.md b/docs/providers/alternative_data.md index e1ea582..559c83d 100644 --- a/docs/providers/alternative_data.md +++ b/docs/providers/alternative_data.md @@ -1,210 +1,45 @@ # Alternative Data Sources -This page maps sources of alternative data: news and text, social and web attention, app and -web usage, job postings, card transactions, foot traffic, satellite and shipping data, ESG -ratings, supply-chain relationships, and prediction markets, plus the marketplaces where such -datasets are found. For each source it lists what the data measures, how far back it goes, what -is known about its point-in-time behavior, the access tier, and whether `ml4t-data` wraps it. +Alternative data covers observations outside standard prices, statements, and macro series. The +main selection questions are when a record first became observable, whether history was backfilled +or revised, how representative the panel is, and whether the proposed use is permitted. !!! note "Verified September 2026" - Alternative data vendors are acquired, merged and repackaged often. Every entry below was - checked in September 2026 against the vendor's own site, a press release, or published reporting. Few - vendors publish their timestamp and revision policy; where it is not public, the table says - "not documented" and the question belongs in your due diligence. Confirm the current terms - before you build on any of them. + Entries were checked against the linked official product documentation. Vendor ownership, + access, and methodology change frequently. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [GDELT](https://www.gdeltproject.org/data.html) | Global news events from 1979 and full-text-derived themes and tone from 2013 | Public files, feeds, and BigQuery | Free | Older history was assembled retrospectively; event and batch timestamps have different meanings | No | +| [SEC EDGAR full-text search](https://www.sec.gov/edgar/search/efts-faq.html) | Searchable US filing text from 2001 | Public search API and filing archives | Free; fair-access policy applies | Use SEC acceptance timestamps, not reporting periods or local download times | No | +| [RavenPack News Analytics](https://www.ravenpack.com/products/news-analytics) | Entity-tagged financial news events, relevance, novelty, and sentiment | API, feeds, and managed files | Institutional; some academic access through WRDS | Vendor taxonomies and model versions can change; confirm whether historical scores are restated | No | +| [LSEG MarketPsych Analytics](https://www.lseg.com/en/data-analytics/market-data/quantitative-economic-data-solutions/marketpsych-analytics-and-models) | News and social-media sentiment across assets and macro topics | Feeds and institutional delivery | Institutional license | Contributor coverage, language models, and revision policy are product-specific | No | +| [Reddit Data API](https://support.reddithelp.com/hc/en-us/articles/16160319875092-Reddit-Data-API-Wiki) | Posts and comments subject to approved use | OAuth API and licensed products | Approval and commercial terms depend on use | Deleted content, community selection, bots, and policy changes create unstable historical samples | No | +| [X API](https://docs.x.com/x-api/getting-started/pricing) | Posts, users, and engagement subject to product access | REST and streaming APIs | Pay-per-use and enterprise access | Sampling, deletions, bot activity, and changing access tiers affect longitudinal research | No | +| [Similarweb Data and APIs](https://www.similarweb.com/corp/ourdata/) | Modeled website and app traffic and engagement | API and managed exports | Commercial license | Panel composition and estimation models are not direct observations of all traffic | No | +| [Sensor Tower App Performance](https://sensortower.com/product/mobile-app/app-performance-insights) | Estimated app downloads, revenue, rankings, and usage | API and exports | Commercial license | Modeled estimates and store coverage require product-specific validation | No | +| [Revelio Labs workforce data](https://www.reveliolabs.com/data) | Standardized job postings, employee profiles, and workforce measures | API, cloud, and managed files | Commercial license | Source-platform coverage, deduplication, and profile updates can revise history | No | +| [Consumer Edge transaction data](https://consumer-edge.com/data/) | Aggregated consumer transaction panels and company metrics | Feeds and managed files | Institutional license | Panel representativeness, merchant mapping, and privacy controls are central to validity | No | +| [Foursquare Movement](https://foursquare.com/products/movement/) | Aggregated foot-traffic and visitation measures | API and managed delivery | Commercial license | Device-panel composition, venue mapping, and privacy thresholds can change coverage | No | +| [Planet Monitoring](https://www.planet.com/products/monitoring/) | Repeated satellite imagery over land areas | API and cloud delivery | Commercial license; research programs may apply | Clouds, revisit gaps, sensor changes, and labeling choices can dominate a derived signal | No | +| [MarineTraffic API](https://www.marinetraffic.com/en/ais-api-services) | AIS vessel positions, port calls, and voyage information | REST and streaming APIs | Commercial license | Terrestrial and satellite AIS have different coverage; vessel identifiers and reported fields can be missing or false | No | +| [OpenWeather API](https://openweathermap.org/api) | Current, forecast, and plan-dependent historical weather | REST API and bulk products | API key; plan-dependent | Station and model changes, spatial interpolation, and forecast vintage determine valid use | No | +| [USGS EarthExplorer](https://earthexplorer.usgs.gov/) | Public remote-sensing and geospatial archives, including Landsat products | Search API, bulk download, and cloud mirrors | Free account for some downloads | Sensor generations, processing levels, and cloud masks must be made consistent | No | + +Prediction markets are documented separately because contract rules, order books, and settlement +make them a market-data category rather than a general alternative-data feed. See +[Prediction-Market Data Sources](prediction_markets.md). + +## Selection Notes + +- Record first-availability time separately from event time and observation period. +- Ask whether backfills, panel changes, or model updates rewrite history. +- Validate coverage by company, geography, sector, and time before testing a signal. +- Resolve provenance, privacy, material non-public information, and redistribution rights before + acquisition. + +## Related References ---- - -## Choosing a Source - -### When was each record first observable? - -A backtest is only valid if every record is used after the time it could first have been seen. -Ask the vendor for the timestamp that marks first availability, not the event date, and whether -history was backfilled when the dataset launched or when coverage expanded. A backfilled history -looks continuous but was assembled later, often by a method that did not exist at the time. - -### Revisions and methodology changes - -Panels change composition (card and device panels gain and lose users), models are re-estimated, -and rating methodologies are revised. ESG ratings are the clearest case: providers disagree with -each other and revise their own methods over time. Ask whether old values are restated when the -method changes and whether earlier vintages are kept. - -### Coverage and representativeness - -Check which names, sectors and regions a dataset actually covers, and whether coverage is stable -across the sample. A signal that exists only for large US consumer companies is a narrower -signal than its marketing suggests. - -### Legal and privacy - -Provenance, permitted use and redistribution rights are hard constraints. Narrow panels can carry -material non-public information, and granular location or device data carries privacy risk even -without names. These questions are answered before any backtest, not after. - -### In the book - -- Chapter 4, "Understanding alternative data" (section 4.4): "The alternative data landscape", - "The evaluation framework", "Build versus buy for alternative data", and "Data sources and - implementation". -- Chapter 4, "Using text data for NLP features" (section 4.5): extracting and storing text from - filings point-in-time. Chapter 10 covers turning text into features. -- Chapter 2, "A modern taxonomy of financial data" (section 2.1), subsection "Alternative data - - High variety, high validation burden". - ---- - -## News, Text and Sentiment - -| Source | What it measures | History | Point-in-time and revisions | Access | ml4t-data | -|--------|------------------|---------|-----------------------------|--------|-----------| -| [GDELT Project](https://gdeltproject.org/data.html) | Global news events and tone (Event database); themes, entities and sentiment from full text (Global Knowledge Graph) | Events from 1979; Knowledge Graph from April 2013 | Published in 15-minute batches; the batch time is a usable first-availability time. Older history was built retrospectively. | Free and open | No | -| [RavenPack](https://www.ravenpack.com/) | Entity-level news analytics: relevance, sentiment and event classification | From 2000 (Dow Jones edition), 2007 (web edition) | Records carry the news timestamp; revision policy not documented publicly | Institutional; academic via WRDS | No | -| [LSEG MarketPsych Analytics](https://www.lseg.com/en/data-analytics/market-data/quantitative-economic-data-solutions/marketpsych-analytics-and-models) | Sentiment and emotion scores from news and social media across equities, macro, FX, crypto and commodities | From 1998 | Minute-level to daily series; revision policy not documented publicly | Institutional | No | -| [Dow Jones Factiva DNA](https://www.dowjones.com/business-intelligence/factiva/) | Licensed premium news archive with metadata, via API, snapshots and streams | Archive depth varies by publication | Publication timestamps per article | Institutional | No | -| [Benzinga Newsfeed API](https://docs.benzinga.com/api-reference/news-api/overview) | Financial news headlines and articles, Wilshire 5000 and TSX coverage | Not documented | `created` and `updated` timestamps per article; `updatedSince` returns changes | Paid | No | -| [SEC EDGAR full-text search](https://www.sec.gov/edgar/search/efts-faq.html) | Full text of SEC filings | From 2001 | Every filing carries its acceptance timestamp | Free | No | - ---- - -## Social Media and Web Attention - -Access to social-media data has narrowed since 2023, so check the current terms before planning -a project around any of these. - -| Source | What it measures | Status | Access | ml4t-data | -|--------|------------------|--------|--------|-----------| -| [Stocktwits API](https://api.stocktwits.com/developers) | Investor message stream with cashtags and self-reported sentiment | Closed to new developer registrations | Not open to new developers | No | -| [Reddit Data API](https://support.reddithelp.com/hc/en-us/articles/16160319875092-Reddit-Data-API-Wiki) | Posts and comments, including investing subreddits | Paid for commercial use since July 2023 | Commercial license required | No | -| [X API](https://docs.x.com/x-api/getting-started/pricing) | Posts and engagement | Pay-per-use credits since February 2026, replacing subscription tiers | Paid | No | - ---- - -## Web, App and Workforce Data - -| Source | What it measures | History | Point-in-time and revisions | Access | ml4t-data | -|--------|------------------|---------|-----------------------------|--------|-----------| -| [Similarweb](https://www.similarweb.com/corp/ourdata/) | Website and app traffic and engagement estimates | About 10 years | Modeled estimates; revision policy not documented publicly | Free tools, paid, enterprise | No | -| [Sensor Tower](https://sensortower.com/product/mobile-app/app-performance-insights) | App downloads, revenue, rankings and usage estimates; includes the former data.ai | Not documented | Modeled estimates; revision policy not documented publicly | Enterprise; free top charts | No | -| [Thinknum](https://www.thinknum.com/) | Web-collected company data: job listings, headcount, product prices, store locations | Not documented | Not documented | Institutional | No | -| [YipitData](https://www.yipitdata.com/) | Company KPI and revenue estimates from web, app and transaction data | Not documented | Not documented | Institutional | No | -| [Revelio Labs](https://www.reveliolabs.com/data) | Workforce composition and flows, job postings, employee sentiment, layoff notices | Workforce composition from 2007, transitions from 2008, postings from 2021 | Not documented | Paid; academic via WRDS | No | -| [LinkUp](https://www.linkup.com/data) | Job postings collected directly from employer websites | From 2007 | New, updated and removed listings are captured nightly | Institutional | No | - ---- - -## Card Transactions - -Card panels measure the spending of a sample of consumers. Panel composition changes over time, -so ask how the vendor weights the panel and whether history is reweighted when it changes. - -| Source | What it measures | History | Access | ml4t-data | -|--------|------------------|---------|--------|-----------| -| [Bloomberg Second Measure](https://secondmeasure.com/) | US consumer card spending by merchant and company | 8+ years; 2 to 7 day reporting lag | Bloomberg Terminal and enterprise | No | -| [Consumer Edge Transact](https://www.consumeredge.com/products/transact/) | Card transactions by merchant, company and demographic; now includes Earnest Analytics | About 9 years | Institutional | No | -| [Facteus](https://facteus.com/) | Debit and credit card transactions, synthesized for privacy | Not documented; about 1-day lag | Institutional feeds (API, AWS, Snowflake) | No | - ---- - -## Foot Traffic, Satellite and Shipping - -| Source | What it measures | History | Access | ml4t-data | -|--------|------------------|---------|--------|-----------| -| [Placer.ai](https://www.placer.ai/products/api) | Visits to stores and venues from a mobile-device panel | Not documented | Free tools, paid platform, enterprise API | No | -| [Advan Research Patterns+](https://advanresearch.com/products/patternsplus) | Foot traffic to points of interest, US and Canada; took over SafeGraph's Patterns product | Weekly updates; history from 2019 | Institutional | No | -| [Planet Labs](https://www.planet.com/industries/education-and-research/) | Satellite imagery: near-daily medium resolution and tasked high resolution | Archive from 2009 (RapidEye, 2009 to 2020) | Commercial; education and research program for university users | No | -| [RS Metrics](https://rsmetrics.com/) | Asset-level signals from satellite imagery: metals production at smelters, retail parking lots | Not documented | Institutional; data for research | No | -| [Kpler](https://www.kpler.com/product/commodities) | Commodity flows from ship tracking (AIS), satellite, customs and port data; now owns MarineTraffic and Spire Maritime | Not documented | Institutional | No | - ---- - -## ESG - -ESG ratings disagree across providers and change when methodologies change. Treat a rating -history as a sequence of vintages, and check which methodology produced each value. - -| Source | What it measures | History and methodology | Access | ml4t-data | -|--------|------------------|-------------------------|--------|-----------| -| [MSCI ESG Ratings](https://www.msci.com/data-and-analytics/sustainability-solutions/esg-ratings) | Industry-relative ESG ratings (AAA to CCC), about 17,000 issuers | Time series since 2007 | Institutional; free public lookup for a subset | No | -| [Morningstar Sustainalytics ESG Risk Ratings](https://www.sustainalytics.com/esg-data) | Unmanaged ESG risk score, 16,000+ companies | A score change log exists; vintage policy not documented publicly | Institutional | No | -| [LSEG ESG Scores](https://www.lseg.com/en/media-centre/press-releases/2026/lseg-launches-new-suite-esg-scores-sustainability-analytics) | ESG scores from 220+ indicators, plus controversies | New suite launched March 2026 on 220 standardised indicators; how it relates to earlier LSEG scores is not documented publicly | Institutional | No | -| [S&P Global ESG Scores](https://www.marketplace.spglobal.com/en/datasets/s-p-global-esg-scores-(171)) | Scores from the annual Corporate Sustainability Assessment | Annual assessments; history not documented publicly | Institutional | No | -| [RepRisk](https://www.reprisk.com/insights/resources/methodology) | Reputational and conduct risk from news and stakeholder sources | Daily series with a consistent methodology since January 2007 | Institutional | No | - ---- - -## Supply Chain and Trade - -| Source | What it measures | History | Access | ml4t-data | -|--------|------------------|---------|--------|-----------| -| [FactSet Supply Chain Relationships](https://www.factset.com/marketplace/catalog/product/factset-supply-chain-relationships) (formerly Revere) | Customer, supplier, partner and competitor links between companies | North America from 2003; other regions from 2011 to 2016 | Institutional; academic via WRDS | No | -| [Bloomberg Supply Chain (SPLC)](https://professional.bloomberg.com/institutions/corporations/supply-chain/) | Supplier and customer relationships with revenue and cost exposure, 100,000+ companies | From 2006 | Bloomberg Terminal and enterprise feed | No | -| [S&P Global Panjiva](https://www.marketplace.spglobal.com/en/datasets/panjiva-supply-chain-intelligence-(22)) | Shipment-level customs records, 2 billion+ records | Not documented | Institutional | No | -| [ImportGenius](https://www.importgenius.com/pricing) | US and international bill-of-lading records | US imports from 2006, exports from January 2014 on the Pro and Enterprise tiers | Self-serve subscriptions; enterprise plans | No | - ---- - -## Prediction Markets - -Prediction-market prices are market data: each trade and quote has an exchange timestamp and is -never revised. The caveats are thin liquidity outside headline contracts and markets that are -listed and resolved over time, so a set of markets picked today is a survivor sample. - -| Source | What it measures | History | Access | ml4t-data | -|--------|------------------|---------|--------|-----------| -| [Kalshi](https://docs.kalshi.com/welcome) | CFTC-regulated event contracts: economics, politics, weather, sports | From the July 2021 launch | Public market-data API; trading needs a verified account | Yes: [`KalshiProvider`](kalshi.md) | -| [Polymarket](https://docs.polymarket.com/) | Event contracts on a crypto platform; CFTC approval for intermediated US access, November 2025 | From 2020 | Public market data | Yes: [`PolymarketProvider`](polymarket.md) | - ---- - -## Marketplaces and Discovery - -Marketplaces simplify procurement; they do not validate point-in-time behavior or methodology. - -| Source | What it offers | Access | -|--------|----------------|--------| -| [Nasdaq Data Link](https://data.nasdaq.com/) (formerly Quandl) | Catalog of financial, economic and alternative datasets, some free | Free account for free datasets; paid for premium | -| [Snowflake Marketplace](https://docs.snowflake.com/en/collaboration/collaboration-marketplace-about) | Datasets shared into a Snowflake account and queried in place | Free, trial and paid listings | -| [AWS Data Exchange](https://aws.amazon.com/data-exchange/) | Datasets delivered into AWS, billed through AWS | Per-product subscriptions | -| [Eagle Alpha](https://www.eaglealpha.com/) | Alternative data discovery and advisory, 2,500+ dataset profiles | Institutional | -| [Neudata](https://www.neudata.co/) | Alternative data discovery and research, 7,000+ datasets in its catalog | Institutional | -| [Datarade](https://datarade.ai/platforms) | Directory of data providers and data marketplaces | Free to browse; each provider sets its own terms | - -Of the free datasets that made Quandl popular, the WIKI prices table is frozen at April 2018 -(`ml4t-data` wraps it as [Wiki Prices](wiki_prices.md)), and the Zillow real-estate database -no longer has a catalog page and was last refreshed in July 2025. - ---- - -## Alternative Data in ml4t-data - -`ml4t-data` wraps two prediction-market sources, [Kalshi](kalshi.md) and -[Polymarket](polymarket.md). The FXMacroData provider also exposes news, risk-sentiment and CFTC -positioning endpoints alongside its release-timestamped macro data. Everything else on this page -needs its own client. - ---- - -## Name Changes and Closures - -| Then | Now | -|------|-----| -| data.ai (formerly App Annie) | Part of Sensor Tower since 2024 | -| Earnest Analytics | Part of Consumer Edge | -| SafeGraph Patterns (foot traffic) | Sold to Advan Research; SafeGraph now sells places, geometry and spend data | -| Orbital Insight | Acquired by Privateer, May 2024 | -| MarineTraffic | Acquired by Kpler, March 2023 | -| Spire Maritime | Acquired by Kpler, April 2025 | -| Refinitiv ESG scores | LSEG ESG data; a new LSEG ESG Scores suite launched March 2026 | -| Reddit and X APIs | Paid access (Reddit since July 2023; X pay-per-use since February 2026) | -| Stocktwits API | Closed to new developers | -| Polymarket (closed to US users from 2022) | [Amended CFTC order of designation](https://www.prnewswire.com/news-releases/polymarket-receives-cftc-approval-of-amended-order-of-designation-enabling-intermediated-us-market-access-302625833.html) for intermediated US access, November 2025 | -| PredictIt | [CFTC approval to operate a regulated exchange](https://www.bloomberg.com/news/articles/2025-09-05/predictit-gains-cftc-approval-to-launch-regulated-exchange), September 2025 | -| Quandl | Nasdaq Data Link (September 2021) | - -## See Also - -- [Provider comparison](index.md) -- [Market data sources](market_data.md) - [Fundamental data sources](fundamentals.md) +- [Prediction-market data sources](prediction_markets.md) +- [Provider comparison](index.md) diff --git a/docs/providers/aqr.md b/docs/providers/aqr.md index 392e617..7403a17 100644 --- a/docs/providers/aqr.md +++ b/docs/providers/aqr.md @@ -91,6 +91,7 @@ When using AQR data, cite the relevant papers: ## See Also +- [Research factor data sources](factors.md) - [AQR Datasets](https://www.aqr.com/Insights/Datasets) - [Fama-French Provider](fama_french.md) - [Provider reference](index.md) diff --git a/docs/providers/binance.md b/docs/providers/binance.md index a084946..22a8c1b 100644 --- a/docs/providers/binance.md +++ b/docs/providers/binance.md @@ -72,6 +72,7 @@ Binance may block access from certain countries. Consider using `BinancePublicPr ## See Also +- [Cryptocurrency data sources](crypto.md) - [Binance API](https://www.binance.com/en/binance-api) - [BinancePublic Provider](binance_public.md) - [Provider reference](index.md) diff --git a/docs/providers/binance_public.md b/docs/providers/binance_public.md index 07e636f..8c5af3b 100644 --- a/docs/providers/binance_public.md +++ b/docs/providers/binance_public.md @@ -74,6 +74,7 @@ Same as Binance: ## See Also +- [Cryptocurrency data sources](crypto.md) - [Binance Data Portal](https://data.binance.vision) - [Binance Provider](binance.md) - [Provider reference](index.md) diff --git a/docs/providers/coingecko.md b/docs/providers/coingecko.md index bc78277..dbaa6e6 100644 --- a/docs/providers/coingecko.md +++ b/docs/providers/coingecko.md @@ -74,5 +74,6 @@ be converted into correct daily OHLC values. ## See Also +- [Cryptocurrency data sources](crypto.md) - [CoinGecko API](https://www.coingecko.com/en/api/pricing) - [Provider reference](index.md) diff --git a/docs/providers/crypto.md b/docs/providers/crypto.md new file mode 100644 index 0000000..8419ac2 --- /dev/null +++ b/docs/providers/crypto.md @@ -0,0 +1,30 @@ +# Cryptocurrency Data Sources + +Cryptocurrency data is fragmented across venues, instrument types, and chains. Preserve exchange, +pair, quote currency, contract type, and symbol history. Aggregated prices can conceal venue +closures, wash trading, and differences between spot, futures, perpetuals, and options. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [Binance Spot API](https://github.com/binance/binance-spot-api-docs) | Binance spot pairs, trades, order books, and bars | REST and WebSocket APIs | Public market endpoints; credentials for private or higher-limit use | Venue availability and product access vary by jurisdiction | `BinanceProvider` | +| [Binance Public Data](https://data.binance.vision/) | Bulk spot and futures market files | Public archive download | Free | Files are venue-specific and symbol coverage changes over time | `BinancePublicProvider` | +| [OKX Market Data](https://www.okx.com/docs-v5/en/#rest-api-market-data) | Spot and derivatives, including perpetual funding and premium data | REST and WebSocket APIs | Public market endpoints | Instrument and jurisdiction availability change; retain instrument metadata | `OKXProvider` | +| [CoinGecko API](https://docs.coingecko.com/) | Aggregated asset, exchange, market, and on-chain DEX data | REST API | Demo and paid API plans | Aggregated asset identity and market prices are not a substitute for venue-level execution data | `CoinGeckoProvider` | +| [CryptoCompare API](https://developers.cryptocompare.com/documentation) | Aggregated and exchange-specific crypto market data | REST and streaming APIs | API key for supported usage | Release qualification depends on live credential validation; aggregation methodology matters | `CryptoCompareProvider` | +| [Kaiko Market Data](https://www.kaiko.com/products/market-data) | Centralized and decentralized venues, spot and derivatives, trades and order books | API, streaming, and cloud delivery | Institutional license | Venue and instrument history depend on the contracted product | No | +| [Tardis.dev historical data](https://docs.tardis.dev/historical-data-details/overview) | Raw and normalized messages for centralized crypto exchanges, including closed venues | API and downloadable files | Commercial plan; limited samples | Reconstruction requires exchange-specific message semantics and snapshot handling | No | +| [CoinAPI Market Data](https://www.coinapi.io/products/market-data-api) | Multi-exchange spot and derivatives market data | REST, WebSocket, FIX, and files | API key; plan-dependent | Normalized symbols and aggregate feeds can hide exchange-specific contract details | No | + +On-chain metrics, developer activity, and social signals belong in the +[alternative-data reference](alternative_data.md). This page covers tradable market data. + +## Related References + +- [Alternative data sources](alternative_data.md) +- [Futures data sources](futures.md) +- [Options data sources](options.md) +- [Market data selection](market_data.md) diff --git a/docs/providers/cryptocompare.md b/docs/providers/cryptocompare.md index 5234f2c..11a1bbb 100644 --- a/docs/providers/cryptocompare.md +++ b/docs/providers/cryptocompare.md @@ -68,5 +68,6 @@ Consult CryptoCompare's current terms before use. Access and limits were not ver ## See Also +- [Cryptocurrency data sources](crypto.md) - [CryptoCompare Pricing](https://min-api.cryptocompare.com/pricing) - [Provider reference](index.md) diff --git a/docs/providers/databento.md b/docs/providers/databento.md index b280baf..be1e086 100644 --- a/docs/providers/databento.md +++ b/docs/providers/databento.md @@ -181,6 +181,7 @@ streaming, use `provider.client` directly. ## See Also +- [Equity](equities.md), [futures](futures.md), and [options](options.md) source references - [Databento Pricing](https://databento.com/pricing) - [Databento Reference](databento_reference.md) - Detailed schema guide - [Provider reference](index.md) diff --git a/docs/providers/eodhd.md b/docs/providers/eodhd.md index b624e25..c69a24f 100644 --- a/docs/providers/eodhd.md +++ b/docs/providers/eodhd.md @@ -112,5 +112,6 @@ Get your API key at [eodhd.com/register](https://eodhd.com/register). ## See Also +- [Equity](equities.md), [ETF](etfs.md), and [fundamental](fundamentals.md) source references - [EODHD Pricing](https://eodhd.com/pricing) - [Provider reference](index.md) diff --git a/docs/providers/equities.md b/docs/providers/equities.md new file mode 100644 index 0000000..a3a0f29 --- /dev/null +++ b/docs/providers/equities.md @@ -0,0 +1,37 @@ +# Equity Data Sources + +This reference compares reproducible equity price and microstructure products. For cross-sectional +research, prefer sources that retain delisted securities and expose corporate-action inputs. For +intraday research, distinguish consolidated US feeds from a single exchange or venue. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [Yahoo Finance](https://finance.yahoo.com/) | Global listed equities; vendor-defined history | Web-backed API through the provider | No credential | Not a survivorship-free research panel; adjustment and retention rules are vendor controlled | `YahooFinanceProvider` | +| [Alpaca Market Data](https://docs.alpaca.markets/docs/about-market-data-api) | US equities and ETFs from 2016 | REST and WebSocket APIs | Account; feed entitlement depends on plan | The basic feed is IEX rather than the consolidated SIP; delisted coverage is not documented | `AlpacaDataProvider` | +| [Tiingo End-of-Day](https://www.tiingo.com/documentation/end-of-day) | US equities, funds, and ETFs with raw and adjusted prices | REST API | API key; plan-dependent limits | Not documented as a survivorship-free universe | `TiingoProvider` | +| [EODHD](https://eodhd.com/financial-apis/api-for-historical-data-and-volumes/) | Global end-of-day and intraday securities data | REST API | API key; coverage depends on plan | Exchange, adjustment, and delisted coverage vary by market and subscription | `EODHDProvider` | +| [Finnhub](https://finnhub.io/docs/api) | US quotes plus plan-dependent global and historical data | REST and WebSocket APIs | API key; endpoint entitlement depends on plan | Free access does not establish a complete historical universe | `FinnhubProvider` | +| [Twelve Data](https://twelvedata.com/docs) | Multi-exchange equities and other assets | REST and WebSocket APIs | API key; plan and exchange entitlements apply | Coverage and delays differ across exchanges | `TwelveDataProvider` | +| [Massive Stocks](https://massive.com/stocks) | US stock trades, quotes, aggregates, and reference data | REST, WebSocket, and flat files | API key; history and feed depend on plan | Confirm SIP versus exchange coverage and corporate-action treatment | `MassiveProvider` | +| [CRSP US Stock Databases](https://www.crsp.org/research/crsp-us-stock-databases/) | Active and inactive US securities, monthly from 1925 and daily from 1962 | Institutional files, commonly through WRDS | Academic or institutional license | Identifier and delisting-return conventions require deliberate handling | No | +| [Norgate Data](https://norgatedata.com/data-content-tables.php) | US, Australian, and Canadian stocks with plan-dependent delisted history | Local database and client integrations | Paid subscription | Daily data only; survivorship and index-membership features depend on tier | No | +| [Sharadar Equity Prices](https://data.nasdaq.com/databases/SEP) | Active and delisted US equities from 1998 | API and bulk delivery | Paid subscription | Daily bars; adjustment fields must be selected consistently | No | +| [NYSE TAQ](https://www.nyse.com/market-data/historical/daily-taq) | Trades and quotes reported by US exchanges | Daily institutional files | Exchange data license | Raw records need symbol-history, correction, and session processing | No | +| [AlgoSeek US Equities](https://www.algoseek.com/products.html#us_equity) | US trades, quotes, bars, reference data, and historical constituents | Cloud delivery and bulk files | Commercial license; samples available | Product schemas and SIP-derived fields differ; confirm the exact package | No | +| [Databento Nasdaq TotalView-ITCH](https://databento.com/datasets/XNAS.ITCH) | Nasdaq order-level data from 2018 | API and batch download | Usage-based account | Raw exchange messages are not adjusted and do not represent the full US consolidated market | `DataBentoProvider` | +| [Nasdaq TotalView-ITCH samples](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/) | Selected full trading days for Nasdaq-listed securities | Public binary files | Free samples | Samples are sparse and unsuitable for a broad historical panel | `ITCHSampleProvider` | +| [LSEG Tick History](https://www.lseg.com/en/data-analytics/market-data/data-feeds/tick-history) | Cross-asset trades, quotes, and depth from global venues | API and managed file delivery | Institutional license | Venue, contributor, and field coverage depend on the licensed package | No | + +`DataBentoProvider` and `MassiveProvider` are multi-asset classes. The mappings above describe only +their equity capabilities. See the [provider comparison](index.md) for package-level credentials +and interfaces. + +## Related References + +- [ETF data sources](etfs.md) +- [Fundamental data sources](fundamentals.md) +- [Market data selection](market_data.md) diff --git a/docs/providers/etfs.md b/docs/providers/etfs.md new file mode 100644 index 0000000..1a0eb8b --- /dev/null +++ b/docs/providers/etfs.md @@ -0,0 +1,31 @@ +# ETF Data Sources + +ETF research often needs more than traded prices. Fund identity, liquidation history, holdings, +classifications, fees, distributions, and net asset value can each come from a different product. +Do not treat an active-fund screener as a historical universe. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [Yahoo Finance](https://finance.yahoo.com/markets/etfs/) | Listed ETF prices and current descriptive data | Web-backed API through the provider | No credential | Current listings and vendor-adjusted prices do not form a survivorship-free fund panel | `YahooFinanceProvider` | +| [Alpaca Market Data](https://docs.alpaca.markets/docs/about-market-data-api) | US-listed ETF trades, quotes, and bars from 2016 | REST and WebSocket APIs | Account; feed entitlement depends on plan | Provides market data, not a historical holdings or classification database | `AlpacaDataProvider` | +| [Tiingo End-of-Day](https://www.tiingo.com/documentation/end-of-day) | US equity, mutual-fund, and ETF prices | REST API | API key; plan-dependent limits | Not documented as a complete dead-fund universe | `TiingoProvider` | +| [Tiingo Fund and ETF Fees](https://www.tiingo.com/documentation/mutual-fund-and-etf-fees) | Current and historical fee records for funds and ETFs | Enterprise data delivery | Institutional agreement | Fee history is a separate product from prices and holdings | `TiingoProvider` for prices only | +| [EODHD](https://eodhd.com/financial-apis/stock-etfs-fundamental-data-feeds) | ETF prices plus plan-dependent holdings and fund fields | REST API | API key; plan-dependent | Holdings dates, classifications, and constituent weights require point-in-time checks | `EODHDProvider` | +| [Massive Stocks](https://massive.com/stocks) | US-listed ETF trades, quotes, aggregates, and reference data | REST, WebSocket, and flat files | API key; plan-dependent | ETF-specific holdings and net asset value are not the same as exchange market data | `MassiveProvider` | +| [Sharadar Fund Prices](https://sharadar.com/docs/funds) | Active and delisted US-listed funds, ETFs, closed-end funds, ETNs, and ETDs from December 1997 | API and bulk delivery | Paid subscription | Daily price product; verify whether a separate dataset is needed for holdings or classifications | No | +| [Norgate Data](https://norgatedata.com/data-package-faq.php) | US ETF and ETN history from 1993, with plan-dependent delisted coverage | Local database and client integrations | Paid subscription | Daily frequency; constituent and classification features depend on package | No | +| [CRSP Survivor-Bias-Free US Mutual Fund Database](https://www.crsp.org/research/) | Active and inactive US open-end mutual funds with fund characteristics and returns | Institutional files, commonly through WRDS | Academic or institutional license | It is a mutual-fund research database, not a complete ETF holdings product | No | + +The same ticker can represent different share classes or change its fund objective. Preserve stable +identifiers and effective dates for classifications, fees, and holdings rather than joining only on +the current ticker. + +## Related References + +- [Equity data sources](equities.md) +- [Fixed-income data sources](fixed_income.md) +- [Market data selection](market_data.md) diff --git a/docs/providers/factors.md b/docs/providers/factors.md new file mode 100644 index 0000000..4780b98 --- /dev/null +++ b/docs/providers/factors.md @@ -0,0 +1,27 @@ +# Research Factor Data Sources + +Factor libraries publish returns from research portfolios, not returns from investable products. +Choose a source by universe, formation method, weighting, frequency, and revision policy. Do not +combine similarly named factors without reconciling their construction. + +!!! note "Verified September 2026" + This category has enough distinct, maintained sources for a standalone reference. Entries were + checked against the linked official or author-maintained data page. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [Kenneth French Data Library](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) | US and international factors, sorted portfolios, industries, and breakpoints; core US monthly series begin in 1926 | Downloadable ZIP, CSV, and text files | Free | Histories can be reconstructed after upstream CRSP revisions; archive vintages when reproducibility matters | `FamaFrenchProvider` | +| [AQR Data Library](https://www.aqr.com/Insights/Datasets) | Equity and cross-asset value, momentum, quality, low-beta, trend, and long-history research portfolios | Downloadable spreadsheets | Free under dataset terms | Series are hypothetical research portfolios, not AQR product returns; methodology differs by paper | `AQRFactorProvider` | +| [Global Factor Data](https://jkpfactors.com/data) | 153 characteristics across 93 countries and four regions, with portfolio sorts and reference files | Configurable file downloads; stock-level data through WRDS | Factor returns free under CC BY-NC 4.0; stock-level access requires WRDS | Noncommercial data license and global data-screening choices constrain reuse and comparison | No | + +`FamaFrenchProvider.fetch()` selects supported factor, portfolio, industry, international, or +breakpoint datasets. `AQRFactorProvider.download()` acquires the spreadsheets and `fetch()` reads a +supported local dataset. Neither class computes live portfolio returns. + +## Selection Notes + +- Use Fama-French for canonical academic benchmarks and portfolio sorts. +- Use AQR for paper-specific equity and cross-asset premia such as QMJ, BAB, VME, and TSMOM. +- Use Global Factor Data for broad characteristic definitions and consistent country coverage. +- Store the downloaded file and retrieval date because published research histories may be + revised. diff --git a/docs/providers/fama_french.md b/docs/providers/fama_french.md index cddbf12..2717692 100644 --- a/docs/providers/fama_french.md +++ b/docs/providers/fama_french.md @@ -110,6 +110,7 @@ ff5_mom = provider.fetch_combined(["ff5", "mom"]) ## See Also +- [Research factor data sources](factors.md) - [Ken French Data Library](https://mba.tuck.dartmouth.edu/pages/faculty/ken.french/data_library.html) - [AQR Provider](aqr.md) - [Provider reference](index.md) diff --git a/docs/providers/finnhub.md b/docs/providers/finnhub.md index f1e2977..51d6729 100644 --- a/docs/providers/finnhub.md +++ b/docs/providers/finnhub.md @@ -74,5 +74,6 @@ Get your API key at [finnhub.io/register](https://finnhub.io/register). ## See Also +- [Equity](equities.md) and [fundamental](fundamentals.md) source references - [Finnhub Pricing](https://finnhub.io/pricing) - [Provider reference](index.md) diff --git a/docs/providers/fixed_income.md b/docs/providers/fixed_income.md new file mode 100644 index 0000000..d769ab4 --- /dev/null +++ b/docs/providers/fixed_income.md @@ -0,0 +1,29 @@ +# Fixed-Income Data Sources + +Fixed-income research combines security reference data, evaluated or traded prices, yields, +cash flows, ratings, and transactions. A yield curve or evaluated price is a model output, while a +TRACE record is an executed trade. Keep those sources distinct. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [FRED and ALFRED](https://fred.stlouisfed.org/docs/api/fred/) | Treasury yields, credit spreads, policy rates, indexes, and other fixed-income series from contributing agencies | REST API | Free API key | Many series are aggregates or model outputs; use ALFRED vintages when release-time correctness matters | `FREDProvider` | +| [US Treasury Fiscal Data](https://fiscaldata.treasury.gov/api-documentation/) | Marketable debt, interest expense, auction, ownership, and related fiscal datasets | REST API and downloadable files | Free | Fiscal series and security records are not secondary-market quotes | No | +| [New York Fed Markets Data](https://www.newyorkfed.org/markets/reference-rates) | SOFR, EFFR, OBFR, repo rates, operations, and related money-market data | Markets Data APIs and downloads | Free | Reference rates are published aggregates with documented revision rules, not executable quotes | No | +| [FINRA Fixed Income and TRACE](https://www.finra.org/finra-data/fixed-income) | Corporate, agency, securitized, and 144A security and trade data | Website, daily files, feeds, and enhanced historical products | Public summaries or licensed subscription | TRACE disseminates executed trades, not quotes; caps, delays, and corrections affect interpretation | No | +| [CRSP US Treasury Database](https://www.crsp.org/research/) | Bills, notes, bonds, and fixed-term indexes for academic research | Institutional files | Academic or institutional license | CRSP conventions and month-end index calculations differ from live execution data | No | +| [ICE Fixed Income Data](https://www.ice.com/fixed-income-data-services) | Evaluated pricing, reference data, indexes, analytics, and market data across global debt markets | Feeds, APIs, and managed files | Institutional license | Evaluated prices are estimates and methodology, contributor coverage, and redistribution rights are product-specific | No | + +For exchange-traded interest-rate and bond futures, use the [futures data reference](futures.md). +For policy rates and economic releases, use the [macroeconomic data reference](macro.md). + +## Selection Notes + +- Use security-level reference data with dated identifiers, coupons, call schedules, and cash + flows. CUSIPs can change and a current description is not a historical master. +- Avoid treating a stale last trade as a current market price. Corporate bonds can trade + infrequently, and evaluated prices add a pricing model. +- Preserve publication and revision timestamps for curves and macro-derived spreads. diff --git a/docs/providers/fred.md b/docs/providers/fred.md index b21aa0f..f8c4cb4 100644 --- a/docs/providers/fred.md +++ b/docs/providers/fred.md @@ -74,6 +74,7 @@ Get your free API key at [fred.stlouisfed.org/docs/api/fred](https://fred.stloui ## See Also +- [Macroeconomic](macro.md) and [fixed-income](fixed_income.md) source references - [FRED API Docs](https://fred.stlouisfed.org/docs/api/fred/) - [Example Config](https://github.com/ml4t/data/blob/main/examples/configs/fred_economic.yaml) - [Provider reference](index.md) diff --git a/docs/providers/fundamentals.md b/docs/providers/fundamentals.md index 9076283..f9e1c10 100644 --- a/docs/providers/fundamentals.md +++ b/docs/providers/fundamentals.md @@ -1,157 +1,47 @@ # Fundamental Data Sources -This page compares sources of equity fundamentals: financial statements, as-reported versus -restated values, and analyst estimates. It covers the free SEC routes, retail APIs, and the -institutional and academic databases, and says which ones `ml4t-data` can fetch. - -!!! note "Verified 2026-09" - Vendor coverage, access terms, endpoint names and ownership change often. Every entry below - was checked against the vendor's own documentation in September 2026. Confirm the current terms - before you build a dataset on any of them. - ---- - -## Choosing a Source - -### Point-in-time first - -A backtest must only see the value that was public at the decision date. Companies restate -earlier periods, and vendors often overwrite the original figure with the restated one. A -dataset that stores only the latest value leaks future information into every historical -signal built from it, and the leak is invisible in the data itself. - -A source is point-in-time when each value carries the date it became public and superseded -versions are kept. The tables below use three grades: - -| Grade | Meaning | -|-------|---------| -| **Yes** | Each value carries its filing or release date, and earlier versions of a restated period remain available. The field that makes it so is named. | -| **Partial** | A date is present but has a caveat, for example the date of the latest filing that included the period rather than the first, or point-in-time only in a paid dataset. | -| **No** | Only the current (possibly restated) value, with no date of public availability. | - -Two further points apply even with a point-in-time source: - -- A filing date is not a trading date. Filings arrive during or after the session; use the - acceptance timestamp where available and apply the value from the next session. -- The SEC XBRL Frames API returns one value per company and period, the last one filed. It - looks convenient for cross-sections and is not point-in-time. - -### Survivorship - -The universe must include companies that were later delisted, acquired or went bankrupt. A -source that covers only today's listed companies biases every cross-sectional result toward -survivors. Sharadar, Zacks, Compustat and CRSP state delisted coverage explicitly; for the others, -check before relying on it. - -### Identifier mapping - -Tickers change and are reused. Fundamentals are keyed by the issuer (SEC CIK, Compustat GVKEY, -vendor IDs); prices are usually keyed by the security (ticker, CRSP PERMNO, FIGI). Joining the two -needs a dated mapping, and building one is usually more work than fetching either dataset. - -### In the book - -- Chapter 2, "A due diligence framework for data sourcing" (section 2.3): point-in-time - correctness, survivorship bias, identifier integrity, vendor tiers, and a vendor due diligence - checklist. -- Chapter 4, "The point-in-time pipeline" (section 4.1): building a bitemporal fundamentals - dataset from SEC EDGAR, and why the Frames API is not safe for backtests. "Entity resolution and - mapping" (section 4.2) covers the identifier problem. - ---- - -## Free SEC Routes - -The SEC publishes every XBRL financial statement that US registrants file. Detailed XBRL tagging -phased in from 2009 (large filers first) to 2011. Access needs no key; requests must send a -`User-Agent` header that identifies the caller, and the SEC's fair-access policy limits request -rates. - -| Source | What it returns | Point-in-time | Estimates | ml4t-data | -|--------|-----------------|---------------|-----------|-----------| -| [EDGAR `companyfacts` / `companyconcept` APIs](https://www.sec.gov/search-filings/edgar-application-programming-interfaces) | Every XBRL fact for one company (all concepts, or one concept), from every filing that reported it | **Yes**: each fact carries `filed`, `accn` (accession number) and `form` | No | No | -| [EDGAR `submissions` API](https://www.sec.gov/search-filings/edgar-application-programming-interfaces) | Filing history per company: forms, filing and acceptance dates, former names and tickers | Filing metadata, used to date the facts above | No | No | -| [EDGAR `frames` API](https://www.sec.gov/search-filings/edgar-application-programming-interfaces) | One value per company for a concept and calendar period | **No**: the last-filed value for each period | No | No | -| [Financial Statement Data Sets](https://www.sec.gov/data-research/sec-markets-data/financial-statement-data-sets) and [Financial Statement and Notes Data Sets](https://www.sec.gov/data-research/sec-markets-data/financial-statement-notes-data-sets) | Quarterly bulk files from 2009: submissions (`sub`), numeric values (`num`), tags, and for the Notes sets the footnote detail | **Yes**: values are as filed; `sub` holds `filed` and `accepted` per submission | No | No | - -`companyfacts.zip` and `submissions.zip` provide the same content as the APIs in one nightly bulk -download. Python wrappers such as `edgartools` parse filings and statements on top of these -endpoints. - ---- - -## Retail and Developer APIs - -| Source | Coverage | Point-in-time | Estimates | Access | ml4t-data | -|--------|----------|---------------|-----------|--------|-----------| -| [Sharadar Core US Fundamentals (SF1)](https://sharadar.com/docs/fundamentals) | About 18,000 active and delisted US companies, from 1998 | **Yes**: as-reported dimensions (`ARQ`, `ARY`, `ART`) are indexed by `datekey`, the SEC filing date. The most-recent dimensions (`MRQ`, `MRY`, `MRT`) are restated. | No | Paid. Sold direct at sharadar.com since July 2026; institutional access remains through Nasdaq Data Link. | No | -| [SimFin](https://www.simfin.com/en/fundamental-data-download/) | About 5,000 US stocks, from 2003 | **Partial**: rows carry `Publish Date` and `Restated Date`, but the standard datasets hold the latest restated value. As-reported bulk datasets are on paid tiers. | No | Free tier (5 years of fundamentals, delayed bulk download); paid tiers extend history. | No | -| [Financial Modeling Prep](https://site.financialmodelingprep.com/developer/docs/stable/income-statement) | Global, 70,000+ securities; up to 30+ years for large caps | **Partial**: statements carry `filingDate` and `acceptedDate`; separate as-reported endpoints return the filed values. Confirm how a restatement is versioned. | Yes | Free tier (US only); paid tiers for global coverage and full history. | No | -| [EODHD](https://eodhd.com/financial-apis/stock-etfs-fundamental-data-feeds) | 70+ exchanges; major US companies from 1985, non-US from 2000 | **Partial**: each yearly and quarterly entry has `filing_date`; the documentation does not say whether values are restated. | Yes: current consensus and revision trend | Paid; fundamentals are not in the free plan. | Yes: `EODHDProvider.fetch_financials()`, `fetch_company_metrics()` | -| [Intrinio](https://intrinio.com/products/us-fundamentals) | US SEC filers; annual from 2007, quarterly from 2009 | **Yes**: each fundamental has `filing_date`, and `is_latest` marks whether a later filing superseded it. Both reported and standardized feeds. | Yes: Zacks estimates, sold separately | Paid | No | -| [Zacks](https://zacksdata.com/datasets/fundamental-data/) | US and Canada; 19,500+ companies including 9,000+ delisted on Nasdaq Data Link (ZFA); Zacks Data history from 1979 | **Yes** in Zacks Data's point-in-time history; check the specific Nasdaq Data Link table before assuming it | Yes: Zacks consensus is the core product | Paid: Nasdaq Data Link, or Zacks Data for institutions | No | -| [Finnhub](https://finnhub.io/docs/api/financials-reported) | Global standardized statements; as-reported statements from SEC filings | **Partial**: `/stock/financials-reported` returns each filing with `filedDate` and `acceptedDate`. The standardized `/stock/financials` endpoint has no filing date. | Yes (premium) | Free tier includes as-reported statements; standardized statements and estimates are premium. | Yes: `FinnhubProvider.fetch_financials()` (standardized endpoint), `fetch_company_metrics()` | -| [Massive](https://massive.com/docs/rest/stocks/fundamentals/income-statements) (formerly Polygon.io) | About 6,700 US companies, from 2009 | **Partial**: `filing_date` is the date of the most recent SEC filing that included the period, not the original filing. Use the EDGAR filings index for the first filing date. | No | Paid: Stocks Advanced plan or the Financials & Ratios expansion. | Partial: `MassiveProvider.fetch_company_metrics()` (ratios) | -| [Tiingo](https://www.tiingo.com/documentation/fundamentals) | 5,500+ US equities and ADRs, 20+ years | **Partial**: `asReported=true` returns as-reported instead of restated values; `date` is the release date | No | Paid add-on; a free evaluation covers the Dow 30 for 3 years. | No (the Tiingo provider covers prices) | -| Yahoo Finance, via `yfinance` (unofficial) | Most listed tickers; about five annual and five quarterly periods | **No**: latest values only, no filing date | Yes: current consensus only | Free, unofficial; Yahoo's terms restrict use. | Yes: `YahooFinanceProvider.fetch_financials()`, `fetch_company_metrics()` | - ---- - -## Institutional and Academic Databases - -These are licensed by firms and universities. Students usually reach them through -[WRDS](https://wrds-www.wharton.upenn.edu/) (Wharton Research Data Services) when their -institution subscribes; which databases are available depends on the institution's licenses. - -| Source | Coverage | Point-in-time | Estimates | Access | ml4t-data | -|--------|----------|---------------|-----------|--------|-----------| -| [Compustat](https://www.marketplace.spglobal.com/en/datasets/compustat-financials-(8)) (S&P Global Market Intelligence) | North America and Global | **No** in the standard annual and quarterly files, which hold restated values | No (see Capital IQ) | Institutional; academic via WRDS | No | -| Compustat Point-in-Time and Compustat Snapshot | North America, snapshots from 1987 | **Yes**: every value as it was known at each snapshot date, including preliminary figures | No | Institutional; Snapshot via WRDS where licensed | No | -| CRSP/Compustat Merged (CCM) | US; links CRSP securities (PERMNO) to Compustat issuers (GVKEY) | Inherits the Compustat file it is joined to; the link table itself is dated | No | Academic via WRDS. CRSP was acquired by Morningstar in February 2026. | No | -| [S&P Capital IQ Financials](https://www.marketplace.spglobal.com/en/datasets/s-p-capital-iq-financials-(10)) | Global | **Yes**: Premium Financials carries filing dates for the full history and product delivery dates from 2004 | Yes: Capital IQ Estimates | Institutional (Capital IQ Pro, Xpressfeed, Snowflake); some via WRDS | No | -| [FactSet Fundamentals](https://www.factset.com/marketplace/catalog/product/factset-fundamentals-point-in-time) | Global | **Yes**: FactSet Fundamentals Point-in-Time, from February 1999 | Yes: FactSet Estimates, with point-in-time consensus | Institutional | No | -| [LSEG Company Fundamentals](https://www.lseg.com/en/data-catalogue/company-data) (includes Worldscope) and [I/B/E/S](https://www.lseg.com/en/data-catalogue/company-data/ibes-estimates/broker-estimates) (formerly Refinitiv, Thomson Reuters) | Fundamentals: 120,000+ companies on 150+ exchanges, US from the early 1980s, other markets from the 1990s. I/B/E/S: North America from 1976, other markets from 1987 | **Yes**: point-in-time versions of fundamentals and estimates are offered | Yes: I/B/E/S | Institutional; academic via WRDS where licensed | No | -| [Bloomberg](https://professional.bloomberg.com/products/data/enterprise-catalog/investment-research-data/) | Global | **Yes**: Company Financials, Estimates and Pricing Point-in-Time through Data License | Yes: Bloomberg Estimates (BEst) | Institutional (Terminal, Data License) | No | - ---- - -## Fundamentals in ml4t-data - -The Yahoo, EODHD, Finnhub and Massive providers expose `fetch_financials()` and -`fetch_company_metrics()`. Statements come back in one long-format schema: - -```python -from ml4t.data.providers import EODHDProvider - -provider = EODHDProvider() # reads EODHD_API_KEY -income = provider.fetch_financials("AAPL", statement="income", period="quarterly") -# columns: symbol, provider, statement_type, period_type, period_end, line_item, value, -# currency, fiscal_year, fiscal_period, filed_at, source -``` - -`filed_at` is filled when the vendor response carries a filing date (`filing_date`, `filingDate`, -`filedDate` or `acceptedDate`) and is empty otherwise. Treat `period_end` as the end of the -reporting period, never as the date the value was known. When `filed_at` is empty, the frame is -not point-in-time; use it for exploration, not for backtests. - -`MassiveProvider.fetch_financials()` still targets the retired Financials endpoint; Massive now -serves statements from `/stocks/financials/v1/income-statements`, `balance-sheets` and -`cash-flow-statements`. - ---- - -## Name Changes and Closures - -| Then | Now | -|------|-----| -| Quandl | Nasdaq Data Link (September 2021); legacy `quandl` names persist in URLs and code | -| Polygon.io | Massive (October 30, 2025); `api.polygon.io` still answers alongside `api.massive.com` | -| Refinitiv (Thomson Reuters Financial & Risk) | LSEG Data & Analytics | -| CRSP (University of Chicago) | Owned by Morningstar since February 2, 2026; CRSP indexes are being renamed Morningstar indexes | -| Sharadar on Nasdaq Data Link only | Also sold direct at sharadar.com since July 2026 | -| IEX Cloud | Retired August 31, 2024; its endpoints no longer answer | - -## See Also - -- [Provider reference](index.md) -- [EODHD](eodhd.md), [Finnhub](finnhub.md), [Massive](massive.md), [Yahoo Finance](yahoo.md) +Fundamental research needs the value that was public at the decision date. Companies amend +filings, vendors standardize line items, and current databases often overwrite earlier values. +Keep the filing or release timestamp, accession or version, period end, currency, and issuer +identifier with every observation. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [SEC EDGAR APIs](https://www.sec.gov/search-filings/edgar-application-programming-interfaces) | US registrant submissions and XBRL facts; structured statement coverage phases in from 2009 to 2011 | REST APIs and nightly bulk ZIP files | Free; identifying `User-Agent` and fair-access limits apply | `companyfacts` retains filing metadata, but the cross-sectional `frames` endpoint returns the last-filed value for a period | No | +| [SEC Financial Statement Data Sets](https://www.sec.gov/data-research/sec-markets-data/financial-statement-data-sets) | Quarterly as-filed statement facts from 2009 | Bulk ZIP files | Free | Raw XBRL tags and duplicate facts require issuer, accession, unit, period, and form-level normalization | No | +| [Yahoo Finance](https://finance.yahoo.com/) | Current statements, ratios, and analyst fields for listed securities | Web-backed API through the provider | No credential | Latest values lack reliable first-publication and revision history | `YahooFinanceProvider` | +| [EODHD Fundamentals](https://eodhd.com/financial-apis/stock-etfs-fundamental-data-feeds) | Global company statements, estimates, and related fields with market-dependent history | REST API | API key; paid fundamentals entitlement | Filing dates are present, but the public documentation does not establish complete restatement versioning | `EODHDProvider` | +| [Finnhub reported financials](https://finnhub.io/docs/api/financials-reported) | Global standardized data and SEC as-reported statements | REST API | API key; endpoint entitlement depends on plan | The as-reported endpoint carries filing dates; standardized statements do not provide the same point-in-time contract | `FinnhubProvider` | +| [Massive stock financials](https://massive.com/docs/rest/stocks/fundamentals/income-statements) | US company statements and ratios from SEC-derived data | REST API | API key; plan-dependent | A filing date can identify the latest filing that contains a period rather than its first publication | `MassiveProvider` for company metrics only | +| [Sharadar Core US Fundamentals](https://sharadar.com/docs/fundamentals) | Active and delisted US companies from 1998 with as-reported and restated dimensions | API and bulk delivery | Paid subscription | Select AR dimensions for as-reported research and preserve `datekey`; MR dimensions are restated | No | +| [SimFin fundamentals](https://www.simfin.com/en/fundamental-data-download/) | Standardized company statements with plan-dependent US and international history | API and bulk files | Free samples and paid tiers | Standard files can contain latest restatements; as-reported availability depends on product | No | +| [Financial Modeling Prep statements](https://site.financialmodelingprep.com/developer/docs/stable/income-statement) | Global standardized and as-reported statements with plan-dependent history | REST API and bulk products | API key; plan-dependent | Filing and acceptance dates do not by themselves establish that every superseded version is retained | No | +| [Intrinio US Fundamentals](https://intrinio.com/products/us-fundamentals) | US SEC filers with reported and standardized statements | REST API and bulk products | Commercial license | Standardization and restatement fields are product-specific; verify the licensed endpoint | No | +| [Compustat Financials](https://www.marketplace.spglobal.com/en/datasets/compustat-financials-(8)) | North American and global standardized financials | Xpressfeed, cloud, and institutional files; academic access through WRDS | Institutional license | Standard annual and quarterly files are restated; point-in-time products are separate | No | +| [FactSet Fundamentals Point-in-Time](https://www.factset.com/marketplace/catalog/product/factset-fundamentals-point-in-time) | Global normalized company fundamentals with historical versions | APIs, feeds, and managed files | Institutional license | Point-in-time coverage begins later than some standard-history products and depends on the subscription | No | +| [LSEG Company Fundamentals](https://www.lseg.com/en/data-catalogue/company-data) | Global company statements, estimates, and related reference data | APIs, feeds, cloud, and institutional files | Institutional license | As-reported, standardized, estimates, and point-in-time histories are separate products | No | + +## Package Contract + +Yahoo, EODHD, Finnhub, and Massive expose fundamental or company-metric methods. Statement frames +use a long schema with `period_end` and an optional `filed_at`. An empty `filed_at` means the frame +does not establish when the value became public and should not be used as point-in-time evidence. + +`MassiveProvider.fetch_financials()` still targets a retired vendor endpoint. The supported mapping +above is limited to `fetch_company_metrics()` until that runtime contract changes. + +## Selection Notes + +- Use EDGAR or an explicitly point-in-time product when filing chronology is central. +- Include delisted issuers and a dated mapping between issuer identifiers and traded securities. +- Treat analyst estimates separately from reported facts and preserve every estimate vintage. + +## Related References + +- [Equity data sources](equities.md) +- [Research factor data sources](factors.md) +- [Provider comparison](index.md) diff --git a/docs/providers/futures.md b/docs/providers/futures.md new file mode 100644 index 0000000..25620b5 --- /dev/null +++ b/docs/providers/futures.md @@ -0,0 +1,27 @@ +# Futures Data Sources + +Futures history is stored by expiring contract. A continuous series is a research construction, +so record the contract-selection rule, roll trigger, and price adjustment along with the output. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [Databento CME Globex MDP 3.0](https://databento.com/datasets/GLBX.MDP3) | CME Group futures and options from June 2010; order-level depth from March 2017 | API and batch download | Usage-based account | Continuous symbols embed a roll rule; retain individual contracts for reproducibility | `DataBentoProvider` | +| [Massive Futures](https://massive.com/futures) | Futures aggregates, trades, quotes, and reference data | REST, WebSocket, and flat files | API key; plan-dependent | Product history and exchange entitlements depend on the account | `MassiveProvider` | +| [CME DataMine](https://www.cmegroup.com/datamine.html) | Historical CME, CBOT, NYMEX, and COMEX futures and options; history varies by dataset | API, exchange downloads, and cloud delivery | Paid by dataset; academic program available | Data is contract-specific; continuous series and rolls are user-defined | No | +| [ICE Futures market data](https://www.ice.com/market-data) | ICE futures markets across energy, agriculture, financials, and other products | Feeds, APIs, and historical files | Exchange or vendor license | Coverage, history, and redistribution rights are product-specific | No | +| [Norgate Futures](https://norgatedata.com/futurespackage.php) | Global futures and cash commodities | Local database and client integrations | Paid subscription | Daily data; back-adjustment and roll settings materially affect returns | No | +| [FirstRate Data Futures](https://firstratedata.com/it/futures) | Individual and continuous histories for actively traded futures | Downloadable files | One-time or subscription purchase | Vendor-built continuous series may not match a strategy's intended roll rule | No | +| [CFTC Commitments of Traders](https://www.cftc.gov/MarketReports/CommitmentsofTraders/index.htm) | Weekly aggregate positions for reportable US futures and options markets | Public bulk files and reports | Free | Position reports are delayed aggregates, not price or order-book data | No | + +Exchange codes, contract multipliers, tick values, trading hours, and symbology change over time. +Keep dated reference data with prices and open interest. + +## Related References + +- [Options data sources](options.md) +- [Fixed-income data sources](fixed_income.md) +- [Market data selection](market_data.md) diff --git a/docs/providers/fx.md b/docs/providers/fx.md new file mode 100644 index 0000000..26a44f8 --- /dev/null +++ b/docs/providers/fx.md @@ -0,0 +1,26 @@ +# Foreign Exchange Data Sources + +Spot foreign exchange trades over the counter, so there is no consolidated tape. A dealer quote, +an interdealer venue price, and an official reference rate answer different research questions. +Reported volume is source-specific. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [OANDA v20](https://developer.oanda.com/rest-live-v20/introduction/) | OANDA account prices and candles for supported instruments | REST and streaming APIs | OANDA account and token | One broker's executable or indicative prices are not the whole OTC market | `OandaProvider` | +| [Twelve Data](https://twelvedata.com/docs) | Currency-pair time series and streaming data | REST and WebSocket APIs | API key; plan-dependent | Provider aggregation and delay differ by pair and account | `TwelveDataProvider` | +| [FXMacroData](https://fxmacrodata.com/) | Public USD rates and plan-dependent FX macro context | REST API | Public endpoints or API key | Macro and reference series are not executable dealer quotes | `FXMacroDataProvider` | +| [Massive Forex](https://massive.com/docs/rest/forex/overview) | Currency aggregates, quotes, and reference data | REST, WebSocket, and flat files | API key; plan-dependent | Confirm contributor, timestamp, and quote-side semantics | `MassiveProvider` | +| [Dukascopy historical data](https://www.dukascopy.com/swiss/english/marketwatch/historical/) | FX and CFD history from the Dukascopy trading environment | Download interface | Free | Single broker and venue context; CFD instruments are not spot transactions | No | +| [TrueFX historical downloads](https://www.truefx.com/truefx-historical-downloads/) | Top-of-book quotes for major and cross currency pairs | Registered file download | Free registration | Contributor set and retention differ from an institutional consolidated product | No | +| [ECB Data Portal exchange rates](https://data.ecb.europa.eu/key-figures/ecb-interest-rates-and-exchange-rates/exchange-rates) | Official euro reference rates and related statistical series | [SDMX REST API](https://data.ecb.europa.eu/help/getting-data-web-services-sdmx-0) and downloads | Free | Daily reference rates are not intraday or executable market prices | No | +| [LSEG Tick History](https://www.lseg.com/en/data-analytics/market-data/data-feeds/tick-history) | Contributed and venue FX quotes within a cross-asset archive | API and managed file delivery | Institutional license | Contributor and venue coverage depend on the licensed package | No | + +## Related References + +- [Macroeconomic data sources](macro.md) +- [Futures data sources](futures.md) +- [Market data selection](market_data.md) diff --git a/docs/providers/fxmacrodata.md b/docs/providers/fxmacrodata.md new file mode 100644 index 0000000..e9ff525 --- /dev/null +++ b/docs/providers/fxmacrodata.md @@ -0,0 +1,20 @@ +# FXMacroData + +`FXMacroDataProvider` retrieves FX-oriented macroeconomic and reference-rate data. It is a +specialized provider rather than an OHLCV adapter, so use its macro methods directly instead of +passing it to `DataManager.fetch()`. + +## Access + +Public USD endpoints work without a credential. Set `FXMACRODATA_API_KEY` or `FXMD_API_KEY` when +the selected endpoint or account requires a key. + +```python +from ml4t.data.providers import FXMacroDataProvider + +provider = FXMacroDataProvider() +``` + +The provider does not turn macro or reference series into executable currency quotes. For spot +and intraday FX products, compare the [foreign exchange data sources](fx.md). For economic +release and revision requirements, see [macroeconomic data sources](macro.md). diff --git a/docs/providers/index.md b/docs/providers/index.md index 6b4af85..003058e 100644 --- a/docs/providers/index.md +++ b/docs/providers/index.md @@ -2,9 +2,13 @@ ML4T Data supports 20+ live and specialized data providers, plus synthetic and testing providers. -For the wider vendor landscape, including sources the library does not wrap, see -[Market Data Sources](market_data.md), [Fundamental Data Sources](fundamentals.md) and -[Alternative Data Sources](alternative_data.md). +For the wider vendor landscape, including sources the library does not wrap, start with +[Market Data Sources](market_data.md) or go directly to the [equity](equities.md), +[ETF](etfs.md), [futures](futures.md), [options](options.md), [foreign exchange](fx.md), or +[cryptocurrency](crypto.md) reference. Separate references cover [fixed income](fixed_income.md), +[macroeconomic data](macro.md), [fundamentals](fundamentals.md), +[alternative data](alternative_data.md), [research factors](factors.md), and +[prediction markets](prediction_markets.md). ## Provider Comparison @@ -82,8 +86,9 @@ provider's capabilities before passing it to `DataManager` or `async_batch_load( | Futures and options | Databento, Massive | | Cryptocurrency | Binance, Binance Public, OKX, CoinGecko, CryptoCompare | | Foreign exchange | Oanda, Twelve Data, FXMacroData | -| Economic series | FRED, FXMacroData | -| Academic factors | Fama-French, AQR | +| Economic series | FRED, FXMacroData; external official sources are compared in [Macroeconomic Data Sources](macro.md) | +| Academic factors | Fama-French, AQR; external libraries are compared in [Research Factor Data Sources](factors.md) | +| Prediction markets | Kalshi, Polymarket; external markets are compared in [Prediction-Market Data Sources](prediction_markets.md) | | Equity fundamentals | Yahoo Finance, EODHD, Finnhub, Massive; other vendors and the SEC routes are compared in [Fundamental Data Sources](fundamentals.md) | Provider access, coverage, retention, and redistribution terms can differ by account tier. Confirm diff --git a/docs/providers/kalshi.md b/docs/providers/kalshi.md index e8bd59c..d85807c 100644 --- a/docs/providers/kalshi.md +++ b/docs/providers/kalshi.md @@ -57,6 +57,7 @@ Kalshi is regulated by the CFTC (Commodity Futures Trading Commission) as a Desi ## See Also +- [Prediction-market data sources](prediction_markets.md) - [Kalshi Developer Docs](https://kalshi.com/developer) - [Polymarket Provider](polymarket.md) - [Provider reference](index.md) diff --git a/docs/providers/macro.md b/docs/providers/macro.md new file mode 100644 index 0000000..5716a13 --- /dev/null +++ b/docs/providers/macro.md @@ -0,0 +1,33 @@ +# Macroeconomic Data Sources + +Macroeconomic observations are released, revised, benchmarked, and sometimes replaced. A valid +historical model uses the vintage available at the decision date, not the latest value assigned to +an old observation period. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the source and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [FRED and ALFRED](https://fred.stlouisfed.org/docs/api/fred/) | US and international economic and financial series from many contributing agencies; ALFRED stores vintages | REST API | Free API key | FRED's latest value can include revisions; use real-time periods or ALFRED for point-in-time research | `FREDProvider` | +| [FXMacroData](https://fxmacrodata.com/) | FX-oriented macroeconomic and reference-rate series | REST API | Public USD endpoints or API key | Coverage and release metadata vary by endpoint; reference series are not executable FX quotes | `FXMacroDataProvider` | +| [US Bureau of Labor Statistics](https://www.bls.gov/developers/) | Employment, prices, productivity, compensation, and related US labor statistics | Public Data API and files | Free; registration key raises limits | Seasonal adjustment and annual benchmark revisions can alter history | No | +| [US Bureau of Economic Analysis](https://apps.bea.gov/api/) | National, industry, international, and regional economic accounts | REST API | Free API key | Vintage availability differs by dataset; later benchmark revisions can be large | No | +| [World Bank Indicators API](https://datahelpdesk.worldbank.org/knowledgebase/articles/889392) | Nearly 16,000 indicators across more than 45 databases, with many series extending over 50 years | REST API and bulk downloads | Free, no key | Country definitions, source agencies, observation status, and revisions differ by indicator | No | +| [Eurostat APIs](https://ec.europa.eu/eurostat/web/user-guides/data-browser/api-data-access/) | European economic, demographic, trade, and social statistics | Statistics and SDMX REST APIs plus bulk files | Free | The public database exposes the latest dataset and does not preserve prior versions | No | +| [ECB Data Portal](https://data.ecb.europa.eu/help/getting-data-web-services-sdmx-0) | Euro-area monetary, financial, market, banking, and economic statistics | SDMX REST API and downloads | Free | Frequency, seasonal adjustment, and revision policy are series-specific | No | + +## Release-Time Checklist + +- Store observation period, publication timestamp, revision or vintage timestamp, units, + seasonal-adjustment status, and source agency. +- Join low-frequency releases to the first trading session after publication. Do not forward-fill + from the observation period before the release occurred. +- Treat forecasts, flash estimates, and final releases as separate vintages. + +## Related References + +- [Foreign exchange data sources](fx.md) +- [Fixed-income data sources](fixed_income.md) +- [Prediction-market data sources](prediction_markets.md) diff --git a/docs/providers/market_data.md b/docs/providers/market_data.md index 2b67c84..55f16be 100644 --- a/docs/providers/market_data.md +++ b/docs/providers/market_data.md @@ -1,175 +1,49 @@ # Market Data Sources -This page maps the market data vendor landscape: US equities and ETFs (daily and intraday), -futures, options, FX and crypto, including the institutional and academic sources that -`ml4t-data` does not wrap. For the providers the library does wrap (free tiers, async support, -credentials), the [provider comparison](index.md) is the reference; this page links to it rather -than repeating it. +Use the asset-class references below to compare data products that `ml4t-data` wraps with +external sources that have a reproducible API, bulk-download route, or institutional feed. +Provider support means that the named public class is included in the package. It does not imply +that every product or field sold by the source is implemented. !!! note "Verified September 2026" - Vendor coverage, history, access terms and ownership change often. Every entry below was - checked in September 2026 against the vendor's own documentation or a published announcement. - Where a vendor does not document a property, the table says "not documented" instead of - guessing. Confirm the current terms before you build a dataset on any of them. - ---- - -## Choosing a Source - -### Survivorship - -A research universe must contain the securities that later delisted, merged or went bankrupt. -Free price sources usually carry only what trades today, so any cross-sectional backtest built -on them overstates returns. The "Delisted" columns below say whether a vendor keeps dead -securities. - -### Adjustments and corporate actions - -Splits, dividends and spin-offs break raw price series. Vendors differ in what they deliver: -unadjusted prices, adjusted prices, or both with the adjustment factors. Keep the unadjusted -series and the factors if you can: adjusted history is rewritten every time a new corporate -action occurs, so an adjusted series downloaded today differs from one downloaded last year. -Futures have the equivalent problem at each roll; a continuous contract is a construction choice, -not a market price. - -### Point-in-time universe membership - -An index strategy must trade the constituents as of each date, not today's members. Only a few -vendors sell historical index membership; without it, "S&P 500 stocks" silently means today's -survivors. - -### Granularity and timestamps - -Daily bars, minute bars, trades and quotes, and full order-book messages are different products. -For US equities, a consolidated (SIP) feed and a single-venue feed (for example IEX only) report -different volumes and prices. Check which timestamp a vendor records (exchange, SIP or receipt) -and how it defines the trading session. - -### Licensing - -Exchange data carries its own license terms. Redistribution, display and derived-data rights -vary by vendor and tier, and a dataset licensed for research may not be licensed for production. - -### In the book - -- Chapter 2, "The asset-class market data landscape" (section 2.2): what each asset class lets - you observe, its failure modes, and the engineering decisions it forces. -- Chapter 2, "A due diligence framework for data sourcing" (section 2.3): survivorship, - corporate actions, identifier integrity, and a vendor checklist. -- Chapter 3, "The anatomy of modern market data feeds" (section 3.2) and "Microstructure data - quality and sessionization" (section 3.6): feeds, timestamps and session handling for intraday - data. - ---- - -## Providers ml4t-data Wraps - -Yahoo Finance, Alpaca, EODHD, Tiingo, Twelve Data, Massive (formerly Polygon.io), Finnhub, -Databento, Oanda, Binance, Binance Public, OKX, CoinGecko, CryptoCompare, Wiki Prices and NASDAQ -ITCH samples all have `ml4t-data` providers. Their asset classes, free tiers and credentials are -in the [provider comparison](index.md) and on each provider's page. Three of them appear in the -tables below as well, because they are also the reference source for a data class: Databento -(order-book and CME data), Alpaca (US equities with a stated history start) and the NASDAQ ITCH -samples. - ---- - -## US Equities and ETFs: Daily Research Panels - -| Source | Coverage and history | Delisted | Adjustments | Finest granularity | Point-in-time index membership | Access | ml4t-data | -|--------|----------------------|----------|-------------|--------------------|--------------------------------|--------|-----------| -| [CRSP US Stock Databases (on WRDS)](https://wrds-www.wharton.upenn.edu/pages/about/data-vendors/center-for-research-in-security-prices-crsp/) | NYSE monthly from December 1925 and daily from July 1962; NASDAQ from December 1972 | Yes, keyed by PERMNO and PERMCO | Returns with and without dividends; full corporate-action history | Daily and monthly | Yes, index constituent files on WRDS | Academic and institutional, mainly via WRDS | No | -| [Norgate Data](https://norgatedata.com/data-content-tables.php) | US, Australian and Canadian stocks and ETFs; US listed and delisted coverage essentially complete from late 1992 | Yes, on the Platinum and Diamond tiers | Four modes: unadjusted, capital-reconstruction adjusted, plus special distributions, total return | Daily | Yes, on the Platinum and Diamond tiers | Retail paid subscription | No | -| [Sharadar Equity Prices (SEP)](https://data.nasdaq.com/databases/SEP) | About 21,000 active and delisted US tickers, from 1998 | Yes | OHLCV adjusted for splits and stock dividends; an unadjusted close; a close also adjusted for cash dividends and spin-offs | Daily | Separate S&P 500 constituents table (`SHARADAR/SP500`) | Paid; direct at sharadar.com since July 2026, or Nasdaq Data Link | No | -| [FirstRate Data](https://firstratedata.com/about/FAQ) | 16,000+ US tickers from 2000, including 7,000+ delisted | Partly: included where the vendor could source them | Unadjusted, split-adjusted, and split and dividend adjusted | 1-minute bars | No | Retail paid, one-off purchase | No | -| [Kibot](https://www.kibot.com/faq.html) | US stocks and ETFs; minute bars from 1998 | Partly: active and delisted lists, stated as incomplete | Split and dividend adjusted series plus an adjustments API | Tick with bid and ask | No | Free guest access (daily only), paid tiers | No | -| [Alpaca](https://docs.alpaca.markets/us/docs/about-market-data-api) | US equities and ETFs, from 2016 | Not documented | `adjustment` parameter: raw, split, dividend, spin-off or all | Trades and quotes | No | Free (IEX feed, delayed SIP); paid for real-time SIP | Yes: `AlpacaProvider` | - -The free wrapped sources (Yahoo Finance, Tiingo, EODHD free tier) carry currently listed -securities and vendor-adjusted prices; treat them as prototyping data, not as a survivorship-free -panel. Wiki Prices stopped updating in April 2018. - ---- - -## US Equities: Trades, Quotes and Order Books - -| Source | Coverage and history | Delisted | Adjustments | Finest granularity | Point-in-time index membership | Access | ml4t-data | -|--------|----------------------|----------|-------------|--------------------|--------------------------------|--------|-----------| -| [NYSE TAQ](https://wrds-www.wharton.upenn.edu/pages/about/data-vendors/nyse-trade-and-quote-taq/) | All US exchanges; Monthly TAQ 1993 to 2014, Daily TAQ (millisecond timestamps) from September 2003 | Not applicable: daily files of every security traded that day | Raw prices; adjust with CRSP factors | Trades and NBBO quotes | No | Academic and institutional via WRDS | No | -| [AlgoSeek](https://algoseek.com/us-equities-package/) | Every listed US equity from 2007 (SIP), including delisted securities; adjustment factors from 2007 | Yes | Raw and adjusted prices plus adjustment-factor files | Trades and quotes with nanosecond timestamps; bars from 1 second | Yes, Index Components dataset | Institutional; free sandbox with up to one year of data | No | -| [Databento](https://databento.com/datasets/XNAS.ITCH) | US equities across exchanges and ATSs; Nasdaq TotalView-ITCH from 2018 | Not applicable: raw exchange feeds | Unadjusted; corporate actions sold as separate reference data | Full order book (market by order) | No | Usage-based; see the [Databento page](databento.md) | Yes: `DataBentoProvider` | -| [LSEG Tick History](https://www.lseg.com/en/data-analytics/market-data/data-feeds/tick-history) (formerly Refinitiv Tick History) | Cross-asset, 580+ venues and contributors, from January 1996 | Not documented | Not documented | Trades, quotes and order-book depth | No | Institutional; some universities license it | No | -| [Nasdaq TotalView-ITCH samples](https://emi.nasdaq.com/ITCH/Nasdaq%20ITCH/) | Selected full-day files for Nasdaq-listed stocks | Not applicable | Raw messages | Every order add, cancel and execution | No | Free samples; the full archive is a paid Nasdaq product | Yes: [NASDAQ ITCH provider](nasdaq_itch.md) | -| [DTN IQFeed](https://www.iqfeed.net/dev/) | Equities, futures, options, FX; tick history rolling 180 days, minute bars from 2005 to 2007 depending on asset | Not documented | Not documented | Trades and quotes | No | Retail paid | No | -| [Kinetick](https://kinetick.com/) | Market data service bundled with NinjaTrader; stocks, futures, FX | Not documented | Not documented | Tick | No | Retail; free end-of-day tier | No | - ---- - -## Futures - -| Source | Coverage and history | Expired contracts | Continuous series | Finest granularity | Access | ml4t-data | -|--------|----------------------|-------------------|-------------------|--------------------|--------|-----------| -| [CME DataMine](https://www.cmegroup.com/datamine.html) | CME, CBOT, NYMEX and COMEX futures and options; history varies by dataset, some back to the 1970s | Yes, data is per contract | No, individual contracts | Market depth and time and sales | Paid per dataset; academic discount | No | -| [Databento CME Globex (GLBX.MDP3)](https://databento.com/datasets/GLBX.MDP3) | CME Group futures and options from June 2010; full order-by-order depth from March 2017 | Yes, per contract | Continuous symbology by roll rule | Full order book | Usage-based | Yes: `DataBentoProvider.fetch_continuous_futures()` | -| [Norgate Data](https://norgatedata.com/futurespackage.php) | Global futures and cash commodities | Yes, individual contract histories | Unadjusted and back-adjusted continuous contracts | Daily | Retail paid | No | -| [FirstRate Data](https://firstratedata.com/it/futures) | 130 most active futures, intraday from 2007 to 2008 | Yes, individual contracts | Unadjusted, absolute-adjusted and ratio-adjusted continuous series | 1-minute bars | Retail paid | No | - ---- - -## Options - -| Source | Coverage and history | Delisted underlyings | Analytics | Finest granularity | Access | ml4t-data | -|--------|----------------------|----------------------|-----------|--------------------|--------|-----------| -| [OptionMetrics IvyDB](https://optionmetrics.com/united-states/) | US equity and index options from 1996; Europe from 2002 | Yes | Implied volatility, Greeks, volatility surfaces; adjusts for dividends, splits and spin-offs | Daily; a separate intraday product exists | Academic via WRDS, institutional | No | -| [ORATS](https://orats.com/data-api) | US equity, ETF and index options; end of day from 2007, 1-minute from August 2020 | Not documented | Smoothed implied volatility and Greeks | 1-minute snapshots | Retail paid, enterprise | No | -| [Cboe DataShop](https://datashop.cboe.com/documentation) (LiveVol) | Options trades, quotes and end of day from January 2012; VIX futures trades and quotes from April 2004 | Not documented | Greeks and implied volatility (Cboe Hanweck) as separate products | Trades and quotes | Usage-based | No | -| [Databento OPRA](https://databento.com/datasets/OPRA.PILLAR) | All US equity options from April 2013 | Not applicable | None: raw feed | Trades and quotes | Usage-based | Yes: `DataBentoProvider.fetch_option_chain()`, `fetch_option_quotes()` | -| [AlgoSeek](https://algoseek.com/us-equities-package/) | US equity options, full OPRA feed | Not documented | Greeks and implied volatility dataset | Trades and quotes | Institutional | No | - ---- - -## Foreign Exchange - -Spot FX trades over the counter, so there is no consolidated tape: every source reports one -dealer's or one venue's prices, and volumes are not comparable across sources. - -| Source | Coverage and history | Finest granularity | Access | ml4t-data | -|--------|----------------------|--------------------|--------|-----------| -| [Dukascopy](https://www.dukascopy.com/swiss/english/marketwatch/historical/) | FX, CFDs, commodities, indices from one Swiss bank's platform | Tick | Free download | No | -| [TrueFX](https://www.truefx.com/truefx-historical-downloads-2/) | Major and cross pairs, top of book | Tick, millisecond timestamps | Free historical downloads with registration | No | -| [LSEG Tick History](https://www.lseg.com/en/data-analytics/market-data/data-feeds/tick-history) | Contributed and venue FX quotes from 1996 | Tick | Institutional | No | -| [Oanda](oanda.md) | One broker's quotes | Seconds-level candles | Account required | Yes: `OandaProvider` | - ---- - -## Crypto - -Crypto trades on many venues around the clock, and reported volume on some venues is unreliable. -A source that keeps delisted pairs and closed exchanges (FTX, for example) matters for the same -survivorship reason as delisted stocks. - -| Source | Coverage and history | Delisted pairs and venues | Finest granularity | Access | ml4t-data | -|--------|----------------------|---------------------------|--------------------|--------|-----------| -| [Kaiko](https://www.kaiko.com/products/l1-l2-data) | 100+ centralized exchanges plus DeFi; trades from 2010, order-book snapshots from 2015 | Not documented | Trades, best bid and offer, full order book | Institutional | No | -| [Tardis.dev](https://docs.tardis.dev/historical-data-details/overview) | 50+ exchanges, spot, derivatives and options; most from March 2019 | Yes: closed venues such as FTX and BitMEX history retained | Raw exchange messages, full order-book depth | Paid; academic, solo and pro tiers cover the last four years, business tiers from March 2019 | No | -| [CoinAPI](https://www.coinapi.io/products/market-data-api) | 400+ exchanges; spot, futures, perpetuals, options | Not documented | Trades, quotes, order books | Free credits, paid, enterprise | No | -| Binance, Binance Public, OKX, CoinGecko, CryptoCompare | Venue or aggregator data; see the [provider comparison](index.md) | Venue-specific | Bars; funding rates and premium index for perpetuals | Free or free tier | Yes | - ---- - -## Name Changes and Closures - -| Then | Now | -|------|-----| -| CRSP (University of Chicago) | Owned by Morningstar since February 2, 2026; the CRSP Market Indexes were renamed Morningstar Market Indexes in 2026 | -| Thomson Reuters / Refinitiv Tick History | LSEG Tick History | -| Polygon.io | Massive (October 30, 2025); `api.polygon.io` still answers alongside `api.massive.com` | -| Sharadar on Nasdaq Data Link only | Also sold direct at sharadar.com since July 2026 | -| Quandl WIKI Prices | Frozen since April 2018; still available as a static dataset | -| IEX Cloud | Retired August 31, 2024 | + Coverage, history, access, and licensing change. The linked official product pages support the + entries as of September 2026. Confirm current terms before building a durable dataset. + +## Asset Classes + +| Reference | Main selection questions | +|-----------|--------------------------| +| [Equities](equities.md) | Survivorship, corporate actions, consolidated versus venue data, and timestamp semantics | +| [ETFs](etfs.md) | Delisted funds, holdings and classifications, fees, and adjusted prices | +| [Futures](futures.md) | Individual contracts, roll rules, continuous-series construction, and exchange licenses | +| [Options](options.md) | Contract coverage, quote history, implied-volatility methodology, and corporate actions | +| [Foreign exchange](fx.md) | Dealer or venue provenance, reference rates versus executable quotes, and volume interpretation | +| [Cryptocurrency](crypto.md) | Venue survivorship, instrument type, market fragmentation, and exchange reliability | + +## Shared Due Diligence + +- A historical universe should include securities, contracts, pairs, and venues that later + disappeared. Current-symbol lists create survivorship bias. +- Preserve raw prices and corporate-action or roll inputs where possible. Adjusted histories and + continuous futures are constructed series that can change when recomputed. +- Daily bars, minute bars, trades and quotes, and full order-book messages are different products. + Confirm session definitions, timestamp origin, and venue coverage. +- Exchange and vendor licenses can restrict redistribution, display, production use, or derived + data. A research entitlement does not establish production rights. + +## Name Changes and Retired Products + +| Previous name or product | Current status | +|--------------------------|----------------| +| Thomson Reuters / Refinitiv Tick History | [LSEG Tick History](https://www.lseg.com/en/data-analytics/market-data/data-feeds/tick-history) | +| Polygon.io | [Massive](https://massive.com/), renamed in October 2025; the library class is `MassiveProvider` | +| Quandl WIKI Prices | Frozen since April 2018; available through `WikiPricesProvider` as a static research dataset | +| IEX Cloud | Retired August 31, 2024; excluded from the source references | ## See Also - [Provider comparison](index.md) +- [Fixed-income data sources](fixed_income.md) +- [Macroeconomic data sources](macro.md) - [Fundamental data sources](fundamentals.md) - [Alternative data sources](alternative_data.md) diff --git a/docs/providers/massive.md b/docs/providers/massive.md index 7da5f06..7984ba9 100644 --- a/docs/providers/massive.md +++ b/docs/providers/massive.md @@ -83,3 +83,7 @@ failures use the shared provider exception types. See the current [Massive API documentation](https://massive.com/docs) and [account plans](https://massive.com/pricing) for service-side coverage and limits. + +Compare the relevant products in the [equity](equities.md), [ETF](etfs.md), +[futures](futures.md), [options](options.md), [foreign exchange](fx.md), and +[fundamental](fundamentals.md) source references. diff --git a/docs/providers/nasdaq_itch.md b/docs/providers/nasdaq_itch.md index d146feb..2a6dc1e 100644 --- a/docs/providers/nasdaq_itch.md +++ b/docs/providers/nasdaq_itch.md @@ -68,5 +68,6 @@ Free sample files available from NASDAQ: ## See Also +- [Equity data sources](equities.md) - [NASDAQ ITCH Specification](https://www.nasdaqtrader.com/content/technicalsupport/specifications/dataproducts/NQTVITCHSpecification.pdf) - [Provider reference](index.md) diff --git a/docs/providers/oanda.md b/docs/providers/oanda.md index 1fe74ad..417d41f 100644 --- a/docs/providers/oanda.md +++ b/docs/providers/oanda.md @@ -71,5 +71,6 @@ Get credentials by opening a practice account at [oanda.com](https://www.oanda.c ## See Also +- [Foreign exchange data sources](fx.md) - [Oanda API](https://developer.oanda.com) - [Provider reference](index.md) diff --git a/docs/providers/okx.md b/docs/providers/okx.md index 09ceddf..149a1ff 100644 --- a/docs/providers/okx.md +++ b/docs/providers/okx.md @@ -66,5 +66,6 @@ async with OKXProvider() as provider: ## See Also +- [Cryptocurrency data sources](crypto.md) - [OKX API v5 Documentation](https://www.okx.com/docs-v5/en/) - [Provider reference](index.md) diff --git a/docs/providers/options.md b/docs/providers/options.md new file mode 100644 index 0000000..f1b7a14 --- /dev/null +++ b/docs/providers/options.md @@ -0,0 +1,29 @@ +# Options Data Sources + +Options research needs dated contract terms, underlying corporate actions, and a clear distinction +between observed quotes and vendor-computed analytics. Implied volatility and Greeks from two +vendors can differ even when they begin with the same market data. + +!!! note "Verified September 2026" + Entries were checked against the linked official product documentation. Coverage and access + depend on the vendor contract and can change. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [Databento OPRA](https://databento.com/datasets/OPRA.PILLAR) | US equity and index option trades and quotes from April 2013 | API and batch download | Usage-based account | Raw OPRA data does not include a vendor volatility surface or survivorship-cleaned underlying panel | `DataBentoProvider` | +| [Massive Options](https://massive.com/options) | US option chains, trades, quotes, aggregates, and plan-dependent analytics | REST, WebSocket, and flat files | API key; plan-dependent | Confirm whether Greeks and implied volatility are observed snapshots or recomputed fields | `MassiveProvider` | +| [OptionMetrics IvyDB US](https://optionmetrics.com/united-states/) | US equity and index options from 1996 | Institutional files, commonly through WRDS | Academic or institutional license | Daily standardized analytics are vendor estimates; intraday data is a separate product | No | +| [ORATS Data API](https://orats.com/data-api) | US equity, ETF, and index options; end-of-day history from 2007 and intraday snapshots | REST API and bulk files | Commercial account | Smoothed volatility and Greeks depend on ORATS models and cleaning rules | No | +| [Cboe DataShop](https://datashop.cboe.com/) | Historical option trades, quotes, summaries, and analytics across Cboe products and OPRA | Download and cloud delivery | Product-specific purchase | Coverage and analytics differ across DataShop products; select the exact schema before comparing results | No | +| [AlgoSeek US Options](https://www.algoseek.com/products.html#us_options) | OPRA-derived trades, quotes, bars, and analytics | Cloud delivery and bulk files | Commercial license; samples available | Quote cleaning and calculated analytics depend on the selected product | No | +| [CME DataMine options](https://www.cmegroup.com/datamine.html) | Options on CME Group futures across product families | API, exchange downloads, and cloud delivery | Paid by dataset | Futures-option symbology and exercise terms require the matching contract reference data | No | + +`DataBentoProvider` exposes option-chain and option-quote helpers. `MassiveProvider` exposes +multi-asset market data, including options. Neither mapping implies support for every analytics +field sold by the vendor. + +## Related References + +- [Futures data sources](futures.md) +- [Equity data sources](equities.md) +- [Market data selection](market_data.md) diff --git a/docs/providers/polymarket.md b/docs/providers/polymarket.md index 90beae0..6d26733 100644 --- a/docs/providers/polymarket.md +++ b/docs/providers/polymarket.md @@ -268,6 +268,9 @@ provider.close() # Close when done ## Changelog +See [Prediction-Market Data Sources](prediction_markets.md) for source and market-mechanism +comparisons. + ### 2025-12-07 - Fixed token resolution for new API format (`clobTokenIds` JSON string) - Fixed `get_token_prices()` to use `outcomes`/`outcomePrices` arrays diff --git a/docs/providers/prediction_markets.md b/docs/providers/prediction_markets.md new file mode 100644 index 0000000..a059084 --- /dev/null +++ b/docs/providers/prediction_markets.md @@ -0,0 +1,25 @@ +# Prediction-Market Data Sources + +Prediction-market prices are contract-specific. Preserve the complete question, resolution +source, close and settlement rules, outcome token, market status, and fee model. A quoted price is +an implied market probability only after accounting for spread, liquidity, and contract terms. + +!!! note "Verified September 2026" + This category has enough distinct sources and selection criteria for a standalone reference. + Entries were checked against current official API documentation. + +| Source and product | Coverage or history | Acquisition | Access | Material research caveat | ml4t-data provider | +|--------------------|---------------------|-------------|--------|--------------------------|--------------------| +| [Kalshi API](https://docs.kalshi.com/getting_started/quick_start_market_data) | Regulated event contracts across economics, politics, climate, technology, and other categories | REST and WebSocket APIs | Public market data; credentials for authenticated endpoints | Market tickers encode series and contract terms; use settled outcomes and rule changes, not titles alone | `KalshiProvider` | +| [Polymarket APIs](https://docs.polymarket.com/) | Binary and multi-outcome markets represented by outcome tokens on Polygon | Gamma and CLOB REST APIs plus WebSocket market data | Public market data; trading has additional requirements | Slugs, condition IDs, and outcome token IDs are different identifiers; YES and NO prices include spread | `PolymarketProvider` | +| [Manifold API and dumps](https://docs.manifold.markets/) | User-created social prediction markets across many question types | REST API and downloadable data dumps | Public data under documented licensing | Play-money markets, user-defined resolution, and automated-market-maker mechanics are not directly comparable with regulated cash markets | No | + +`KalshiProvider` lists series and markets and converts native candlesticks to the package OHLCV +schema. `PolymarketProvider` resolves slugs or condition IDs to outcome tokens and aggregates price +history. Public data access does not imply that trading is available in every jurisdiction. + +## Selection Notes + +- Use Kalshi when regulated contract definitions and exchange settlement are required. +- Use Polymarket for crypto-settled CLOB markets and on-chain outcome tokens. +- Use Manifold for social forecasting research where play-money incentives are acceptable. diff --git a/docs/providers/tiingo.md b/docs/providers/tiingo.md index 8ba3fda..fc00763 100644 --- a/docs/providers/tiingo.md +++ b/docs/providers/tiingo.md @@ -60,5 +60,6 @@ Get your API key at [tiingo.com/account/api/token](https://api.tiingo.com/accoun ## See Also +- [Equity](equities.md) and [ETF](etfs.md) source references - [Tiingo Pricing](https://tiingo.com/about/pricing) - [Provider reference](index.md) diff --git a/docs/providers/twelve_data.md b/docs/providers/twelve_data.md index 01fb605..fe6ae99 100644 --- a/docs/providers/twelve_data.md +++ b/docs/providers/twelve_data.md @@ -70,5 +70,6 @@ Get your API key at [twelvedata.com/account](https://twelvedata.com/account). ## See Also +- [Equity](equities.md) and [foreign exchange](fx.md) source references - [TwelveData Pricing](https://twelvedata.com/pricing) - [Provider reference](index.md) diff --git a/docs/providers/yahoo.md b/docs/providers/yahoo.md index 1334d45..b88ccb0 100644 --- a/docs/providers/yahoo.md +++ b/docs/providers/yahoo.md @@ -105,4 +105,5 @@ These features are available via yfinance but not yet in ml4t-data: ## See Also +- [Equity](equities.md) and [ETF](etfs.md) source references - [Provider reference](index.md) - All providers diff --git a/mkdocs.yml b/mkdocs.yml index d1409f5..8e35972 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -125,11 +125,22 @@ nav: - providers/index.md - Fundamental Data Sources: providers/fundamentals.md - Market Data Sources: providers/market_data.md + - Equity Data Sources: providers/equities.md + - ETF Data Sources: providers/etfs.md + - Futures Data Sources: providers/futures.md + - Options Data Sources: providers/options.md + - Foreign Exchange Data Sources: providers/fx.md + - Cryptocurrency Data Sources: providers/crypto.md + - Fixed-Income Data Sources: providers/fixed_income.md + - Macroeconomic Data Sources: providers/macro.md - Alternative Data Sources: providers/alternative_data.md + - Research Factor Data Sources: providers/factors.md + - Prediction-Market Data Sources: providers/prediction_markets.md - Yahoo Finance: providers/yahoo.md - Alpaca: providers/alpaca.md - CoinGecko: providers/coingecko.md - FRED: providers/fred.md + - FXMacroData: providers/fxmacrodata.md - Fama-French: providers/fama_french.md - AQR: providers/aqr.md - Wiki Prices: providers/wiki_prices.md diff --git a/scripts/validate_source_references.py b/scripts/validate_source_references.py new file mode 100644 index 0000000..578532a --- /dev/null +++ b/scripts/validate_source_references.py @@ -0,0 +1,201 @@ +"""Validate asset-class source-reference pages against the provider registry.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any + +import yaml + +from ml4t.data.providers.registry import ProviderSpec, advertised_provider_specs + +SOURCE_REFERENCE_PAGES = ( + "providers/equities.md", + "providers/etfs.md", + "providers/futures.md", + "providers/options.md", + "providers/fx.md", + "providers/crypto.md", + "providers/fixed_income.md", + "providers/macro.md", + "providers/fundamentals.md", + "providers/alternative_data.md", + "providers/factors.md", + "providers/prediction_markets.md", +) + +REQUIRED_COLUMNS = ( + "Source and product", + "Coverage or history", + "Acquisition", + "Access", + "Material research caveat", + "ml4t-data provider", +) + +_PROVIDER_CLASS = re.compile(r"`([A-Za-z][A-Za-z0-9_]*Provider)`") + + +class SourceReferenceError(ValueError): + """Raised when a source-reference contract is invalid.""" + + +class _MkDocsLoader(yaml.SafeLoader): + """Load MkDocs configuration without evaluating environment variables.""" + + +def _ignore_env(loader: _MkDocsLoader, node: yaml.Node) -> Any: + """Return the declared environment fallback as inert configuration data.""" + if isinstance(node, yaml.SequenceNode): + values = loader.construct_sequence(node) + return values[-1] if values else None + return loader.construct_scalar(node) + + +def _python_name(_loader: _MkDocsLoader, suffix: str, _node: yaml.Node) -> str: + """Keep MkDocs extension callables as inert dotted names.""" + return suffix + + +_MkDocsLoader.add_constructor("!ENV", _ignore_env) +_MkDocsLoader.add_multi_constructor("tag:yaml.org,2002:python/name:", _python_name) + + +def _markdown_cells(line: str) -> list[str]: + """Split one simple Markdown table row into stripped cells.""" + return [cell.strip() for cell in line.strip().strip("|").split("|")] + + +def _nav_paths(value: Any) -> set[str]: + """Collect Markdown paths from a MkDocs navigation tree.""" + if isinstance(value, str): + return {value} if value.endswith(".md") else set() + if isinstance(value, list): + paths: set[str] = set() + for item in value: + paths.update(_nav_paths(item)) + return paths + if isinstance(value, dict): + paths = set() + for item in value.values(): + paths.update(_nav_paths(item)) + return paths + return set() + + +def _comparison_rows(page: Path) -> list[tuple[int, list[str]]]: + """Return rows from the required comparison table in one page.""" + lines = page.read_text(encoding="utf-8").splitlines() + for index, line in enumerate(lines): + if tuple(_markdown_cells(line)) != REQUIRED_COLUMNS: + continue + if index + 1 >= len(lines): + break + separator = _markdown_cells(lines[index + 1]) + if len(separator) != len(REQUIRED_COLUMNS) or not all( + cell and set(cell) <= {"-", ":"} for cell in separator + ): + raise SourceReferenceError(f"{page}: comparison table has no valid separator row") + + rows: list[tuple[int, list[str]]] = [] + for row_index in range(index + 2, len(lines)): + row = lines[row_index] + if not row.lstrip().startswith("|"): + break + rows.append((row_index + 1, _markdown_cells(row))) + if not rows: + raise SourceReferenceError(f"{page}: comparison table has no source rows") + return rows + raise SourceReferenceError( + f"{page}: missing comparison table with columns {', '.join(REQUIRED_COLUMNS)}" + ) + + +def validate_source_references( + project_root: Path, + *, + provider_specs: Iterable[ProviderSpec] | None = None, + source_pages: Iterable[str] = SOURCE_REFERENCE_PAGES, +) -> None: + """Validate page inventory, navigation, row fields, and provider mappings.""" + project_root = project_root.resolve() + docs_root = project_root / "docs" + config_path = project_root / "mkdocs.yml" + config = yaml.load(config_path.read_text(encoding="utf-8"), Loader=_MkDocsLoader) + nav_paths = _nav_paths(config.get("nav", [])) + + specs = tuple(provider_specs or advertised_provider_specs()) + specs_by_class: Mapping[str, ProviderSpec] = {spec.class_name: spec for spec in specs} + + errors: list[str] = [] + for relative_page in source_pages: + page = docs_root / relative_page + if not page.is_file(): + errors.append(f"missing source-reference page: {relative_page}") + continue + if relative_page not in nav_paths: + errors.append( + f"source-reference page is missing from MkDocs navigation: {relative_page}" + ) + + try: + rows = _comparison_rows(page) + except SourceReferenceError as error: + errors.append(str(error)) + continue + + for line_number, cells in rows: + location = f"{page}:{line_number}" + if len(cells) != len(REQUIRED_COLUMNS): + errors.append( + f"{location}: expected {len(REQUIRED_COLUMNS)} fields, found {len(cells)}" + ) + continue + missing = [ + REQUIRED_COLUMNS[index] for index, value in enumerate(cells) if not value.strip() + ] + if missing: + errors.append(f"{location}: empty required fields: {', '.join(missing)}") + continue + + provider_cell = cells[-1] + if provider_cell == "No": + continue + provider_classes = _PROVIDER_CLASS.findall(provider_cell) + if len(provider_classes) != 1: + errors.append( + f"{location}: provider field must contain one backticked provider class or No" + ) + continue + class_name = provider_classes[0] + spec = specs_by_class.get(class_name) + if spec is None: + errors.append(f"{location}: unknown advertised provider class: {class_name}") + continue + provider_page = docs_root / "providers" / f"{spec.name}.md" + if not provider_page.is_file(): + errors.append( + f"{location}: provider class {class_name} has no provider page at " + f"providers/{spec.name}.md" + ) + + if errors: + raise SourceReferenceError("\n".join(errors)) + + +def main() -> int: + """Validate the checkout containing this script.""" + project_root = Path(__file__).resolve().parents[1] + try: + validate_source_references(project_root) + except SourceReferenceError as error: + print(error) + return 1 + print("Source-reference contracts are valid.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_source_reference_contract.py b/tests/test_source_reference_contract.py new file mode 100644 index 0000000..a6fd575 --- /dev/null +++ b/tests/test_source_reference_contract.py @@ -0,0 +1,99 @@ +"""Tests for asset-class source-reference documentation contracts.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from ml4t.data.providers.registry import ProviderSpec +from scripts.validate_source_references import SourceReferenceError, validate_source_references + +SOURCE_PAGES = ("providers/equities.md", "providers/macro.md") +TABLE_HEADER = ( + "| Source and product | Coverage or history | Acquisition | Access | " + "Material research caveat | ml4t-data provider |" +) +TABLE_SEPARATOR = "|---|---|---|---|---|---|" + + +def _write_fixture( + root: Path, + *, + equity_provider: str = "No", + include_macro_in_nav: bool = True, + equity_row: str | None = None, +) -> ProviderSpec: + """Create the smallest complete source-reference fixture.""" + provider_spec = ProviderSpec( + name="sample", + module="sample.module", + class_name="SampleProvider", + description="Sample", + capabilities=frozenset({"ohlcv"}), + ) + providers = root / "docs" / "providers" + providers.mkdir(parents=True) + (providers / "sample.md").write_text("# Sample\n", encoding="utf-8") + + valid_row = "| Source | Coverage | API | Free | Caveat | No |" + selected_row = equity_row or ( + f"| Source | Coverage | API | Free | Caveat | {equity_provider} |" + ) + (providers / "equities.md").write_text( + f"# Equities\n\n{TABLE_HEADER}\n{TABLE_SEPARATOR}\n{selected_row}\n", + encoding="utf-8", + ) + (providers / "macro.md").write_text( + f"# Macro\n\n{TABLE_HEADER}\n{TABLE_SEPARATOR}\n{valid_row}\n", + encoding="utf-8", + ) + + nav = [" - Equity: providers/equities.md"] + if include_macro_in_nav: + nav.append(" - Macro: providers/macro.md") + (root / "mkdocs.yml").write_text( + "nav:\n - Providers:\n" + "\n".join(nav) + "\n", + encoding="utf-8", + ) + return provider_spec + + +def test_repository_source_references_satisfy_contract(): + validate_source_references(Path(__file__).resolve().parents[1]) + + +def test_unknown_provider_class_is_rejected(tmp_path: Path): + spec = _write_fixture(tmp_path, equity_provider="`UnknownProvider`") + + with pytest.raises(SourceReferenceError, match="unknown advertised provider class"): + validate_source_references( + tmp_path, + provider_specs=(spec,), + source_pages=SOURCE_PAGES, + ) + + +def test_source_page_missing_from_navigation_is_rejected(tmp_path: Path): + spec = _write_fixture(tmp_path, include_macro_in_nav=False) + + with pytest.raises(SourceReferenceError, match="missing from MkDocs navigation"): + validate_source_references( + tmp_path, + provider_specs=(spec,), + source_pages=SOURCE_PAGES, + ) + + +def test_incomplete_source_row_is_rejected(tmp_path: Path): + spec = _write_fixture( + tmp_path, + equity_row="| Source | Coverage | API | | Caveat | No |", + ) + + with pytest.raises(SourceReferenceError, match="empty required fields: Access"): + validate_source_references( + tmp_path, + provider_specs=(spec,), + source_pages=SOURCE_PAGES, + )