diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 00000000..44d5f961 --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,6 @@ +# Revisions listed here are skipped by `git blame`. +# GitHub honors this file automatically. Locally, enable it with: +# git config blame.ignoreRevsFile .git-blame-ignore-revs + +# Normalize line endings to LF via .gitattributes (no content changes) +659fb36bec08d6bde072efe982ddc63303ba6103 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..fd434e98 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# Line-ending policy lives here, not in each contributor's core.autocrlf. +# `text=auto` canonicalizes every file git detects as text to LF *in the +# repository*; working trees may still use CRLF on Windows if that's the +# local preference. +* text=auto + +# Binary fixtures. These are byte-compared by the round-trip tests, so make +# their exemption explicit rather than relying on git's NUL-byte heuristic. +# (`binary` is shorthand for `-text -diff`.) +*.dbn binary +*.dbz binary +*.zst binary +*.frag binary diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2b30626d..59b97d9d 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -1,110 +1,110 @@ -name: CI -on: - push: - branches: - - main - - master - pull_request: - branches: - - main - - master - workflow_dispatch: - -jobs: - test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - version: - - '1.12' # Minimum supported version - - '1' # Latest stable - os: - - ubuntu-latest - - macOS-latest - - windows-latest - steps: - - uses: actions/checkout@v7 - - # Install Rust for compatibility tests with official dbn-cli - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - toolchain: stable - - # Cache Rust dependencies (Unix) - - name: Cache Rust dependencies (Unix) - if: runner.os != 'Windows' - uses: actions/cache@v6 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~/dbn-workspace/dbn/target/ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - # Cache Rust dependencies (Windows) - - name: Cache Rust dependencies (Windows) - if: runner.os == 'Windows' - uses: actions/cache@v6 - with: - path: | - ~/.cargo/bin/ - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - ~\dbn-workspace\dbn\target\ - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ runner.os }}-cargo- - - # Clone and build official dbn for compatibility tests - - name: Build dbn (Unix) - if: runner.os != 'Windows' - run: | - mkdir -p $HOME/dbn-workspace - if [ ! -f "$HOME/dbn-workspace/dbn/Cargo.toml" ]; then - rm -rf $HOME/dbn-workspace/dbn - git clone --depth 1 https://github.com/databento/dbn $HOME/dbn-workspace/dbn - fi - cd $HOME/dbn-workspace/dbn - cargo build --release - - - name: Build dbn (Windows) - if: runner.os == 'Windows' - shell: pwsh - run: | - $workspace = "$env:USERPROFILE\dbn-workspace" - New-Item -ItemType Directory -Force -Path $workspace | Out-Null - if (-not (Test-Path "$workspace\dbn\Cargo.toml")) { - Remove-Item -Recurse -Force -ErrorAction SilentlyContinue "$workspace\dbn" - git clone --depth 1 https://github.com/databento/dbn "$workspace\dbn" - } - cd "$workspace\dbn" - cargo build --release - - - uses: julia-actions/setup-julia@v3 - with: - version: ${{ matrix.version }} - # Omit `arch` so setup-julia picks the runner's native arch - # (macOS-latest is now arm64; setup-julia@v3 errors hard if we - # force x64 on an arm64 runner). - - - uses: julia-actions/cache@v3 - - - uses: julia-actions/julia-buildpkg@v1 - - - uses: julia-actions/julia-runtest@v1 - - - uses: julia-actions/julia-processcoverage@v1 - - - uses: codecov/codecov-action@v7 - with: - files: lcov.info - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false +name: CI +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + workflow_dispatch: + +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.12' # Minimum supported version + - '1' # Latest stable + os: + - ubuntu-latest + - macOS-latest + - windows-latest + steps: + - uses: actions/checkout@v7 + + # Install Rust for compatibility tests with official dbn-cli + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + + # Cache Rust dependencies (Unix) + - name: Cache Rust dependencies (Unix) + if: runner.os != 'Windows' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ~/dbn-workspace/dbn/target/ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + # Cache Rust dependencies (Windows) + - name: Cache Rust dependencies (Windows) + if: runner.os == 'Windows' + uses: actions/cache@v6 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + ~\dbn-workspace\dbn\target\ + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo- + + # Clone and build official dbn for compatibility tests + - name: Build dbn (Unix) + if: runner.os != 'Windows' + run: | + mkdir -p $HOME/dbn-workspace + if [ ! -f "$HOME/dbn-workspace/dbn/Cargo.toml" ]; then + rm -rf $HOME/dbn-workspace/dbn + git clone --depth 1 https://github.com/databento/dbn $HOME/dbn-workspace/dbn + fi + cd $HOME/dbn-workspace/dbn + cargo build --release + + - name: Build dbn (Windows) + if: runner.os == 'Windows' + shell: pwsh + run: | + $workspace = "$env:USERPROFILE\dbn-workspace" + New-Item -ItemType Directory -Force -Path $workspace | Out-Null + if (-not (Test-Path "$workspace\dbn\Cargo.toml")) { + Remove-Item -Recurse -Force -ErrorAction SilentlyContinue "$workspace\dbn" + git clone --depth 1 https://github.com/databento/dbn "$workspace\dbn" + } + cd "$workspace\dbn" + cargo build --release + + - uses: julia-actions/setup-julia@v3 + with: + version: ${{ matrix.version }} + # Omit `arch` so setup-julia picks the runner's native arch + # (macOS-latest is now arm64; setup-julia@v3 errors hard if we + # force x64 on an arm64 runner). + + - uses: julia-actions/cache@v3 + + - uses: julia-actions/julia-buildpkg@v1 + + - uses: julia-actions/julia-runtest@v1 + + - uses: julia-actions/julia-processcoverage@v1 + + - uses: codecov/codecov-action@v7 + with: + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml index 8dc82f8d..1c599132 100644 --- a/.github/workflows/TagBot.yml +++ b/.github/workflows/TagBot.yml @@ -1,33 +1,33 @@ -name: TagBot -on: - issue_comment: - types: - - created - workflow_dispatch: - inputs: - lookback: - default: '3' - -permissions: - actions: read - checks: read - contents: write - deployments: read - issues: read - discussions: read - packages: read - pages: read - pull-requests: read - repository-projects: read - security-events: read - statuses: read - -jobs: - TagBot: - if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' - runs-on: ubuntu-latest - steps: - - uses: JuliaRegistries/TagBot@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - ssh: ${{ secrets.DOCUMENTER_KEY }} +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: + inputs: + lookback: + default: '3' + +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read + +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + ssh: ${{ secrets.DOCUMENTER_KEY }} diff --git a/README.md b/README.md index 219c55a4..0f5cf7b0 100644 --- a/README.md +++ b/README.md @@ -1,244 +1,244 @@ -# DatabentoBinaryEncoding.jl - -Julia implementation of the Databento Binary Encoding (DBN) message encoding and storage format for normalized market data. - -**⚠️ Development Status**: This package is under active development. While core functionality is complete and tested for byte-for-byte compatibility with the official Rust implementation, the API may still evolve. Production use is possible but not yet recommended. - -For more details, read the [introduction to DBN](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). - -## Features - -- ✅ Complete DBN v3 Format Support -- ✅ Efficient streaming support (read and write) -- ✅ Timestamp-paced replay (simulate a live feed at any speed) -- ✅ Zstd file compression support (read and write) -- ✅ Bidirectional format conversion (DBN ↔ JSON/Parquet/CSV) -- ✅ Byte-for-byte compatibility with official implementations -- ✅ All DBN message types (Trades, MBO, MBP, OHLCV, definitions, statistics, imbalance, status) -- ✅ Consolidated & BBO schemas (CBBO, CMBP-1, TCBBO, BBO-1s/1m) -- ✅ Live control records (SymbolMappingMsg, SystemMsg, ErrorMsg), tolerant of v3 unset-`stype` sentinels -- ✅ High-precision timestamp handling -- ✅ Fixed-point price arithmetic - -## Performance - -DatabentoBinaryEncoding.jl is optimized for high-throughput market data processing: -- **Read**: Up to 40M records/sec with near-zero-allocation callback streaming -- **Write**: 11M records/sec with optimized bulk operations -- **Type-specific readers** (`read_trades`, `read_mbo`, etc.) are 5-6x faster than generic `read_dbn()` - -**Note**: This package supports DBN v2 and v3 formats only. DBN v1 files are not supported. To convert DBN v1 files, use the official Databento CLI: -```bash -dbn version1.dbn --output version2.dbn --upgrade -``` - -## Installation - -The package is registered in the General registry: - -```julia -using Pkg -Pkg.add("DatabentoBinaryEncoding") -``` - -Or, for the latest unreleased changes, install directly from GitHub: - -```julia -using Pkg -Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") -``` - -## Usage - -### Reading DBN Files - -```julia -using DatabentoBinaryEncoding -# Tip: `import DatabentoBinaryEncoding as DBN` gives you the terser `DBN.foo` prefix. - -# Read entire file into memory (fastest for bulk loading) -records = read_dbn("trades.dbn") - -# Read with metadata -metadata, records = read_dbn_with_metadata("trades.dbn") - -# Optimized type-specific eager read (5-6x faster than read_dbn) -trades = read_trades("trades.dbn") -mbos = read_mbo("mbo.dbn") - -# Generic streaming for mixed-type files -for record in DBNStream("large_file.dbn.zst") - println("Trade: $(record.price) @ $(record.size)") -end - -# High-performance callback streaming (near-zero allocations!) -# Best for pure processing workloads - 40 M records/sec -total = Ref(0.0) -foreach_trade("trades.dbn") do trade - total[] += price_to_float(trade.price) -end -println("Total: $(total[])") -``` - -### Replaying DBN Files - -Re-emit a file's records in time order, paced by their timestamps, to simulate a -live feed (useful for backtesting, demos, or driving consumers that expect -real-time data): - -```julia -# Replay in real time — each record fires after the real gap to the previous one -replay_dbn("trades.dbn") do rec - println(price_to_float(rec.price)) -end - -# Replay 100x faster, paced by receive time, with overnight gaps capped at 1s -replay_dbn("mbo.dbn.zst"; speed = 100, timestamp = :ts_recv, max_sleep = 1.0) do rec - handle(rec) -end - -# Replay an already-loaded collection (e.g. from read_dbn) -records = read_dbn("trades.dbn") -replay_records(records; speed = 5) do rec - handle(rec) -end -``` - -Key options: `speed` (time-compression multiplier; `Inf` = no waiting), -`timestamp` (`:ts_event` or `:ts_recv`), and `max_sleep` (cap on any single -wait, in seconds). Compression and unknown record types are handled exactly as -in `DBNStream`. - -**Timing resolution.** DBN timestamps are nanosecond precision, but pacing -accuracy is bounded by the sleep function. `Base.sleep` resolves to roughly -1 ms on Unix and as coarse as the system timer tick (~15 ms) on Windows, so -records spaced more tightly than that arrive clumped rather than as distinct -waits. Pacing is anchored to absolute wall-clock targets, so this clumping is -local — the stream re-synchronizes and timing error does not accumulate across -the file, and records sharing a timestamp are delivered back-to-back. For -sub-millisecond fidelity (e.g. dense MBO bursts) pass `precise = true`, which -busy-waits small gaps at the cost of pinning a CPU core: - -```julia -replay_dbn("mbo.dbn"; precise = true) do rec - handle(rec) -end -``` - -### Writing DBN Files - -```julia -using DatabentoBinaryEncoding, Dates - -# Create metadata for trades -metadata = Metadata( - UInt8(3), # DBN version - "XNAS", # dataset - Schema.TRADES, # schema - datetime_to_ts(DateTime(2024, 1, 1)), # start_ts - datetime_to_ts(DateTime(2024, 1, 2)), # end_ts - UInt64(1000), # limit - SType.RAW_SYMBOL, # stype_in - SType.RAW_SYMBOL, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String, String, Int64, Int64}[] # mappings -) - -# Create trade message -trade = TradeMsg( - RecordHeader( - UInt8(sizeof(TradeMsg)), - RType.MBP_0_MSG, - UInt16(1), # publisher_id - UInt32(12345), # instrument_id - UInt64(datetime_to_ts(DateTime(2024, 1, 1, 9, 30))) - ), - float_to_price(100.50), # price - UInt32(100), # size - Action.TRADE, - Side.BID, - UInt8(0), # flags - UInt8(0), # depth - UInt64(datetime_to_ts(DateTime(2024, 1, 1, 9, 30))), # ts_recv - Int32(0), # ts_in_delta - UInt32(1) # sequence -) - -# Write to file -write_dbn("output.dbn", metadata, [trade]) - -# Write compressed file -write_dbn("output.dbn.zst", metadata, [trade]) -``` - -### Streaming Writer - -```julia -# Create streaming writer for real-time data -writer = DBNStreamWriter("live_trades.dbn", "XNAS", Schema.TRADES) - -# Write records as they arrive -for price in [100.0, 100.25, 100.50] - trade = create_trade_message(price, 100) # Your trade creation logic - write_record!(writer, trade) -end - -close_writer!(writer) -``` - -### Data Export - -```julia -# Convert to different formats -dbn_to_csv("trades.dbn", "trades.csv") -dbn_to_json("trades.dbn", "trades.json") -dbn_to_parquet("trades.dbn", "trades.parquet") # ZSTD-compressed by default - -# Convert to DataFrame for analysis -df = records_to_dataframe(records) -``` - -### Data Import - -```julia -# Convert other formats to DBN -json_to_dbn("trades.json", "trades.dbn") -parquet_to_dbn("trades.parquet", "trades.dbn", schema=Schema.TRADES, dataset="XNAS") -csv_to_dbn("trades.csv", "trades.dbn", schema=Schema.TRADES, dataset="XNAS") - -# JSONL format (one record per line) is also supported -json_to_dbn("trades.jsonl", "trades.dbn") -``` - -### Compression - -```julia -# Compress existing files -compress_dbn_file("input.dbn", "output.dbn.zst") - -# Batch compress daily files -compress_daily_files(Date("2024-01-01"), "data/") -``` - -### Utilities - -```julia -# Price conversions (DBN uses fixed-point arithmetic) -price_float = price_to_float(1000000) # Convert to 100.0000 -price_fixed = float_to_price(100.50) # Convert to 1005000 - -# Timestamp conversions -dt = ts_to_datetime(1609459200000000000) # Convert nanoseconds to DateTime -ts = datetime_to_ts(DateTime(2021, 1, 1)) # Convert DateTime to nanoseconds -``` - -## License - -This package is released under the [MIT License](LICENSE). - -I am not affiliated with Databento. The official implementations for -[dbn](https://github.com/databento/dbn) are distributed separately under the -[Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). +# DatabentoBinaryEncoding.jl + +Julia implementation of the Databento Binary Encoding (DBN) message encoding and storage format for normalized market data. + +**⚠️ Development Status**: This package is under active development. While core functionality is complete and tested for byte-for-byte compatibility with the official Rust implementation, the API may still evolve. Production use is possible but not yet recommended. + +For more details, read the [introduction to DBN](https://databento.com/docs/standards-and-conventions/databento-binary-encoding). + +## Features + +- ✅ Complete DBN v3 Format Support +- ✅ Efficient streaming support (read and write) +- ✅ Timestamp-paced replay (simulate a live feed at any speed) +- ✅ Zstd file compression support (read and write) +- ✅ Bidirectional format conversion (DBN ↔ JSON/Parquet/CSV) +- ✅ Byte-for-byte compatibility with official implementations +- ✅ All DBN message types (Trades, MBO, MBP, OHLCV, definitions, statistics, imbalance, status) +- ✅ Consolidated & BBO schemas (CBBO, CMBP-1, TCBBO, BBO-1s/1m) +- ✅ Live control records (SymbolMappingMsg, SystemMsg, ErrorMsg), tolerant of v3 unset-`stype` sentinels +- ✅ High-precision timestamp handling +- ✅ Fixed-point price arithmetic + +## Performance + +DatabentoBinaryEncoding.jl is optimized for high-throughput market data processing: +- **Read**: Up to 40M records/sec with near-zero-allocation callback streaming +- **Write**: 11M records/sec with optimized bulk operations +- **Type-specific readers** (`read_trades`, `read_mbo`, etc.) are 5-6x faster than generic `read_dbn()` + +**Note**: This package supports DBN v2 and v3 formats only. DBN v1 files are not supported. To convert DBN v1 files, use the official Databento CLI: +```bash +dbn version1.dbn --output version2.dbn --upgrade +``` + +## Installation + +The package is registered in the General registry: + +```julia +using Pkg +Pkg.add("DatabentoBinaryEncoding") +``` + +Or, for the latest unreleased changes, install directly from GitHub: + +```julia +using Pkg +Pkg.add(url="https://github.com/tbeason/DatabentoBinaryEncoding.jl") +``` + +## Usage + +### Reading DBN Files + +```julia +using DatabentoBinaryEncoding +# Tip: `import DatabentoBinaryEncoding as DBN` gives you the terser `DBN.foo` prefix. + +# Read entire file into memory (fastest for bulk loading) +records = read_dbn("trades.dbn") + +# Read with metadata +metadata, records = read_dbn_with_metadata("trades.dbn") + +# Optimized type-specific eager read (5-6x faster than read_dbn) +trades = read_trades("trades.dbn") +mbos = read_mbo("mbo.dbn") + +# Generic streaming for mixed-type files +for record in DBNStream("large_file.dbn.zst") + println("Trade: $(record.price) @ $(record.size)") +end + +# High-performance callback streaming (near-zero allocations!) +# Best for pure processing workloads - 40 M records/sec +total = Ref(0.0) +foreach_trade("trades.dbn") do trade + total[] += price_to_float(trade.price) +end +println("Total: $(total[])") +``` + +### Replaying DBN Files + +Re-emit a file's records in time order, paced by their timestamps, to simulate a +live feed (useful for backtesting, demos, or driving consumers that expect +real-time data): + +```julia +# Replay in real time — each record fires after the real gap to the previous one +replay_dbn("trades.dbn") do rec + println(price_to_float(rec.price)) +end + +# Replay 100x faster, paced by receive time, with overnight gaps capped at 1s +replay_dbn("mbo.dbn.zst"; speed = 100, timestamp = :ts_recv, max_sleep = 1.0) do rec + handle(rec) +end + +# Replay an already-loaded collection (e.g. from read_dbn) +records = read_dbn("trades.dbn") +replay_records(records; speed = 5) do rec + handle(rec) +end +``` + +Key options: `speed` (time-compression multiplier; `Inf` = no waiting), +`timestamp` (`:ts_event` or `:ts_recv`), and `max_sleep` (cap on any single +wait, in seconds). Compression and unknown record types are handled exactly as +in `DBNStream`. + +**Timing resolution.** DBN timestamps are nanosecond precision, but pacing +accuracy is bounded by the sleep function. `Base.sleep` resolves to roughly +1 ms on Unix and as coarse as the system timer tick (~15 ms) on Windows, so +records spaced more tightly than that arrive clumped rather than as distinct +waits. Pacing is anchored to absolute wall-clock targets, so this clumping is +local — the stream re-synchronizes and timing error does not accumulate across +the file, and records sharing a timestamp are delivered back-to-back. For +sub-millisecond fidelity (e.g. dense MBO bursts) pass `precise = true`, which +busy-waits small gaps at the cost of pinning a CPU core: + +```julia +replay_dbn("mbo.dbn"; precise = true) do rec + handle(rec) +end +``` + +### Writing DBN Files + +```julia +using DatabentoBinaryEncoding, Dates + +# Create metadata for trades +metadata = Metadata( + UInt8(3), # DBN version + "XNAS", # dataset + Schema.TRADES, # schema + datetime_to_ts(DateTime(2024, 1, 1)), # start_ts + datetime_to_ts(DateTime(2024, 1, 2)), # end_ts + UInt64(1000), # limit + SType.RAW_SYMBOL, # stype_in + SType.RAW_SYMBOL, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String, String, Int64, Int64}[] # mappings +) + +# Create trade message +trade = TradeMsg( + RecordHeader( + UInt8(sizeof(TradeMsg)), + RType.MBP_0_MSG, + UInt16(1), # publisher_id + UInt32(12345), # instrument_id + UInt64(datetime_to_ts(DateTime(2024, 1, 1, 9, 30))) + ), + float_to_price(100.50), # price + UInt32(100), # size + Action.TRADE, + Side.BID, + UInt8(0), # flags + UInt8(0), # depth + UInt64(datetime_to_ts(DateTime(2024, 1, 1, 9, 30))), # ts_recv + Int32(0), # ts_in_delta + UInt32(1) # sequence +) + +# Write to file +write_dbn("output.dbn", metadata, [trade]) + +# Write compressed file +write_dbn("output.dbn.zst", metadata, [trade]) +``` + +### Streaming Writer + +```julia +# Create streaming writer for real-time data +writer = DBNStreamWriter("live_trades.dbn", "XNAS", Schema.TRADES) + +# Write records as they arrive +for price in [100.0, 100.25, 100.50] + trade = create_trade_message(price, 100) # Your trade creation logic + write_record!(writer, trade) +end + +close_writer!(writer) +``` + +### Data Export + +```julia +# Convert to different formats +dbn_to_csv("trades.dbn", "trades.csv") +dbn_to_json("trades.dbn", "trades.json") +dbn_to_parquet("trades.dbn", "trades.parquet") # ZSTD-compressed by default + +# Convert to DataFrame for analysis +df = records_to_dataframe(records) +``` + +### Data Import + +```julia +# Convert other formats to DBN +json_to_dbn("trades.json", "trades.dbn") +parquet_to_dbn("trades.parquet", "trades.dbn", schema=Schema.TRADES, dataset="XNAS") +csv_to_dbn("trades.csv", "trades.dbn", schema=Schema.TRADES, dataset="XNAS") + +# JSONL format (one record per line) is also supported +json_to_dbn("trades.jsonl", "trades.dbn") +``` + +### Compression + +```julia +# Compress existing files +compress_dbn_file("input.dbn", "output.dbn.zst") + +# Batch compress daily files +compress_daily_files(Date("2024-01-01"), "data/") +``` + +### Utilities + +```julia +# Price conversions (DBN uses fixed-point arithmetic) +price_float = price_to_float(1000000) # Convert to 100.0000 +price_fixed = float_to_price(100.50) # Convert to 1005000 + +# Timestamp conversions +dt = ts_to_datetime(1609459200000000000) # Convert nanoseconds to DateTime +ts = datetime_to_ts(DateTime(2021, 1, 1)) # Convert DateTime to nanoseconds +``` + +## License + +This package is released under the [MIT License](LICENSE). + +I am not affiliated with Databento. The official implementations for +[dbn](https://github.com/databento/dbn) are distributed separately under the +[Apache 2.0 License](https://www.apache.org/licenses/LICENSE-2.0.html). diff --git a/benchmark/EXAMPLE_OUTPUT.md b/benchmark/EXAMPLE_OUTPUT.md index 67cc0827..33bbd532 100644 --- a/benchmark/EXAMPLE_OUTPUT.md +++ b/benchmark/EXAMPLE_OUTPUT.md @@ -1,266 +1,266 @@ -# Example Benchmark Output - -This document shows example output from the DatabentoBinaryEncoding.jl benchmark suite. - -## Throughput Benchmark Example - -``` -████████████████████████████████████████████████████████████████████ -█ █ -█ DatabentoBinaryEncoding.jl THROUGHPUT BENCHMARK SUITE █ -█ █ -████████████████████████████████████████████████████████████████████ - -Benchmark runs per test: 5 -Data directory: benchmark/data - -Started at: 2024-01-15 10:30:45 - -Found 12 test files - - -▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼ -Testing: trades.1m.dbn.zst -▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼ - -[1/3] Benchmarking full read (read_dbn)... - -====================================================================== -READ THROUGHPUT - trades.1m.dbn.zst -====================================================================== -File: trades.1m.dbn.zst -Records: 1,000,000 -Mean Time: 0.2450 ± 0.0123 seconds -File Size: 38.50 MB - ----------------------------------------------------------------------- -Throughput: - 4,081,632.65 records/second - 4,081.63 thousand records/second - 4.0816 million records/second - -Bandwidth: 157.14 MB/s -====================================================================== - - -[2/3] Benchmarking streaming read (DBNStream)... - -====================================================================== -STREAMING THROUGHPUT - trades.1m.dbn.zst -====================================================================== -File: trades.1m.dbn.zst -Records: 1,000,000 -Mean Time: 0.2812 ± 0.0089 seconds -File Size: 38.50 MB - ----------------------------------------------------------------------- -Throughput: - 3,556,224.90 records/second - 3,556.22 thousand records/second - 3.5562 million records/second - -Bandwidth: 136.89 MB/s -====================================================================== - - -[3/3] Benchmarking write (write_dbn)... - -====================================================================== -WRITE THROUGHPUT - trades.1m.dbn.zst -====================================================================== -Records: 1,000,000 -Mean Time: 0.3250 ± 0.0156 seconds -File Size: 38.50 MB - ----------------------------------------------------------------------- -Throughput: - 3,076,923.08 records/second - 3,076.92 thousand records/second - 3.0769 million records/second - -Bandwidth: 118.46 MB/s -====================================================================== - - -████████████████████████████████████████████████████████████████████ -█ █ -█ BENCHMARK SUMMARY █ -█ █ -████████████████████████████████████████████████████████████████████ - - -READ THROUGHPUT: -────────────────────────────────────────────────────────────────────── -File Records Time (s) Mrec/s -────────────────────────────────────────────────────────────────────── -trades.1k.dbn 1,000 0.001 1.1235 -trades.1k.dbn.zst 1,000 0.001 0.9876 -trades.10k.dbn 10,000 0.003 3.4521 -trades.10k.dbn.zst 10,000 0.004 2.8912 -trades.100k.dbn 100,000 0.028 3.5714 -trades.100k.dbn.zst 100,000 0.032 3.1250 -trades.1m.dbn 1,000,000 0.236 4.2373 -trades.1m.dbn.zst 1,000,000 0.245 4.0816 -trades.10m.dbn 10,000,000 2.341 4.2719 -trades.10m.dbn.zst 10,000,000 2.512 3.9809 - - -STREAM THROUGHPUT: -────────────────────────────────────────────────────────────────────── -File Records Time (s) Mrec/s -────────────────────────────────────────────────────────────────────── -trades.1k.dbn 1,000 0.001 0.9523 -trades.10k.dbn 10,000 0.004 2.5641 -trades.100k.dbn 100,000 0.032 3.1250 -trades.1m.dbn 1,000,000 0.281 3.5562 - - -WRITE THROUGHPUT: -────────────────────────────────────────────────────────────────────── -File Records Time (s) Mrec/s -────────────────────────────────────────────────────────────────────── -trades.1k.dbn 1,000 0.001 0.8928 -trades.10k.dbn 10,000 0.005 2.0000 -trades.100k.dbn 100,000 0.041 2.4390 -trades.1m.dbn 1,000,000 0.325 3.0769 - -████████████████████████████████████████████████████████████████████ - - -Completed at: 2024-01-15 10:45:23 -``` - -## BenchmarkTools Suite Example - -``` -====================================================================== -DatabentoBinaryEncoding.jl BenchmarkTools Suite -====================================================================== - -Creating benchmark suite... -Suite created with 5 groups - - read: 4 benchmarks - - stream: 2 benchmarks - - write: 3 benchmarks - - convert: 2 benchmarks - - utils: 4 benchmarks - -Running benchmarks (this may take several minutes)... -Started at: 2024-01-15 10:50:00 - - -====================================================================== -BENCHMARK RESULTS -====================================================================== - -read: -────────────────────────────────────────────────────────────────────── - small_uncompressed 0.89 ms 39.23 KB 45 allocs - small_compressed 1.12 ms 39.45 KB 52 allocs - medium_uncompressed 28.45 ms 3.81 MB 450 allocs - medium_compressed 34.23 ms 3.82 MB 465 allocs - -stream: -────────────────────────────────────────────────────────────────────── - small 1.05 ms 12.34 KB 23 allocs - medium 32.10 ms 1.23 MB 234 allocs - -write: -────────────────────────────────────────────────────────────────────── - small_uncompressed 1.23 ms 40.50 KB 67 allocs - small_compressed 2.45 ms 45.20 KB 89 allocs - medium_uncompressed 38.90 ms 3.95 MB 678 allocs - -convert: -────────────────────────────────────────────────────────────────────── - to_json 15.67 ms 89.45 KB 234 allocs - to_csv 12.34 ms 78.23 KB 189 allocs - -utils: -────────────────────────────────────────────────────────────────────── - float_to_price 12.34 ns 0.00 KB 0 allocs - price_to_float 10.23 ns 0.00 KB 0 allocs - datetime_to_ts 45.67 ns 0.00 KB 0 allocs - ts_to_datetime 52.34 ns 0.00 KB 0 allocs - -====================================================================== - -Results saved to: - - benchmark/results_20240115_105000.csv - - benchmark/results_20240115_105000.json - -Completed at: 2024-01-15 11:05:23 -``` - -## CSV Results Example - -The benchmark suite generates CSV files with detailed results: - -```csv -group,benchmark,min_time_ns,median_time_ns,mean_time_ns,max_time_ns,memory_bytes,allocs,gc_time_ns,min_time_ms,median_time_ms,memory_mb -read,small_uncompressed,890000,895000,892500,900000,40192,45,0,0.890,0.895,0.039 -read,small_compressed,1120000,1125000,1122500,1130000,40396,52,0,1.120,1.125,0.039 -read,medium_uncompressed,28450000,28500000,28475000,28600000,3997696,450,0,28.450,28.500,3.813 -read,medium_compressed,34230000,34250000,34240000,34300000,4005888,465,0,34.230,34.250,3.821 -stream,small,1050000,1055000,1052500,1060000,12636,23,0,1.050,1.055,0.012 -stream,medium,32100000,32150000,32125000,32200000,1290240,234,0,32.100,32.150,1.231 -write,small_uncompressed,1230000,1235000,1232500,1240000,41472,67,0,1.230,1.235,0.040 -write,small_compressed,2450000,2455000,2452500,2460000,46284,89,0,2.450,2.455,0.044 -write,medium_uncompressed,38900000,38950000,38925000,39000000,4143104,678,0,38.900,38.950,3.952 -convert,to_json,15670000,15700000,15685000,15720000,91596,234,0,15.670,15.700,0.087 -convert,to_csv,12340000,12360000,12350000,12380000,80107,189,0,12.340,12.360,0.076 -utils,float_to_price,12,12,12,13,0,0,0,0.000,0.000,0.000 -utils,price_to_float,10,10,10,11,0,0,0,0.000,0.000,0.000 -utils,datetime_to_ts,45,46,45,47,0,0,0,0.000,0.000,0.000 -utils,ts_to_datetime,52,53,52,54,0,0,0,0.000,0.000,0.000 -``` - -## Performance Comparison - -You can compare these results with the official Rust implementation: - -### Rust (dbn crate) -```bash -$ time dbn dump trades.1m.dbn.zst --output /dev/null -Processed 1,000,000 records - -real 0m0.198s -user 0m0.185s -sys 0m0.012s -``` -**Throughput:** ~5.05 million records/second - -### Julia (DatabentoBinaryEncoding.jl) -```julia -julia> @time read_dbn("trades.1m.dbn.zst") - 0.245000 seconds (450 allocations: 3.82 MiB) -1000000-element Vector{TradeMsg} -``` -**Throughput:** ~4.08 million records/second - -**Performance Ratio:** Julia achieves ~81% of Rust performance, which is excellent for a high-level language! - -## Interpreting Results - -### Throughput Metrics -- **> 5 Mrec/s**: Excellent (limited by memory bandwidth) -- **3-5 Mrec/s**: Good (typical for compressed data) -- **1-3 Mrec/s**: Fair (acceptable for complex operations) -- **< 1 Mrec/s**: Poor (may indicate performance issues) - -### Memory Usage -- Small files (1K-10K records): < 1 MB -- Medium files (100K records): ~4 MB -- Large files (1M records): ~40 MB -- Very large files (10M records): ~400 MB - -### Allocations -- Low allocation count (< 100): Efficient -- Medium allocation count (100-1000): Acceptable -- High allocation count (> 1000): May cause GC pressure - -### Compression Overhead -- Zstd typically reduces throughput by 20-30% -- File size reduction: 50-70% for market data -- Worth it for: storage, network transfer -- Skip for: in-memory processing, low-latency applications +# Example Benchmark Output + +This document shows example output from the DatabentoBinaryEncoding.jl benchmark suite. + +## Throughput Benchmark Example + +``` +████████████████████████████████████████████████████████████████████ +█ █ +█ DatabentoBinaryEncoding.jl THROUGHPUT BENCHMARK SUITE █ +█ █ +████████████████████████████████████████████████████████████████████ + +Benchmark runs per test: 5 +Data directory: benchmark/data + +Started at: 2024-01-15 10:30:45 + +Found 12 test files + + +▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼ +Testing: trades.1m.dbn.zst +▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼▼ + +[1/3] Benchmarking full read (read_dbn)... + +====================================================================== +READ THROUGHPUT - trades.1m.dbn.zst +====================================================================== +File: trades.1m.dbn.zst +Records: 1,000,000 +Mean Time: 0.2450 ± 0.0123 seconds +File Size: 38.50 MB + +---------------------------------------------------------------------- +Throughput: + 4,081,632.65 records/second + 4,081.63 thousand records/second + 4.0816 million records/second + +Bandwidth: 157.14 MB/s +====================================================================== + + +[2/3] Benchmarking streaming read (DBNStream)... + +====================================================================== +STREAMING THROUGHPUT - trades.1m.dbn.zst +====================================================================== +File: trades.1m.dbn.zst +Records: 1,000,000 +Mean Time: 0.2812 ± 0.0089 seconds +File Size: 38.50 MB + +---------------------------------------------------------------------- +Throughput: + 3,556,224.90 records/second + 3,556.22 thousand records/second + 3.5562 million records/second + +Bandwidth: 136.89 MB/s +====================================================================== + + +[3/3] Benchmarking write (write_dbn)... + +====================================================================== +WRITE THROUGHPUT - trades.1m.dbn.zst +====================================================================== +Records: 1,000,000 +Mean Time: 0.3250 ± 0.0156 seconds +File Size: 38.50 MB + +---------------------------------------------------------------------- +Throughput: + 3,076,923.08 records/second + 3,076.92 thousand records/second + 3.0769 million records/second + +Bandwidth: 118.46 MB/s +====================================================================== + + +████████████████████████████████████████████████████████████████████ +█ █ +█ BENCHMARK SUMMARY █ +█ █ +████████████████████████████████████████████████████████████████████ + + +READ THROUGHPUT: +────────────────────────────────────────────────────────────────────── +File Records Time (s) Mrec/s +────────────────────────────────────────────────────────────────────── +trades.1k.dbn 1,000 0.001 1.1235 +trades.1k.dbn.zst 1,000 0.001 0.9876 +trades.10k.dbn 10,000 0.003 3.4521 +trades.10k.dbn.zst 10,000 0.004 2.8912 +trades.100k.dbn 100,000 0.028 3.5714 +trades.100k.dbn.zst 100,000 0.032 3.1250 +trades.1m.dbn 1,000,000 0.236 4.2373 +trades.1m.dbn.zst 1,000,000 0.245 4.0816 +trades.10m.dbn 10,000,000 2.341 4.2719 +trades.10m.dbn.zst 10,000,000 2.512 3.9809 + + +STREAM THROUGHPUT: +────────────────────────────────────────────────────────────────────── +File Records Time (s) Mrec/s +────────────────────────────────────────────────────────────────────── +trades.1k.dbn 1,000 0.001 0.9523 +trades.10k.dbn 10,000 0.004 2.5641 +trades.100k.dbn 100,000 0.032 3.1250 +trades.1m.dbn 1,000,000 0.281 3.5562 + + +WRITE THROUGHPUT: +────────────────────────────────────────────────────────────────────── +File Records Time (s) Mrec/s +────────────────────────────────────────────────────────────────────── +trades.1k.dbn 1,000 0.001 0.8928 +trades.10k.dbn 10,000 0.005 2.0000 +trades.100k.dbn 100,000 0.041 2.4390 +trades.1m.dbn 1,000,000 0.325 3.0769 + +████████████████████████████████████████████████████████████████████ + + +Completed at: 2024-01-15 10:45:23 +``` + +## BenchmarkTools Suite Example + +``` +====================================================================== +DatabentoBinaryEncoding.jl BenchmarkTools Suite +====================================================================== + +Creating benchmark suite... +Suite created with 5 groups + - read: 4 benchmarks + - stream: 2 benchmarks + - write: 3 benchmarks + - convert: 2 benchmarks + - utils: 4 benchmarks + +Running benchmarks (this may take several minutes)... +Started at: 2024-01-15 10:50:00 + + +====================================================================== +BENCHMARK RESULTS +====================================================================== + +read: +────────────────────────────────────────────────────────────────────── + small_uncompressed 0.89 ms 39.23 KB 45 allocs + small_compressed 1.12 ms 39.45 KB 52 allocs + medium_uncompressed 28.45 ms 3.81 MB 450 allocs + medium_compressed 34.23 ms 3.82 MB 465 allocs + +stream: +────────────────────────────────────────────────────────────────────── + small 1.05 ms 12.34 KB 23 allocs + medium 32.10 ms 1.23 MB 234 allocs + +write: +────────────────────────────────────────────────────────────────────── + small_uncompressed 1.23 ms 40.50 KB 67 allocs + small_compressed 2.45 ms 45.20 KB 89 allocs + medium_uncompressed 38.90 ms 3.95 MB 678 allocs + +convert: +────────────────────────────────────────────────────────────────────── + to_json 15.67 ms 89.45 KB 234 allocs + to_csv 12.34 ms 78.23 KB 189 allocs + +utils: +────────────────────────────────────────────────────────────────────── + float_to_price 12.34 ns 0.00 KB 0 allocs + price_to_float 10.23 ns 0.00 KB 0 allocs + datetime_to_ts 45.67 ns 0.00 KB 0 allocs + ts_to_datetime 52.34 ns 0.00 KB 0 allocs + +====================================================================== + +Results saved to: + - benchmark/results_20240115_105000.csv + - benchmark/results_20240115_105000.json + +Completed at: 2024-01-15 11:05:23 +``` + +## CSV Results Example + +The benchmark suite generates CSV files with detailed results: + +```csv +group,benchmark,min_time_ns,median_time_ns,mean_time_ns,max_time_ns,memory_bytes,allocs,gc_time_ns,min_time_ms,median_time_ms,memory_mb +read,small_uncompressed,890000,895000,892500,900000,40192,45,0,0.890,0.895,0.039 +read,small_compressed,1120000,1125000,1122500,1130000,40396,52,0,1.120,1.125,0.039 +read,medium_uncompressed,28450000,28500000,28475000,28600000,3997696,450,0,28.450,28.500,3.813 +read,medium_compressed,34230000,34250000,34240000,34300000,4005888,465,0,34.230,34.250,3.821 +stream,small,1050000,1055000,1052500,1060000,12636,23,0,1.050,1.055,0.012 +stream,medium,32100000,32150000,32125000,32200000,1290240,234,0,32.100,32.150,1.231 +write,small_uncompressed,1230000,1235000,1232500,1240000,41472,67,0,1.230,1.235,0.040 +write,small_compressed,2450000,2455000,2452500,2460000,46284,89,0,2.450,2.455,0.044 +write,medium_uncompressed,38900000,38950000,38925000,39000000,4143104,678,0,38.900,38.950,3.952 +convert,to_json,15670000,15700000,15685000,15720000,91596,234,0,15.670,15.700,0.087 +convert,to_csv,12340000,12360000,12350000,12380000,80107,189,0,12.340,12.360,0.076 +utils,float_to_price,12,12,12,13,0,0,0,0.000,0.000,0.000 +utils,price_to_float,10,10,10,11,0,0,0,0.000,0.000,0.000 +utils,datetime_to_ts,45,46,45,47,0,0,0,0.000,0.000,0.000 +utils,ts_to_datetime,52,53,52,54,0,0,0,0.000,0.000,0.000 +``` + +## Performance Comparison + +You can compare these results with the official Rust implementation: + +### Rust (dbn crate) +```bash +$ time dbn dump trades.1m.dbn.zst --output /dev/null +Processed 1,000,000 records + +real 0m0.198s +user 0m0.185s +sys 0m0.012s +``` +**Throughput:** ~5.05 million records/second + +### Julia (DatabentoBinaryEncoding.jl) +```julia +julia> @time read_dbn("trades.1m.dbn.zst") + 0.245000 seconds (450 allocations: 3.82 MiB) +1000000-element Vector{TradeMsg} +``` +**Throughput:** ~4.08 million records/second + +**Performance Ratio:** Julia achieves ~81% of Rust performance, which is excellent for a high-level language! + +## Interpreting Results + +### Throughput Metrics +- **> 5 Mrec/s**: Excellent (limited by memory bandwidth) +- **3-5 Mrec/s**: Good (typical for compressed data) +- **1-3 Mrec/s**: Fair (acceptable for complex operations) +- **< 1 Mrec/s**: Poor (may indicate performance issues) + +### Memory Usage +- Small files (1K-10K records): < 1 MB +- Medium files (100K records): ~4 MB +- Large files (1M records): ~40 MB +- Very large files (10M records): ~400 MB + +### Allocations +- Low allocation count (< 100): Efficient +- Medium allocation count (100-1000): Acceptable +- High allocation count (> 1000): May cause GC pressure + +### Compression Overhead +- Zstd typically reduces throughput by 20-30% +- File size reduction: 50-70% for market data +- Worth it for: storage, network transfer +- Skip for: in-memory processing, low-latency applications diff --git a/benchmark/README.md b/benchmark/README.md index 781e9e45..0e981247 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,340 +1,340 @@ -# DatabentoBinaryEncoding.jl Benchmark Suite - -Comprehensive performance benchmarking tools for DatabentoBinaryEncoding.jl, measuring throughput (records/second), latency, memory usage, and bandwidth. - -## Quick Start - -```bash -# Generate test data and run all benchmarks -julia benchmark/run_benchmarks.jl --generate-data - -# Quick benchmark with small datasets -julia benchmark/run_benchmarks.jl --generate-data --quick - -# Run only throughput benchmarks (faster) -julia benchmark/run_benchmarks.jl --throughput-only - -# Run only detailed BenchmarkTools suite -julia benchmark/run_benchmarks.jl --suite-only -``` - -## Components - -### 1. Data Generator (`generate_test_data.jl`) - -Generates realistic test data files of various sizes for benchmarking. - -**Sizes:** -- `1k` - 1,000 records (~40 KB) -- `10k` - 10,000 records (~400 KB) -- `100k` - 100,000 records (~4 MB) -- `1m` - 1,000,000 records (~40 MB) -- `10m` - 10,000,000 records (~400 MB) - -**Message Types:** -- `trades` - Trade execution messages -- `mbo` - Market-by-order messages -- `ohlcv` - OHLCV bar data - -**Formats:** -- Uncompressed (`.dbn`) -- Zstd compressed (`.dbn.zst`) - -**Usage:** -```bash -# Generate all test files -julia benchmark/generate_test_data.jl - -# Generate to custom directory -julia -e 'include("benchmark/generate_test_data.jl"); generate_test_files("my_data/")' -``` - -### 2. Throughput Benchmarks (`throughput.jl`) - -Measures real-world throughput in records/second and MB/s bandwidth. - -**Metrics:** -- Records per second -- Million records per second -- Read/write bandwidth (MB/s) -- Mean time and standard deviation -- File sizes - -**Usage:** -```bash -# Run on default data directory -julia benchmark/throughput.jl - -# Custom data directory -julia benchmark/throughput.jl benchmark/data - -# Custom number of runs -julia benchmark/throughput.jl benchmark/data 10 -``` - -**Example Output:** -``` -====================================================================== -READ THROUGHPUT - trades.1m.dbn.zst -====================================================================== -File: trades.1m.dbn.zst -Records: 1,000,000 -Mean Time: 0.2450 ± 0.0123 seconds -File Size: 38.50 MB - ----------------------------------------------------------------------- -Throughput: - 4,081,632.65 records/second - 4,081.63 thousand records/second - 4.0816 million records/second - -Bandwidth: 157.14 MB/s -====================================================================== -``` - -### 3. BenchmarkTools Suite (`benchmarks.jl`) - -Detailed statistical benchmarks using BenchmarkTools.jl for precise timing and memory profiling. - -**Benchmark Groups:** -- `read` - File reading operations -- `stream` - Streaming operations -- `write` - File writing operations -- `convert` - Format conversion -- `utils` - Price and timestamp utilities - -**Usage:** -```bash -# Run full suite -julia benchmark/benchmarks.jl - -# Run programmatically -julia -e 'include("benchmark/benchmarks.jl"); run_benchmark_suite()' -``` - -**Compare Results:** -```julia -using BenchmarkTools -include("benchmark/benchmarks.jl") - -# Compare two benchmark runs -compare_benchmarks("benchmark/results_20240101_120000.json", - "benchmark/results_20240101_130000.json") -``` - -### 4. Main Runner (`run_benchmarks.jl`) - -Orchestrates the complete benchmark suite with options for different scenarios. - -**Options:** -- `--generate-data` - Generate test data before benchmarking -- `--throughput-only` - Run only throughput benchmarks (faster) -- `--suite-only` - Run only BenchmarkTools suite -- `--quick` - Quick benchmark with small datasets only -- `--data-dir DIR` - Use custom data directory -- `--runs N` - Number of runs for throughput benchmarks (default: 5) - -## Benchmark Results - -Results are automatically saved to: - -1. **CSV Files** (`benchmark/results_*.csv`) - - Timestamped results for analysis - - Easy to import into spreadsheets or Python/R - - Columns: group, benchmark, time, memory, allocations - -2. **JSON Files** (`benchmark/results_*.json`) - - Full BenchmarkTools results - - Can be loaded for comparison with `BenchmarkTools.load()` - -## Performance Targets - -Based on modern hardware (SSD, 16GB RAM): - -| Operation | Target Throughput | Notes | -|-----------|------------------|-------| -| Read uncompressed | > 5M records/sec | Limited by memory bandwidth | -| Read compressed | > 2M records/sec | Limited by decompression | -| Write uncompressed | > 4M records/sec | Limited by disk I/O | -| Write compressed | > 1M records/sec | Limited by compression | -| Streaming | > 3M records/sec | Lower memory overhead | - -## Example Workflow - -### 1. Initial Benchmarking - -```bash -# Generate data and run complete suite -julia benchmark/run_benchmarks.jl --generate-data -``` - -### 2. Quick Regression Testing - -```bash -# Quick check after code changes -julia benchmark/run_benchmarks.jl --quick --throughput-only -``` - -### 3. Detailed Performance Analysis - -```bash -# Run detailed suite and analyze -julia benchmark/run_benchmarks.jl --suite-only -``` - -### 4. Custom Analysis - -```julia -using DatabentoBinaryEncoding -include("benchmark/throughput.jl") - -# Benchmark specific file -result = benchmark_read_throughput("my_file.dbn.zst", runs=10) -println("Throughput: $(result.throughput_mrecs_per_sec) Mrec/s") - -# Custom benchmark -file = "my_data.dbn" -@time records = read_dbn(file) -println("Read $(length(records)) records") -``` - -## Comparing Performance - -### Against Other Implementations - -To compare with the official Rust implementation: - -```bash -# Benchmark Rust version -time dbn dump test_data.dbn --output /dev/null - -# Benchmark Julia version -julia -e 'using DatabentoBinaryEncoding; @time read_dbn("test_data.dbn")' -``` - -### Across Versions - -```julia -# Run benchmarks before changes -julia benchmark/run_benchmarks.jl -# Note the timestamp of results file - -# Make code changes -# ... - -# Run benchmarks after changes -julia benchmark/run_benchmarks.jl - -# Compare -using BenchmarkTools -include("benchmark/benchmarks.jl") -compare_benchmarks("benchmark/results_BEFORE.json", - "benchmark/results_AFTER.json") -``` - -## CI Integration - -To track performance over time, add to your CI pipeline: - -```yaml -# .github/workflows/benchmarks.yml -name: Benchmarks - -on: [push, pull_request] - -jobs: - benchmark: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 - - name: Run benchmarks - run: julia benchmark/run_benchmarks.jl --generate-data --quick - - uses: actions/upload-artifact@v2 - with: - name: benchmark-results - path: benchmark/results_*.csv -``` - -## Profiling - -For detailed profiling of hot spots: - -```julia -using Profile, ProfileView -using DatabentoBinaryEncoding - -# Profile reading -file = "benchmark/data/trades.1m.dbn" -@profile for i in 1:100 - read_dbn(file) -end - -# View results -ProfileView.view() - -# Or generate flamegraph -using FlameGraphs -g = flamegraph() -``` - -## Memory Profiling - -```julia -using DatabentoBinaryEncoding - -# Track allocations -file = "benchmark/data/trades.1m.dbn" -@time read_dbn(file) # Warm up - -# Detailed allocation tracking -@allocated read_dbn(file) - -# Use --track-allocation=user when starting Julia for line-by-line profiling -``` - -## Tips for Accurate Benchmarking - -1. **Warm up the JIT compiler** - Run operations at least once before benchmarking -2. **Clear caches** - Run `GC.gc()` between benchmarks -3. **Use realistic data** - Benchmark with production-sized datasets -4. **Multiple runs** - Average over multiple runs (default: 5) -5. **Minimize background processes** - Close unnecessary applications -6. **Consistent environment** - Use same hardware/OS for comparisons - -## Troubleshooting - -### "No test data found" - -Run with `--generate-data`: -```bash -julia benchmark/run_benchmarks.jl --generate-data -``` - -### Out of Memory - -Use smaller datasets or streaming operations: -```bash -julia benchmark/run_benchmarks.jl --quick -``` - -### Slow Benchmarks - -Use throughput-only mode for faster results: -```bash -julia benchmark/run_benchmarks.jl --throughput-only -``` - -## Contributing - -When adding new features to DatabentoBinaryEncoding.jl: - -1. Add relevant benchmarks to `benchmarks.jl` -2. Run `julia benchmark/run_benchmarks.jl --quick` to verify no regressions -3. Include performance results in PR description if significant changes - -## References - -- [BenchmarkTools.jl Documentation](https://github.com/JuliaCI/BenchmarkTools.jl) -- [Julia Performance Tips](https://docs.julialang.org/en/v1/manual/performance-tips/) -- [DBN Format Specification](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) +# DatabentoBinaryEncoding.jl Benchmark Suite + +Comprehensive performance benchmarking tools for DatabentoBinaryEncoding.jl, measuring throughput (records/second), latency, memory usage, and bandwidth. + +## Quick Start + +```bash +# Generate test data and run all benchmarks +julia benchmark/run_benchmarks.jl --generate-data + +# Quick benchmark with small datasets +julia benchmark/run_benchmarks.jl --generate-data --quick + +# Run only throughput benchmarks (faster) +julia benchmark/run_benchmarks.jl --throughput-only + +# Run only detailed BenchmarkTools suite +julia benchmark/run_benchmarks.jl --suite-only +``` + +## Components + +### 1. Data Generator (`generate_test_data.jl`) + +Generates realistic test data files of various sizes for benchmarking. + +**Sizes:** +- `1k` - 1,000 records (~40 KB) +- `10k` - 10,000 records (~400 KB) +- `100k` - 100,000 records (~4 MB) +- `1m` - 1,000,000 records (~40 MB) +- `10m` - 10,000,000 records (~400 MB) + +**Message Types:** +- `trades` - Trade execution messages +- `mbo` - Market-by-order messages +- `ohlcv` - OHLCV bar data + +**Formats:** +- Uncompressed (`.dbn`) +- Zstd compressed (`.dbn.zst`) + +**Usage:** +```bash +# Generate all test files +julia benchmark/generate_test_data.jl + +# Generate to custom directory +julia -e 'include("benchmark/generate_test_data.jl"); generate_test_files("my_data/")' +``` + +### 2. Throughput Benchmarks (`throughput.jl`) + +Measures real-world throughput in records/second and MB/s bandwidth. + +**Metrics:** +- Records per second +- Million records per second +- Read/write bandwidth (MB/s) +- Mean time and standard deviation +- File sizes + +**Usage:** +```bash +# Run on default data directory +julia benchmark/throughput.jl + +# Custom data directory +julia benchmark/throughput.jl benchmark/data + +# Custom number of runs +julia benchmark/throughput.jl benchmark/data 10 +``` + +**Example Output:** +``` +====================================================================== +READ THROUGHPUT - trades.1m.dbn.zst +====================================================================== +File: trades.1m.dbn.zst +Records: 1,000,000 +Mean Time: 0.2450 ± 0.0123 seconds +File Size: 38.50 MB + +---------------------------------------------------------------------- +Throughput: + 4,081,632.65 records/second + 4,081.63 thousand records/second + 4.0816 million records/second + +Bandwidth: 157.14 MB/s +====================================================================== +``` + +### 3. BenchmarkTools Suite (`benchmarks.jl`) + +Detailed statistical benchmarks using BenchmarkTools.jl for precise timing and memory profiling. + +**Benchmark Groups:** +- `read` - File reading operations +- `stream` - Streaming operations +- `write` - File writing operations +- `convert` - Format conversion +- `utils` - Price and timestamp utilities + +**Usage:** +```bash +# Run full suite +julia benchmark/benchmarks.jl + +# Run programmatically +julia -e 'include("benchmark/benchmarks.jl"); run_benchmark_suite()' +``` + +**Compare Results:** +```julia +using BenchmarkTools +include("benchmark/benchmarks.jl") + +# Compare two benchmark runs +compare_benchmarks("benchmark/results_20240101_120000.json", + "benchmark/results_20240101_130000.json") +``` + +### 4. Main Runner (`run_benchmarks.jl`) + +Orchestrates the complete benchmark suite with options for different scenarios. + +**Options:** +- `--generate-data` - Generate test data before benchmarking +- `--throughput-only` - Run only throughput benchmarks (faster) +- `--suite-only` - Run only BenchmarkTools suite +- `--quick` - Quick benchmark with small datasets only +- `--data-dir DIR` - Use custom data directory +- `--runs N` - Number of runs for throughput benchmarks (default: 5) + +## Benchmark Results + +Results are automatically saved to: + +1. **CSV Files** (`benchmark/results_*.csv`) + - Timestamped results for analysis + - Easy to import into spreadsheets or Python/R + - Columns: group, benchmark, time, memory, allocations + +2. **JSON Files** (`benchmark/results_*.json`) + - Full BenchmarkTools results + - Can be loaded for comparison with `BenchmarkTools.load()` + +## Performance Targets + +Based on modern hardware (SSD, 16GB RAM): + +| Operation | Target Throughput | Notes | +|-----------|------------------|-------| +| Read uncompressed | > 5M records/sec | Limited by memory bandwidth | +| Read compressed | > 2M records/sec | Limited by decompression | +| Write uncompressed | > 4M records/sec | Limited by disk I/O | +| Write compressed | > 1M records/sec | Limited by compression | +| Streaming | > 3M records/sec | Lower memory overhead | + +## Example Workflow + +### 1. Initial Benchmarking + +```bash +# Generate data and run complete suite +julia benchmark/run_benchmarks.jl --generate-data +``` + +### 2. Quick Regression Testing + +```bash +# Quick check after code changes +julia benchmark/run_benchmarks.jl --quick --throughput-only +``` + +### 3. Detailed Performance Analysis + +```bash +# Run detailed suite and analyze +julia benchmark/run_benchmarks.jl --suite-only +``` + +### 4. Custom Analysis + +```julia +using DatabentoBinaryEncoding +include("benchmark/throughput.jl") + +# Benchmark specific file +result = benchmark_read_throughput("my_file.dbn.zst", runs=10) +println("Throughput: $(result.throughput_mrecs_per_sec) Mrec/s") + +# Custom benchmark +file = "my_data.dbn" +@time records = read_dbn(file) +println("Read $(length(records)) records") +``` + +## Comparing Performance + +### Against Other Implementations + +To compare with the official Rust implementation: + +```bash +# Benchmark Rust version +time dbn dump test_data.dbn --output /dev/null + +# Benchmark Julia version +julia -e 'using DatabentoBinaryEncoding; @time read_dbn("test_data.dbn")' +``` + +### Across Versions + +```julia +# Run benchmarks before changes +julia benchmark/run_benchmarks.jl +# Note the timestamp of results file + +# Make code changes +# ... + +# Run benchmarks after changes +julia benchmark/run_benchmarks.jl + +# Compare +using BenchmarkTools +include("benchmark/benchmarks.jl") +compare_benchmarks("benchmark/results_BEFORE.json", + "benchmark/results_AFTER.json") +``` + +## CI Integration + +To track performance over time, add to your CI pipeline: + +```yaml +# .github/workflows/benchmarks.yml +name: Benchmarks + +on: [push, pull_request] + +jobs: + benchmark: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + - name: Run benchmarks + run: julia benchmark/run_benchmarks.jl --generate-data --quick + - uses: actions/upload-artifact@v2 + with: + name: benchmark-results + path: benchmark/results_*.csv +``` + +## Profiling + +For detailed profiling of hot spots: + +```julia +using Profile, ProfileView +using DatabentoBinaryEncoding + +# Profile reading +file = "benchmark/data/trades.1m.dbn" +@profile for i in 1:100 + read_dbn(file) +end + +# View results +ProfileView.view() + +# Or generate flamegraph +using FlameGraphs +g = flamegraph() +``` + +## Memory Profiling + +```julia +using DatabentoBinaryEncoding + +# Track allocations +file = "benchmark/data/trades.1m.dbn" +@time read_dbn(file) # Warm up + +# Detailed allocation tracking +@allocated read_dbn(file) + +# Use --track-allocation=user when starting Julia for line-by-line profiling +``` + +## Tips for Accurate Benchmarking + +1. **Warm up the JIT compiler** - Run operations at least once before benchmarking +2. **Clear caches** - Run `GC.gc()` between benchmarks +3. **Use realistic data** - Benchmark with production-sized datasets +4. **Multiple runs** - Average over multiple runs (default: 5) +5. **Minimize background processes** - Close unnecessary applications +6. **Consistent environment** - Use same hardware/OS for comparisons + +## Troubleshooting + +### "No test data found" + +Run with `--generate-data`: +```bash +julia benchmark/run_benchmarks.jl --generate-data +``` + +### Out of Memory + +Use smaller datasets or streaming operations: +```bash +julia benchmark/run_benchmarks.jl --quick +``` + +### Slow Benchmarks + +Use throughput-only mode for faster results: +```bash +julia benchmark/run_benchmarks.jl --throughput-only +``` + +## Contributing + +When adding new features to DatabentoBinaryEncoding.jl: + +1. Add relevant benchmarks to `benchmarks.jl` +2. Run `julia benchmark/run_benchmarks.jl --quick` to verify no regressions +3. Include performance results in PR description if significant changes + +## References + +- [BenchmarkTools.jl Documentation](https://github.com/JuliaCI/BenchmarkTools.jl) +- [Julia Performance Tips](https://docs.julialang.org/en/v1/manual/performance-tips/) +- [DBN Format Specification](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index ed269eec..1cedbbde 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,335 +1,335 @@ -""" - benchmarks.jl - -Comprehensive benchmark suite using BenchmarkTools for detailed performance analysis. - -This suite provides statistical distributions of performance metrics including -min/median/max times, memory allocations, and garbage collection overhead. -""" - -using BenchmarkTools +""" + benchmarks.jl + +Comprehensive benchmark suite using BenchmarkTools for detailed performance analysis. + +This suite provides statistical distributions of performance metrics including +min/median/max times, memory allocations, and garbage collection overhead. +""" + +using BenchmarkTools using DatabentoBinaryEncoding import DatabentoBinaryEncoding as DBN -using DataFrames -using CSV -using Printf -using Dates - -""" - create_benchmark_suite(data_dir="benchmark/data") - -Create a BenchmarkTools benchmark suite for DBN.jl operations. - -The suite includes: -- Read operations (full and streaming) -- Write operations (to memory and disk) -- Compression operations -- Format conversion operations -- Price and timestamp utilities -""" -function create_benchmark_suite(data_dir="benchmark/data") - suite = BenchmarkGroup() - - # Check if test data exists - if !isdir(data_dir) - @warn "Test data directory not found: $data_dir. Skipping file-based benchmarks." - return suite - end - - # Find test files of different sizes - small_file = nothing - medium_file = nothing - large_file = nothing - - for file in readdir(data_dir, join=true) - if contains(basename(file), "1k.dbn") && !endswith(file, ".zst") - small_file = file - elseif contains(basename(file), "100k.dbn") && !endswith(file, ".zst") - medium_file = file - elseif contains(basename(file), "1m.dbn") && !endswith(file, ".zst") - large_file = file - end - end - - small_zst = isnothing(small_file) ? nothing : small_file * ".zst" - medium_zst = isnothing(medium_file) ? nothing : medium_file * ".zst" - - # === READ BENCHMARKS === - suite["read"] = BenchmarkGroup() - - if !isnothing(small_file) && isfile(small_file) - suite["read"]["small_uncompressed"] = @benchmarkable read_dbn($small_file) - end - - if !isnothing(small_zst) && isfile(small_zst) - suite["read"]["small_compressed"] = @benchmarkable read_dbn($small_zst) - end - - if !isnothing(medium_file) && isfile(medium_file) - suite["read"]["medium_uncompressed"] = @benchmarkable read_dbn($medium_file) - end - - if !isnothing(medium_zst) && isfile(medium_zst) - suite["read"]["medium_compressed"] = @benchmarkable read_dbn($medium_zst) - end - - # === STREAMING BENCHMARKS === - suite["stream"] = BenchmarkGroup() - - if !isnothing(small_file) && isfile(small_file) - suite["stream"]["small"] = @benchmarkable begin - count = 0 - for record in DBNStream($small_file) - count += 1 - end - count - end - end - - if !isnothing(medium_file) && isfile(medium_file) - suite["stream"]["medium"] = @benchmarkable begin - count = 0 - for record in DBNStream($medium_file) - count += 1 - end - count - end - end - - # === WRITE BENCHMARKS === - suite["write"] = BenchmarkGroup() - - if !isnothing(small_file) && isfile(small_file) - metadata, records = read_dbn_with_metadata(small_file) - tmpfile = tempname() - - suite["write"]["small_uncompressed"] = @benchmarkable write_dbn($tmpfile, $metadata, $records) setup=(GC.gc()) teardown=(rm($tmpfile, force=true)) - - tmpfile_zst = tmpfile * ".zst" - suite["write"]["small_compressed"] = @benchmarkable write_dbn($tmpfile_zst, $metadata, $records) setup=(GC.gc()) teardown=(rm($tmpfile_zst, force=true)) - end - - if !isnothing(medium_file) && isfile(medium_file) - metadata, records = read_dbn_with_metadata(medium_file) - tmpfile = tempname() - - suite["write"]["medium_uncompressed"] = @benchmarkable write_dbn($tmpfile, $metadata, $records) setup=(GC.gc()) teardown=(rm($tmpfile, force=true)) - end - - # === CONVERSION BENCHMARKS === - suite["convert"] = BenchmarkGroup() - - if !isnothing(small_file) && isfile(small_file) - tmpjson = tempname() * ".json" - tmpcsv = tempname() * ".csv" - - suite["convert"]["to_json"] = @benchmarkable dbn_to_json($small_file, $tmpjson) setup=(GC.gc()) teardown=(rm($tmpjson, force=true)) - suite["convert"]["to_csv"] = @benchmarkable dbn_to_csv($small_file, $tmpcsv) setup=(GC.gc()) teardown=(rm($tmpcsv, force=true)) - end - - # === UTILITY BENCHMARKS === - suite["utils"] = BenchmarkGroup() - - # Price conversions - suite["utils"]["float_to_price"] = @benchmarkable float_to_price(100.50) - suite["utils"]["price_to_float"] = @benchmarkable price_to_float(1005000000) - - # Timestamp conversions - dt = DateTime(2024, 1, 1, 9, 30) - ts = datetime_to_ts(dt) - suite["utils"]["datetime_to_ts"] = @benchmarkable datetime_to_ts($dt) - suite["utils"]["ts_to_datetime"] = @benchmarkable ts_to_datetime($ts) - - return suite -end - -""" - run_benchmark_suite(data_dir="benchmark/data"; save_results=true) - -Run the complete benchmark suite and optionally save results. - -# Arguments -- `data_dir`: Directory containing test data files -- `save_results`: Whether to save results to CSV files (default: true) - -# Returns -BenchmarkTools results object -""" -function run_benchmark_suite(data_dir="benchmark/data"; save_results=true) - println("\n" * "="^70) - println("DBN.jl BenchmarkTools Suite") - println("="^70) - println("\nCreating benchmark suite...") - - suite = create_benchmark_suite(data_dir) - - if isempty(suite) - error("No benchmarks in suite. Check that test data exists in $data_dir") - end - - println("Suite created with $(length(keys(suite))) groups") - for group in keys(suite) - println(" - $group: $(length(keys(suite[group]))) benchmarks") - end - - println("\nRunning benchmarks (this may take several minutes)...") - println("Started at: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))") - - results = run(suite, verbose=true) - - println("\nCompleted at: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))") - - # Print results - print_benchmark_results(results) - - # Save results if requested - if save_results - timestamp = Dates.format(now(), "yyyymmdd_HHMMSS") - csv_file = "benchmark/results_$(timestamp).csv" - json_file = "benchmark/results_$(timestamp).json" - - save_results_to_csv(results, csv_file) - println("\nResults saved to:") - println(" - $csv_file") - - # Save raw BenchmarkTools results - BenchmarkTools.save(json_file, results) - println(" - $json_file") - end - - return results -end - -""" - print_benchmark_results(results) - -Pretty-print benchmark results in a readable format. -""" -function print_benchmark_results(results) - println("\n\n" * "="^70) - println("BENCHMARK RESULTS") - println("="^70) - - for group_name in sort(collect(keys(results))) - group = results[group_name] - println("\n" * group_name * ":") - println("─"^70) - - for bench_name in sort(collect(keys(group))) - bench_result = group[bench_name] - print_single_result(bench_name, bench_result) - end - end - - println("\n" * "="^70) -end - -""" - print_single_result(name, result) - -Print a single benchmark result. -""" -function print_single_result(name, result) - trial = minimum(result) - - @printf " %-25s" name - - # Time - time_ns = time(trial) - if time_ns < 1_000 - @printf " %8.2f ns" time_ns - elseif time_ns < 1_000_000 - @printf " %8.2f μs" (time_ns / 1_000) - elseif time_ns < 1_000_000_000 - @printf " %8.2f ms" (time_ns / 1_000_000) - else - @printf " %8.2f s " (time_ns / 1_000_000_000) - end - - # Memory - mem_bytes = memory(trial) - if mem_bytes < 1024 - @printf " %8d B" mem_bytes - elseif mem_bytes < 1024^2 - @printf " %8.2f KB" (mem_bytes / 1024) - elseif mem_bytes < 1024^3 - @printf " %8.2f MB" (mem_bytes / 1024^2) - else - @printf " %8.2f GB" (mem_bytes / 1024^3) - end - - # Allocations - @printf " %6d allocs" allocs(trial) - - println() -end - -""" - save_results_to_csv(results, filename) - -Save benchmark results to a CSV file for further analysis. -""" -function save_results_to_csv(results, filename) - rows = [] - - for group_name in keys(results) - group = results[group_name] - for bench_name in keys(group) - bench_result = group[bench_name] - trial = minimum(bench_result) - - push!(rows, ( - group = String(group_name), - benchmark = String(bench_name), - min_time_ns = time(trial), - median_time_ns = median(bench_result).time, - mean_time_ns = mean(bench_result).time, - max_time_ns = maximum(bench_result).time, - memory_bytes = memory(trial), - allocs = allocs(trial), - gc_time_ns = gctime(trial) - )) - end - end - - df = DataFrame(rows) - - # Add computed columns - df.min_time_ms = df.min_time_ns ./ 1e6 - df.median_time_ms = df.median_time_ns ./ 1e6 - df.memory_mb = df.memory_bytes ./ 1024^2 - - mkpath(dirname(filename)) - CSV.write(filename, df) -end - -""" - compare_benchmarks(file1::String, file2::String) - -Compare two benchmark result files to track performance changes. - -# Arguments -- `file1`: Path to first benchmark results JSON file (baseline) -- `file2`: Path to second benchmark results JSON file (current) -""" -function compare_benchmarks(file1::String, file2::String) - baseline = BenchmarkTools.load(file1)[1] - current = BenchmarkTools.load(file2)[1] - - println("\n" * "="^70) - println("BENCHMARK COMPARISON") - println("="^70) - println("Baseline: $file1") - println("Current: $file2") - println("="^70) - - judge_results = judge(current, baseline) - - println(judge_results) -end - -# Run if executed as script -if abspath(PROGRAM_FILE) == @__FILE__ - data_dir = length(ARGS) >= 1 ? ARGS[1] : "benchmark/data" - run_benchmark_suite(data_dir) -end +using DataFrames +using CSV +using Printf +using Dates + +""" + create_benchmark_suite(data_dir="benchmark/data") + +Create a BenchmarkTools benchmark suite for DBN.jl operations. + +The suite includes: +- Read operations (full and streaming) +- Write operations (to memory and disk) +- Compression operations +- Format conversion operations +- Price and timestamp utilities +""" +function create_benchmark_suite(data_dir="benchmark/data") + suite = BenchmarkGroup() + + # Check if test data exists + if !isdir(data_dir) + @warn "Test data directory not found: $data_dir. Skipping file-based benchmarks." + return suite + end + + # Find test files of different sizes + small_file = nothing + medium_file = nothing + large_file = nothing + + for file in readdir(data_dir, join=true) + if contains(basename(file), "1k.dbn") && !endswith(file, ".zst") + small_file = file + elseif contains(basename(file), "100k.dbn") && !endswith(file, ".zst") + medium_file = file + elseif contains(basename(file), "1m.dbn") && !endswith(file, ".zst") + large_file = file + end + end + + small_zst = isnothing(small_file) ? nothing : small_file * ".zst" + medium_zst = isnothing(medium_file) ? nothing : medium_file * ".zst" + + # === READ BENCHMARKS === + suite["read"] = BenchmarkGroup() + + if !isnothing(small_file) && isfile(small_file) + suite["read"]["small_uncompressed"] = @benchmarkable read_dbn($small_file) + end + + if !isnothing(small_zst) && isfile(small_zst) + suite["read"]["small_compressed"] = @benchmarkable read_dbn($small_zst) + end + + if !isnothing(medium_file) && isfile(medium_file) + suite["read"]["medium_uncompressed"] = @benchmarkable read_dbn($medium_file) + end + + if !isnothing(medium_zst) && isfile(medium_zst) + suite["read"]["medium_compressed"] = @benchmarkable read_dbn($medium_zst) + end + + # === STREAMING BENCHMARKS === + suite["stream"] = BenchmarkGroup() + + if !isnothing(small_file) && isfile(small_file) + suite["stream"]["small"] = @benchmarkable begin + count = 0 + for record in DBNStream($small_file) + count += 1 + end + count + end + end + + if !isnothing(medium_file) && isfile(medium_file) + suite["stream"]["medium"] = @benchmarkable begin + count = 0 + for record in DBNStream($medium_file) + count += 1 + end + count + end + end + + # === WRITE BENCHMARKS === + suite["write"] = BenchmarkGroup() + + if !isnothing(small_file) && isfile(small_file) + metadata, records = read_dbn_with_metadata(small_file) + tmpfile = tempname() + + suite["write"]["small_uncompressed"] = @benchmarkable write_dbn($tmpfile, $metadata, $records) setup=(GC.gc()) teardown=(rm($tmpfile, force=true)) + + tmpfile_zst = tmpfile * ".zst" + suite["write"]["small_compressed"] = @benchmarkable write_dbn($tmpfile_zst, $metadata, $records) setup=(GC.gc()) teardown=(rm($tmpfile_zst, force=true)) + end + + if !isnothing(medium_file) && isfile(medium_file) + metadata, records = read_dbn_with_metadata(medium_file) + tmpfile = tempname() + + suite["write"]["medium_uncompressed"] = @benchmarkable write_dbn($tmpfile, $metadata, $records) setup=(GC.gc()) teardown=(rm($tmpfile, force=true)) + end + + # === CONVERSION BENCHMARKS === + suite["convert"] = BenchmarkGroup() + + if !isnothing(small_file) && isfile(small_file) + tmpjson = tempname() * ".json" + tmpcsv = tempname() * ".csv" + + suite["convert"]["to_json"] = @benchmarkable dbn_to_json($small_file, $tmpjson) setup=(GC.gc()) teardown=(rm($tmpjson, force=true)) + suite["convert"]["to_csv"] = @benchmarkable dbn_to_csv($small_file, $tmpcsv) setup=(GC.gc()) teardown=(rm($tmpcsv, force=true)) + end + + # === UTILITY BENCHMARKS === + suite["utils"] = BenchmarkGroup() + + # Price conversions + suite["utils"]["float_to_price"] = @benchmarkable float_to_price(100.50) + suite["utils"]["price_to_float"] = @benchmarkable price_to_float(1005000000) + + # Timestamp conversions + dt = DateTime(2024, 1, 1, 9, 30) + ts = datetime_to_ts(dt) + suite["utils"]["datetime_to_ts"] = @benchmarkable datetime_to_ts($dt) + suite["utils"]["ts_to_datetime"] = @benchmarkable ts_to_datetime($ts) + + return suite +end + +""" + run_benchmark_suite(data_dir="benchmark/data"; save_results=true) + +Run the complete benchmark suite and optionally save results. + +# Arguments +- `data_dir`: Directory containing test data files +- `save_results`: Whether to save results to CSV files (default: true) + +# Returns +BenchmarkTools results object +""" +function run_benchmark_suite(data_dir="benchmark/data"; save_results=true) + println("\n" * "="^70) + println("DBN.jl BenchmarkTools Suite") + println("="^70) + println("\nCreating benchmark suite...") + + suite = create_benchmark_suite(data_dir) + + if isempty(suite) + error("No benchmarks in suite. Check that test data exists in $data_dir") + end + + println("Suite created with $(length(keys(suite))) groups") + for group in keys(suite) + println(" - $group: $(length(keys(suite[group]))) benchmarks") + end + + println("\nRunning benchmarks (this may take several minutes)...") + println("Started at: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))") + + results = run(suite, verbose=true) + + println("\nCompleted at: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))") + + # Print results + print_benchmark_results(results) + + # Save results if requested + if save_results + timestamp = Dates.format(now(), "yyyymmdd_HHMMSS") + csv_file = "benchmark/results_$(timestamp).csv" + json_file = "benchmark/results_$(timestamp).json" + + save_results_to_csv(results, csv_file) + println("\nResults saved to:") + println(" - $csv_file") + + # Save raw BenchmarkTools results + BenchmarkTools.save(json_file, results) + println(" - $json_file") + end + + return results +end + +""" + print_benchmark_results(results) + +Pretty-print benchmark results in a readable format. +""" +function print_benchmark_results(results) + println("\n\n" * "="^70) + println("BENCHMARK RESULTS") + println("="^70) + + for group_name in sort(collect(keys(results))) + group = results[group_name] + println("\n" * group_name * ":") + println("─"^70) + + for bench_name in sort(collect(keys(group))) + bench_result = group[bench_name] + print_single_result(bench_name, bench_result) + end + end + + println("\n" * "="^70) +end + +""" + print_single_result(name, result) + +Print a single benchmark result. +""" +function print_single_result(name, result) + trial = minimum(result) + + @printf " %-25s" name + + # Time + time_ns = time(trial) + if time_ns < 1_000 + @printf " %8.2f ns" time_ns + elseif time_ns < 1_000_000 + @printf " %8.2f μs" (time_ns / 1_000) + elseif time_ns < 1_000_000_000 + @printf " %8.2f ms" (time_ns / 1_000_000) + else + @printf " %8.2f s " (time_ns / 1_000_000_000) + end + + # Memory + mem_bytes = memory(trial) + if mem_bytes < 1024 + @printf " %8d B" mem_bytes + elseif mem_bytes < 1024^2 + @printf " %8.2f KB" (mem_bytes / 1024) + elseif mem_bytes < 1024^3 + @printf " %8.2f MB" (mem_bytes / 1024^2) + else + @printf " %8.2f GB" (mem_bytes / 1024^3) + end + + # Allocations + @printf " %6d allocs" allocs(trial) + + println() +end + +""" + save_results_to_csv(results, filename) + +Save benchmark results to a CSV file for further analysis. +""" +function save_results_to_csv(results, filename) + rows = [] + + for group_name in keys(results) + group = results[group_name] + for bench_name in keys(group) + bench_result = group[bench_name] + trial = minimum(bench_result) + + push!(rows, ( + group = String(group_name), + benchmark = String(bench_name), + min_time_ns = time(trial), + median_time_ns = median(bench_result).time, + mean_time_ns = mean(bench_result).time, + max_time_ns = maximum(bench_result).time, + memory_bytes = memory(trial), + allocs = allocs(trial), + gc_time_ns = gctime(trial) + )) + end + end + + df = DataFrame(rows) + + # Add computed columns + df.min_time_ms = df.min_time_ns ./ 1e6 + df.median_time_ms = df.median_time_ns ./ 1e6 + df.memory_mb = df.memory_bytes ./ 1024^2 + + mkpath(dirname(filename)) + CSV.write(filename, df) +end + +""" + compare_benchmarks(file1::String, file2::String) + +Compare two benchmark result files to track performance changes. + +# Arguments +- `file1`: Path to first benchmark results JSON file (baseline) +- `file2`: Path to second benchmark results JSON file (current) +""" +function compare_benchmarks(file1::String, file2::String) + baseline = BenchmarkTools.load(file1)[1] + current = BenchmarkTools.load(file2)[1] + + println("\n" * "="^70) + println("BENCHMARK COMPARISON") + println("="^70) + println("Baseline: $file1") + println("Current: $file2") + println("="^70) + + judge_results = judge(current, baseline) + + println(judge_results) +end + +# Run if executed as script +if abspath(PROGRAM_FILE) == @__FILE__ + data_dir = length(ARGS) >= 1 ? ARGS[1] : "benchmark/data" + run_benchmark_suite(data_dir) +end diff --git a/benchmark/generate_test_data.jl b/benchmark/generate_test_data.jl index 8929cdf6..012d7697 100644 --- a/benchmark/generate_test_data.jl +++ b/benchmark/generate_test_data.jl @@ -1,289 +1,289 @@ -""" - generate_test_data.jl - -Generate large DBN test files for performance benchmarking. - -This script creates DBN files of various sizes with realistic market data -to enable thorough performance testing. -""" - +""" + generate_test_data.jl + +Generate large DBN test files for performance benchmarking. + +This script creates DBN files of various sizes with realistic market data +to enable thorough performance testing. +""" + using DatabentoBinaryEncoding import DatabentoBinaryEncoding as DBN -using Dates -using Printf - -""" - generate_trade_messages(n_records::Int; - start_time=DateTime(2024, 1, 1, 9, 30), - instrument_id=12345, - publisher_id=1) - -Generate a vector of `n_records` synthetic trade messages with realistic price movements. - -# Arguments -- `n_records`: Number of trade records to generate -- `start_time`: Starting timestamp for the first trade -- `instrument_id`: Instrument ID for the trades -- `publisher_id`: Publisher ID for the trades - -# Returns -- `Vector{TradeMsg}`: Vector of synthetic trade messages -""" -function generate_trade_messages(n_records::Int; - start_time=DateTime(2024, 1, 1, 9, 30), - instrument_id=12345, - publisher_id=1) - trades = TradeMsg[] - sizehint!(trades, n_records) - - # Starting price around $100 - base_price = 100.0 - current_price = base_price - - # Convert start time to nanoseconds - current_ts = datetime_to_ts(start_time) - - for i in 1:n_records - # Simulate price movement (random walk with small steps) - price_change = (rand() - 0.5) * 0.10 # +/- $0.05 - current_price = max(base_price * 0.9, min(base_price * 1.1, current_price + price_change)) - - # Random size between 1 and 1000 shares - size = UInt32(rand(1:1000)) - - # Random side - side = rand([Side.BID, Side.ASK]) - - # Increment timestamp by ~100-500 microseconds - current_ts += rand(100_000:500_000) - - trade = TradeMsg( - RecordHeader( - UInt8(sizeof(TradeMsg) ÷ 4), # length in 4-byte units - RType.MBP_0_MSG, - UInt16(publisher_id), - UInt32(instrument_id), - UInt64(current_ts) - ), - float_to_price(current_price), - size, - Action.TRADE, - side, - UInt8(0), # flags - UInt8(0), # depth - current_ts, # ts_recv - Int32(0), # ts_in_delta - UInt32(i) # sequence - ) - - push!(trades, trade) - end - - return trades -end - -""" - generate_mbo_messages(n_records::Int; kwargs...) - -Generate synthetic market-by-order messages. -""" -function generate_mbo_messages(n_records::Int; - start_time=DateTime(2024, 1, 1, 9, 30), - instrument_id=12345, - publisher_id=1) - messages = MBOMsg[] - sizehint!(messages, n_records) - - base_price = 100.0 - current_ts = datetime_to_ts(start_time) - order_id_counter = UInt64(1000000) - - for i in 1:n_records - price_offset = (rand() - 0.5) * 1.0 - price = base_price + price_offset - size = UInt32(rand(1:1000)) - side = rand([Side.BID, Side.ASK]) - action = rand([Action.ADD, Action.MODIFY, Action.CANCEL, Action.TRADE]) - - current_ts += rand(10_000:100_000) - order_id_counter += 1 - - msg = MBOMsg( - RecordHeader( - UInt8(sizeof(MBOMsg) ÷ 4), - RType.MBO_MSG, - UInt16(publisher_id), - UInt32(instrument_id), - UInt64(current_ts) - ), - order_id_counter, - float_to_price(price), - size, - UInt8(0), # flags - UInt8(1), # channel_id - action, - side, - current_ts, - Int32(0), - UInt32(i) - ) - - push!(messages, msg) - end - - return messages -end - -""" - generate_ohlcv_messages(n_records::Int; kwargs...) - -Generate synthetic OHLCV messages. -""" -function generate_ohlcv_messages(n_records::Int; - start_time=DateTime(2024, 1, 1, 9, 30), - instrument_id=12345, - publisher_id=1) - messages = OHLCVMsg[] - sizehint!(messages, n_records) - - base_price = 100.0 - current_ts = datetime_to_ts(start_time) - - for i in 1:n_records - # Each bar is 1 minute apart - current_ts += 60_000_000_000 # 60 seconds in nanoseconds - - open_price = base_price + (rand() - 0.5) * 2.0 - high = open_price + rand() * 1.0 - low = open_price - rand() * 1.0 - close_price = open_price + (rand() - 0.5) * 1.5 - volume = UInt64(rand(1000:100000)) - - msg = OHLCVMsg( - RecordHeader( - UInt8(sizeof(OHLCVMsg) ÷ 4), - RType.OHLCV_1M_MSG, - UInt16(publisher_id), - UInt32(instrument_id), - UInt64(current_ts) - ), - float_to_price(open_price), - float_to_price(high), - float_to_price(low), - float_to_price(close_price), - volume - ) - - push!(messages, msg) - end - - return messages -end - -""" - create_metadata(schema::Schema.T, records, dataset="TEST") - -Create metadata for a set of records. -""" -function create_metadata(schema::Schema.T, records, dataset="TEST") - start_ts = records[1].hd.ts_event - end_ts = records[end].hd.ts_event - - return Metadata( - UInt8(DBN_VERSION), - dataset, - schema, - start_ts, - end_ts, - UInt64(length(records)), - SType.RAW_SYMBOL, - SType.RAW_SYMBOL, - false, - String[], - String[], - String[], - Tuple{String, String, Int64, Int64}[] - ) -end - -""" - generate_test_files(output_dir="benchmark/data") - -Generate a suite of test files with various sizes and types. - -Creates the following files: -- Small files (1K, 10K records) - for quick tests -- Medium files (100K, 1M records) - for realistic benchmarks -- Large files (10M records) - for stress testing -- Different message types (TRADES, MBO, OHLCV) -- Both compressed and uncompressed versions -""" -function generate_test_files(output_dir="benchmark/data") - mkpath(output_dir) - - sizes = [ - ("100k", 100_000), - ("1m", 1_000_000), - ("10m", 10_000_000), - ] - - for (size_label, n_records) in sizes - println("\nGenerating $size_label records...") - - # TRADES - println(" - trades.$size_label...") - trades = generate_trade_messages(n_records) - metadata = create_metadata(Schema.TRADES, trades, "XNAS") - - trades_file = joinpath(output_dir, "trades.$size_label.dbn") - write_dbn(trades_file, metadata, trades) - - # Create compressed version - trades_zst = joinpath(output_dir, "trades.$size_label.dbn.zst") - write_dbn(trades_zst, metadata, trades) - - # MBO (skip for 10M to save time) - if n_records <= 1_000_000 - println(" - mbo.$size_label...") - mbo = generate_mbo_messages(n_records) - metadata_mbo = create_metadata(Schema.MBO, mbo, "XNAS") - - mbo_file = joinpath(output_dir, "mbo.$size_label.dbn") - write_dbn(mbo_file, metadata_mbo, mbo) - - mbo_zst = joinpath(output_dir, "mbo.$size_label.dbn.zst") - write_dbn(mbo_zst, metadata_mbo, mbo) - end - - # OHLCV - only generate 10M for performance benchmarking - if size_label == "10m" - println(" - ohlcv.$size_label...") - ohlcv = generate_ohlcv_messages(n_records) - metadata_ohlcv = create_metadata(Schema.OHLCV_1M, ohlcv, "XNAS") - - ohlcv_file = joinpath(output_dir, "ohlcv.$size_label.dbn") - write_dbn(ohlcv_file, metadata_ohlcv, ohlcv) - - ohlcv_zst = joinpath(output_dir, "ohlcv.$size_label.dbn.zst") - write_dbn(ohlcv_zst, metadata_ohlcv, ohlcv) - end - end - - println("\n" * "="^60) - println("Test data generation complete!") - println("Output directory: $output_dir") - println("="^60) - - # Print file sizes - println("\nGenerated files:") - for file in readdir(output_dir, join=true) - size_mb = filesize(file) / 1024^2 - @printf " %-40s %8.2f MB\n" basename(file) size_mb - end -end - -# Run if executed as script -if abspath(PROGRAM_FILE) == @__FILE__ - println("Generating benchmark test data...") - generate_test_files() -end +using Dates +using Printf + +""" + generate_trade_messages(n_records::Int; + start_time=DateTime(2024, 1, 1, 9, 30), + instrument_id=12345, + publisher_id=1) + +Generate a vector of `n_records` synthetic trade messages with realistic price movements. + +# Arguments +- `n_records`: Number of trade records to generate +- `start_time`: Starting timestamp for the first trade +- `instrument_id`: Instrument ID for the trades +- `publisher_id`: Publisher ID for the trades + +# Returns +- `Vector{TradeMsg}`: Vector of synthetic trade messages +""" +function generate_trade_messages(n_records::Int; + start_time=DateTime(2024, 1, 1, 9, 30), + instrument_id=12345, + publisher_id=1) + trades = TradeMsg[] + sizehint!(trades, n_records) + + # Starting price around $100 + base_price = 100.0 + current_price = base_price + + # Convert start time to nanoseconds + current_ts = datetime_to_ts(start_time) + + for i in 1:n_records + # Simulate price movement (random walk with small steps) + price_change = (rand() - 0.5) * 0.10 # +/- $0.05 + current_price = max(base_price * 0.9, min(base_price * 1.1, current_price + price_change)) + + # Random size between 1 and 1000 shares + size = UInt32(rand(1:1000)) + + # Random side + side = rand([Side.BID, Side.ASK]) + + # Increment timestamp by ~100-500 microseconds + current_ts += rand(100_000:500_000) + + trade = TradeMsg( + RecordHeader( + UInt8(sizeof(TradeMsg) ÷ 4), # length in 4-byte units + RType.MBP_0_MSG, + UInt16(publisher_id), + UInt32(instrument_id), + UInt64(current_ts) + ), + float_to_price(current_price), + size, + Action.TRADE, + side, + UInt8(0), # flags + UInt8(0), # depth + current_ts, # ts_recv + Int32(0), # ts_in_delta + UInt32(i) # sequence + ) + + push!(trades, trade) + end + + return trades +end + +""" + generate_mbo_messages(n_records::Int; kwargs...) + +Generate synthetic market-by-order messages. +""" +function generate_mbo_messages(n_records::Int; + start_time=DateTime(2024, 1, 1, 9, 30), + instrument_id=12345, + publisher_id=1) + messages = MBOMsg[] + sizehint!(messages, n_records) + + base_price = 100.0 + current_ts = datetime_to_ts(start_time) + order_id_counter = UInt64(1000000) + + for i in 1:n_records + price_offset = (rand() - 0.5) * 1.0 + price = base_price + price_offset + size = UInt32(rand(1:1000)) + side = rand([Side.BID, Side.ASK]) + action = rand([Action.ADD, Action.MODIFY, Action.CANCEL, Action.TRADE]) + + current_ts += rand(10_000:100_000) + order_id_counter += 1 + + msg = MBOMsg( + RecordHeader( + UInt8(sizeof(MBOMsg) ÷ 4), + RType.MBO_MSG, + UInt16(publisher_id), + UInt32(instrument_id), + UInt64(current_ts) + ), + order_id_counter, + float_to_price(price), + size, + UInt8(0), # flags + UInt8(1), # channel_id + action, + side, + current_ts, + Int32(0), + UInt32(i) + ) + + push!(messages, msg) + end + + return messages +end + +""" + generate_ohlcv_messages(n_records::Int; kwargs...) + +Generate synthetic OHLCV messages. +""" +function generate_ohlcv_messages(n_records::Int; + start_time=DateTime(2024, 1, 1, 9, 30), + instrument_id=12345, + publisher_id=1) + messages = OHLCVMsg[] + sizehint!(messages, n_records) + + base_price = 100.0 + current_ts = datetime_to_ts(start_time) + + for i in 1:n_records + # Each bar is 1 minute apart + current_ts += 60_000_000_000 # 60 seconds in nanoseconds + + open_price = base_price + (rand() - 0.5) * 2.0 + high = open_price + rand() * 1.0 + low = open_price - rand() * 1.0 + close_price = open_price + (rand() - 0.5) * 1.5 + volume = UInt64(rand(1000:100000)) + + msg = OHLCVMsg( + RecordHeader( + UInt8(sizeof(OHLCVMsg) ÷ 4), + RType.OHLCV_1M_MSG, + UInt16(publisher_id), + UInt32(instrument_id), + UInt64(current_ts) + ), + float_to_price(open_price), + float_to_price(high), + float_to_price(low), + float_to_price(close_price), + volume + ) + + push!(messages, msg) + end + + return messages +end + +""" + create_metadata(schema::Schema.T, records, dataset="TEST") + +Create metadata for a set of records. +""" +function create_metadata(schema::Schema.T, records, dataset="TEST") + start_ts = records[1].hd.ts_event + end_ts = records[end].hd.ts_event + + return Metadata( + UInt8(DBN_VERSION), + dataset, + schema, + start_ts, + end_ts, + UInt64(length(records)), + SType.RAW_SYMBOL, + SType.RAW_SYMBOL, + false, + String[], + String[], + String[], + Tuple{String, String, Int64, Int64}[] + ) +end + +""" + generate_test_files(output_dir="benchmark/data") + +Generate a suite of test files with various sizes and types. + +Creates the following files: +- Small files (1K, 10K records) - for quick tests +- Medium files (100K, 1M records) - for realistic benchmarks +- Large files (10M records) - for stress testing +- Different message types (TRADES, MBO, OHLCV) +- Both compressed and uncompressed versions +""" +function generate_test_files(output_dir="benchmark/data") + mkpath(output_dir) + + sizes = [ + ("100k", 100_000), + ("1m", 1_000_000), + ("10m", 10_000_000), + ] + + for (size_label, n_records) in sizes + println("\nGenerating $size_label records...") + + # TRADES + println(" - trades.$size_label...") + trades = generate_trade_messages(n_records) + metadata = create_metadata(Schema.TRADES, trades, "XNAS") + + trades_file = joinpath(output_dir, "trades.$size_label.dbn") + write_dbn(trades_file, metadata, trades) + + # Create compressed version + trades_zst = joinpath(output_dir, "trades.$size_label.dbn.zst") + write_dbn(trades_zst, metadata, trades) + + # MBO (skip for 10M to save time) + if n_records <= 1_000_000 + println(" - mbo.$size_label...") + mbo = generate_mbo_messages(n_records) + metadata_mbo = create_metadata(Schema.MBO, mbo, "XNAS") + + mbo_file = joinpath(output_dir, "mbo.$size_label.dbn") + write_dbn(mbo_file, metadata_mbo, mbo) + + mbo_zst = joinpath(output_dir, "mbo.$size_label.dbn.zst") + write_dbn(mbo_zst, metadata_mbo, mbo) + end + + # OHLCV - only generate 10M for performance benchmarking + if size_label == "10m" + println(" - ohlcv.$size_label...") + ohlcv = generate_ohlcv_messages(n_records) + metadata_ohlcv = create_metadata(Schema.OHLCV_1M, ohlcv, "XNAS") + + ohlcv_file = joinpath(output_dir, "ohlcv.$size_label.dbn") + write_dbn(ohlcv_file, metadata_ohlcv, ohlcv) + + ohlcv_zst = joinpath(output_dir, "ohlcv.$size_label.dbn.zst") + write_dbn(ohlcv_zst, metadata_ohlcv, ohlcv) + end + end + + println("\n" * "="^60) + println("Test data generation complete!") + println("Output directory: $output_dir") + println("="^60) + + # Print file sizes + println("\nGenerated files:") + for file in readdir(output_dir, join=true) + size_mb = filesize(file) / 1024^2 + @printf " %-40s %8.2f MB\n" basename(file) size_mb + end +end + +# Run if executed as script +if abspath(PROGRAM_FILE) == @__FILE__ + println("Generating benchmark test data...") + generate_test_files() +end diff --git a/benchmark/run_benchmarks.jl b/benchmark/run_benchmarks.jl index 7b82339a..4ea4a5d2 100644 --- a/benchmark/run_benchmarks.jl +++ b/benchmark/run_benchmarks.jl @@ -1,285 +1,285 @@ -""" - run_benchmarks.jl - -Main entry point for running all DatabentoBinaryEncoding.jl benchmarks. - -Usage: - julia benchmark/run_benchmarks.jl [options] - -Options: - --generate-data Generate test data before benchmarking - --throughput-only Run only throughput benchmarks (faster) - --suite-only Run only BenchmarkTools suite (detailed) - --quick Quick benchmark with small datasets only - --data-dir DIR Use custom data directory (default: benchmark/data) - --runs N Number of runs for throughput benchmarks (default: 5) -""" - -using Pkg - -# Ensure we're in the right environment -# Check if DatabentoBinaryEncoding is either a dependency or if we're in the -# DatabentoBinaryEncoding.jl project itself. -proj = Pkg.project() -if !haskey(proj.dependencies, "DatabentoBinaryEncoding") && proj.name != "DatabentoBinaryEncoding" - error("DatabentoBinaryEncoding package not found. Make sure you're in the DatabentoBinaryEncoding.jl project directory.") -end - -# Activate the project -Pkg.activate(".") - -# Load DBN -using DatabentoBinaryEncoding -import DatabentoBinaryEncoding as DBN -using Printf -using Dates - -# Load all benchmark modules at global scope to avoid world age issues -include("generate_test_data.jl") -include("throughput.jl") -include("benchmarks.jl") - -""" - parse_args(args) - -Parse command-line arguments. -""" -function parse_args(args) - options = Dict{Symbol, Any}( - :generate_data => false, - :throughput_only => false, - :suite_only => false, - :quick => false, - :data_dir => "benchmark/data", - :runs => 5 - ) - - i = 1 - while i <= length(args) - arg = args[i] - - if arg == "--generate-data" - options[:generate_data] = true - elseif arg == "--throughput-only" - options[:throughput_only] = true - elseif arg == "--suite-only" - options[:suite_only] = true - elseif arg == "--quick" - options[:quick] = true - elseif arg == "--data-dir" - i += 1 - if i <= length(args) - options[:data_dir] = args[i] - else - error("--data-dir requires a directory path") - end - elseif arg == "--runs" - i += 1 - if i <= length(args) - options[:runs] = parse(Int, args[i]) - else - error("--runs requires a number") - end - elseif arg == "--help" || arg == "-h" - print_help() - exit(0) - else - @warn "Unknown argument: $arg" - end - - i += 1 - end - - return options -end - -""" - print_help() - -Print usage information. -""" -function print_help() - println(""" - DatabentoBinaryEncoding.jl Benchmark Suite - - Usage: - julia benchmark/run_benchmarks.jl [options] - - Options: - --generate-data Generate test data before benchmarking - --throughput-only Run only throughput benchmarks (faster) - --suite-only Run only BenchmarkTools suite (detailed) - --quick Quick benchmark with small datasets only - --data-dir DIR Use custom data directory (default: benchmark/data) - --runs N Number of runs for throughput benchmarks (default: 5) - --help, -h Show this help message - - Examples: - # Full benchmark suite (generates data, runs all benchmarks) - julia benchmark/run_benchmarks.jl --generate-data - - # Quick benchmark with existing data - julia benchmark/run_benchmarks.jl --quick - - # Throughput-only benchmark - julia benchmark/run_benchmarks.jl --throughput-only - - # Custom data directory - julia benchmark/run_benchmarks.jl --data-dir /path/to/data - """) -end - -""" - print_banner() - -Print an attractive banner for the benchmark suite. -""" -function print_banner() - println("\n") - println("█"^80) - println("█" * " "^78 * "█") - println("█" * " DatabentoBinaryEncoding.jl - Databento Binary Encoding Performance Benchmark" * " "^12 * "█") - println("█" * " "^78 * "█") - println("█"^80) - println() -end - -""" - check_data_exists(data_dir) - -Check if benchmark data exists. -""" -function check_data_exists(data_dir) - if !isdir(data_dir) - return false - end - - test_files = filter(f -> endswith(f, ".dbn") || endswith(f, ".dbn.zst"), - readdir(data_dir)) - - return !isempty(test_files) -end - -""" - main() - -Main benchmark runner. -""" -function main() - options = parse_args(ARGS) - - print_banner() - - println("Configuration:") - println(" Data directory: $(options[:data_dir])") - println(" Throughput runs: $(options[:runs])") - println(" Quick mode: $(options[:quick])") - println() - - # Check if data exists - data_exists = check_data_exists(options[:data_dir]) - - if !data_exists && !options[:generate_data] - println("⚠️ No test data found in $(options[:data_dir])") - println() - println("You need to generate test data first. Run with:") - println(" julia benchmark/run_benchmarks.jl --generate-data") - println() - println("Or generate data separately:") - println(" julia benchmark/generate_test_data.jl") - println() - exit(1) - end - - # Step 1: Generate test data if requested - if options[:generate_data] - println("\n" * "▶"^80) - println("STEP 1: Generating Test Data") - println("▶"^80) - println() - - if options[:quick] - println("Quick mode: Generating small datasets only...") - # Generate only small files for quick benchmarking - generate_test_files_quick(options[:data_dir]) - else - generate_test_files(options[:data_dir]) - end - - println("\n✓ Test data generation complete!") - end - - # Step 2: Run throughput benchmarks - if !options[:suite_only] - println("\n" * "▶"^80) - println("STEP 2: Throughput Benchmarks (Records/Second)") - println("▶"^80) - println() - - run_throughput_benchmarks(options[:data_dir]; runs=options[:runs]) - - println("\n✓ Throughput benchmarks complete!") - end - - # Step 3: Run BenchmarkTools suite - if !options[:throughput_only] - println("\n" * "▶"^80) - println("STEP 3: Detailed BenchmarkTools Suite") - println("▶"^80) - println() - - results = run_benchmark_suite(options[:data_dir]) - - println("\n✓ BenchmarkTools suite complete!") - end - - # Final summary - println("\n\n") - println("█"^80) - println("█" * " "^78 * "█") - println("█" * " BENCHMARK SUITE COMPLETED" * " "^50 * "█") - println("█" * " "^78 * "█") - println("█"^80) - println() - println("All benchmarks completed successfully at: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))") - println() - println("Results saved in:") - println(" - benchmark/results_*.csv") - println(" - benchmark/results_*.json") - println() -end - -""" - generate_test_files_quick(output_dir) - -Generate only small test files for quick benchmarking. -""" -function generate_test_files_quick(output_dir) - mkpath(output_dir) - - sizes = [ - ("1k", 1_000), - ("10k", 10_000), - ] - - for (size_label, n_records) in sizes - println("Generating $size_label records...") - - # TRADES - trades = generate_trade_messages(n_records) - metadata = create_metadata(Schema.TRADES, trades, "XNAS") - - trades_file = joinpath(output_dir, "trades.$size_label.dbn") - write_dbn(trades_file, metadata, trades) - - trades_zst = joinpath(output_dir, "trades.$size_label.dbn.zst") - write_dbn(trades_zst, metadata, trades) - end - - println("\nQuick test data generation complete!") -end - -# Run main if executed as script -if abspath(PROGRAM_FILE) == @__FILE__ - main() -end +""" + run_benchmarks.jl + +Main entry point for running all DatabentoBinaryEncoding.jl benchmarks. + +Usage: + julia benchmark/run_benchmarks.jl [options] + +Options: + --generate-data Generate test data before benchmarking + --throughput-only Run only throughput benchmarks (faster) + --suite-only Run only BenchmarkTools suite (detailed) + --quick Quick benchmark with small datasets only + --data-dir DIR Use custom data directory (default: benchmark/data) + --runs N Number of runs for throughput benchmarks (default: 5) +""" + +using Pkg + +# Ensure we're in the right environment +# Check if DatabentoBinaryEncoding is either a dependency or if we're in the +# DatabentoBinaryEncoding.jl project itself. +proj = Pkg.project() +if !haskey(proj.dependencies, "DatabentoBinaryEncoding") && proj.name != "DatabentoBinaryEncoding" + error("DatabentoBinaryEncoding package not found. Make sure you're in the DatabentoBinaryEncoding.jl project directory.") +end + +# Activate the project +Pkg.activate(".") + +# Load DBN +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN +using Printf +using Dates + +# Load all benchmark modules at global scope to avoid world age issues +include("generate_test_data.jl") +include("throughput.jl") +include("benchmarks.jl") + +""" + parse_args(args) + +Parse command-line arguments. +""" +function parse_args(args) + options = Dict{Symbol, Any}( + :generate_data => false, + :throughput_only => false, + :suite_only => false, + :quick => false, + :data_dir => "benchmark/data", + :runs => 5 + ) + + i = 1 + while i <= length(args) + arg = args[i] + + if arg == "--generate-data" + options[:generate_data] = true + elseif arg == "--throughput-only" + options[:throughput_only] = true + elseif arg == "--suite-only" + options[:suite_only] = true + elseif arg == "--quick" + options[:quick] = true + elseif arg == "--data-dir" + i += 1 + if i <= length(args) + options[:data_dir] = args[i] + else + error("--data-dir requires a directory path") + end + elseif arg == "--runs" + i += 1 + if i <= length(args) + options[:runs] = parse(Int, args[i]) + else + error("--runs requires a number") + end + elseif arg == "--help" || arg == "-h" + print_help() + exit(0) + else + @warn "Unknown argument: $arg" + end + + i += 1 + end + + return options +end + +""" + print_help() + +Print usage information. +""" +function print_help() + println(""" + DatabentoBinaryEncoding.jl Benchmark Suite + + Usage: + julia benchmark/run_benchmarks.jl [options] + + Options: + --generate-data Generate test data before benchmarking + --throughput-only Run only throughput benchmarks (faster) + --suite-only Run only BenchmarkTools suite (detailed) + --quick Quick benchmark with small datasets only + --data-dir DIR Use custom data directory (default: benchmark/data) + --runs N Number of runs for throughput benchmarks (default: 5) + --help, -h Show this help message + + Examples: + # Full benchmark suite (generates data, runs all benchmarks) + julia benchmark/run_benchmarks.jl --generate-data + + # Quick benchmark with existing data + julia benchmark/run_benchmarks.jl --quick + + # Throughput-only benchmark + julia benchmark/run_benchmarks.jl --throughput-only + + # Custom data directory + julia benchmark/run_benchmarks.jl --data-dir /path/to/data + """) +end + +""" + print_banner() + +Print an attractive banner for the benchmark suite. +""" +function print_banner() + println("\n") + println("█"^80) + println("█" * " "^78 * "█") + println("█" * " DatabentoBinaryEncoding.jl - Databento Binary Encoding Performance Benchmark" * " "^12 * "█") + println("█" * " "^78 * "█") + println("█"^80) + println() +end + +""" + check_data_exists(data_dir) + +Check if benchmark data exists. +""" +function check_data_exists(data_dir) + if !isdir(data_dir) + return false + end + + test_files = filter(f -> endswith(f, ".dbn") || endswith(f, ".dbn.zst"), + readdir(data_dir)) + + return !isempty(test_files) +end + +""" + main() + +Main benchmark runner. +""" +function main() + options = parse_args(ARGS) + + print_banner() + + println("Configuration:") + println(" Data directory: $(options[:data_dir])") + println(" Throughput runs: $(options[:runs])") + println(" Quick mode: $(options[:quick])") + println() + + # Check if data exists + data_exists = check_data_exists(options[:data_dir]) + + if !data_exists && !options[:generate_data] + println("⚠️ No test data found in $(options[:data_dir])") + println() + println("You need to generate test data first. Run with:") + println(" julia benchmark/run_benchmarks.jl --generate-data") + println() + println("Or generate data separately:") + println(" julia benchmark/generate_test_data.jl") + println() + exit(1) + end + + # Step 1: Generate test data if requested + if options[:generate_data] + println("\n" * "▶"^80) + println("STEP 1: Generating Test Data") + println("▶"^80) + println() + + if options[:quick] + println("Quick mode: Generating small datasets only...") + # Generate only small files for quick benchmarking + generate_test_files_quick(options[:data_dir]) + else + generate_test_files(options[:data_dir]) + end + + println("\n✓ Test data generation complete!") + end + + # Step 2: Run throughput benchmarks + if !options[:suite_only] + println("\n" * "▶"^80) + println("STEP 2: Throughput Benchmarks (Records/Second)") + println("▶"^80) + println() + + run_throughput_benchmarks(options[:data_dir]; runs=options[:runs]) + + println("\n✓ Throughput benchmarks complete!") + end + + # Step 3: Run BenchmarkTools suite + if !options[:throughput_only] + println("\n" * "▶"^80) + println("STEP 3: Detailed BenchmarkTools Suite") + println("▶"^80) + println() + + results = run_benchmark_suite(options[:data_dir]) + + println("\n✓ BenchmarkTools suite complete!") + end + + # Final summary + println("\n\n") + println("█"^80) + println("█" * " "^78 * "█") + println("█" * " BENCHMARK SUITE COMPLETED" * " "^50 * "█") + println("█" * " "^78 * "█") + println("█"^80) + println() + println("All benchmarks completed successfully at: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))") + println() + println("Results saved in:") + println(" - benchmark/results_*.csv") + println(" - benchmark/results_*.json") + println() +end + +""" + generate_test_files_quick(output_dir) + +Generate only small test files for quick benchmarking. +""" +function generate_test_files_quick(output_dir) + mkpath(output_dir) + + sizes = [ + ("1k", 1_000), + ("10k", 10_000), + ] + + for (size_label, n_records) in sizes + println("Generating $size_label records...") + + # TRADES + trades = generate_trade_messages(n_records) + metadata = create_metadata(Schema.TRADES, trades, "XNAS") + + trades_file = joinpath(output_dir, "trades.$size_label.dbn") + write_dbn(trades_file, metadata, trades) + + trades_zst = joinpath(output_dir, "trades.$size_label.dbn.zst") + write_dbn(trades_zst, metadata, trades) + end + + println("\nQuick test data generation complete!") +end + +# Run main if executed as script +if abspath(PROGRAM_FILE) == @__FILE__ + main() +end diff --git a/benchmark/verify_setup.sh b/benchmark/verify_setup.sh index 6c21d330..0ac7a7e6 100755 --- a/benchmark/verify_setup.sh +++ b/benchmark/verify_setup.sh @@ -1,63 +1,63 @@ -#!/bin/bash -# Verify benchmark suite setup - -echo "==================================================" -echo " DatabentoBinaryEncoding.jl Benchmark Suite Verification" -echo "==================================================" -echo - -# Check if files exist -echo "Checking benchmark files..." -files=( - "benchmark/generate_test_data.jl" - "benchmark/throughput.jl" - "benchmark/benchmarks.jl" - "benchmark/run_benchmarks.jl" - "benchmark/README.md" -) - -all_exist=true -for file in "${files[@]}"; do - if [ -f "$file" ]; then - echo " ✓ $file" - else - echo " ✗ $file (missing)" - all_exist=false - fi -done - -echo - -# Check Julia installation -if command -v julia &> /dev/null; then - echo "✓ Julia found: $(julia --version)" - echo - - # Check if DBN can be loaded - echo "Testing DBN package load..." - julia --project=. -e 'using DatabentoBinaryEncoding; println("✓ DatabentoBinaryEncoding.jl loaded successfully")' 2>&1 - - if [ $? -eq 0 ]; then - echo - echo "==================================================" - echo " Setup verification PASSED" - echo "==================================================" - echo - echo "You can now run benchmarks with:" - echo " julia benchmark/run_benchmarks.jl --generate-data --quick" - echo - else - echo - echo "⚠ DatabentoBinaryEncoding.jl could not be loaded. Run:" - echo " julia --project=. -e 'using Pkg; Pkg.instantiate()'" - echo - fi -else - echo "⚠ Julia not found. Please install Julia 1.12 or later." - echo - if [ "$all_exist" = true ]; then - echo "Benchmark files are present and ready to use once Julia is installed." - fi -fi - -echo +#!/bin/bash +# Verify benchmark suite setup + +echo "==================================================" +echo " DatabentoBinaryEncoding.jl Benchmark Suite Verification" +echo "==================================================" +echo + +# Check if files exist +echo "Checking benchmark files..." +files=( + "benchmark/generate_test_data.jl" + "benchmark/throughput.jl" + "benchmark/benchmarks.jl" + "benchmark/run_benchmarks.jl" + "benchmark/README.md" +) + +all_exist=true +for file in "${files[@]}"; do + if [ -f "$file" ]; then + echo " ✓ $file" + else + echo " ✗ $file (missing)" + all_exist=false + fi +done + +echo + +# Check Julia installation +if command -v julia &> /dev/null; then + echo "✓ Julia found: $(julia --version)" + echo + + # Check if DBN can be loaded + echo "Testing DBN package load..." + julia --project=. -e 'using DatabentoBinaryEncoding; println("✓ DatabentoBinaryEncoding.jl loaded successfully")' 2>&1 + + if [ $? -eq 0 ]; then + echo + echo "==================================================" + echo " Setup verification PASSED" + echo "==================================================" + echo + echo "You can now run benchmarks with:" + echo " julia benchmark/run_benchmarks.jl --generate-data --quick" + echo + else + echo + echo "⚠ DatabentoBinaryEncoding.jl could not be loaded. Run:" + echo " julia --project=. -e 'using Pkg; Pkg.instantiate()'" + echo + fi +else + echo "⚠ Julia not found. Please install Julia 1.12 or later." + echo + if [ "$all_exist" = true ]; then + echo "Benchmark files are present and ready to use once Julia is installed." + fi +fi + +echo diff --git a/src/DatabentoBinaryEncoding.jl b/src/DatabentoBinaryEncoding.jl index aeda549b..f7ac875f 100644 --- a/src/DatabentoBinaryEncoding.jl +++ b/src/DatabentoBinaryEncoding.jl @@ -1,135 +1,135 @@ -""" - DatabentoBinaryEncoding - -Julia implementation of the Databento Binary Encoding (DBN) format for normalized market data. - -# Overview - -DatabentoBinaryEncoding.jl provides complete support for reading and writing DBN v3 format files with: -- Efficient streaming support for large files -- Automatic Zstd compression/decompression -- All DBN v3 message types -- Timestamp utilities with nanosecond precision -- Price conversion utilities with fixed-point arithmetic - -# Main Functions - -## Reading Data -- `read_dbn(filename)`: Read entire file into memory -- `DBNStream(filename)`: Memory-efficient streaming iterator -- `DBNDecoder(filename)`: Low-level decoder with manual control - -## Writing Data -- `write_dbn(filename, metadata, records)`: Write complete file -- `DBNStreamWriter(filename, dataset, schema)`: Real-time streaming writer -- `DBNEncoder(io, metadata)`: Low-level encoder - -## Compression -- `compress_dbn_file(input, output)`: Compress single file -- `compress_daily_files(date, directory)`: Batch compress files - -## Format Conversion -- `dbn_to_csv(input, output)`: Convert DBN to CSV -- `dbn_to_json(input, output)`: Convert DBN to JSON -- `dbn_to_parquet(input, output)`: Convert DBN to Parquet -- `json_to_dbn(input, output)`: Convert JSON to DBN -- `parquet_to_dbn(input, output)`: Convert Parquet to DBN -- `csv_to_dbn(input, output)`: Convert CSV to DBN - -## Utilities -- `price_to_float(price)` / `float_to_price(value)`: Price conversions -- `ts_to_datetime(ts)` / `datetime_to_ts(dt)`: Timestamp conversions -- `DBNTimestamp(ns)`: High-precision timestamp handling - -# Example Usage - -```julia -using DatabentoBinaryEncoding -# Or, for terser internal references: `import DatabentoBinaryEncoding as DBN` - -# Reading data -records = read_dbn("data.dbn") -for record in DBNStream("large_file.dbn.zst") - process(record) -end - -# Writing data -metadata = Metadata(3, "XNAS", Schema.TRADES, start_ts, end_ts, - length(records), SType.RAW_SYMBOL, SType.RAW_SYMBOL, - false, symbols, [], [], []) -write_dbn("output.dbn", metadata, records) - -# Streaming writer -writer = DBNStreamWriter("live.dbn", "XNAS", Schema.TRADES) -write_record!(writer, trade_msg) -close_writer!(writer) -``` - -# Supported Message Types - -- Market Data: `MBOMsg`, `TradeMsg`, `MBP1Msg`, `MBP10Msg`, `OHLCVMsg` -- Consolidated: `CMBP1Msg`, `CBBO1sMsg`, `CBBO1mMsg`, `TCBBOMsg`, `BBO1sMsg`, `BBO1mMsg` -- Status: `StatusMsg`, `ImbalanceMsg`, `StatMsg` -- System: `ErrorMsg`, `SymbolMappingMsg`, `SystemMsg` -- Definition: `InstrumentDefMsg` - -See the [DBN specification](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) -for complete format documentation. -""" -module DatabentoBinaryEncoding - -# All using statements at the top -using Dates -using CodecZstd -using TranscodingStreams -using EnumX -using DataFrames -using CSV -using DuckDB -using DBInterface -using JSON3 -using StructTypes - - -# Include all the component files -include("types.jl") -include("messages.jl") -include("show.jl") -include("buffered_io.jl") -include("decode.jl") -include("encode.jl") -include("streaming.jl") -include("replay.jl") -include("export.jl") -include("symbols.jl") -include("import.jl") - -# Exports -export DBNDecoder, DBNEncoder, read_dbn, read_dbn_with_metadata, read_dbn_typed, write_dbn -export read_trades, read_mbo, read_mbp1, read_mbp10, read_tbbo # Market depth readers -export read_ohlcv, read_ohlcv_1s, read_ohlcv_1m, read_ohlcv_1h, read_ohlcv_1d # OHLCV readers -export read_cmbp1, read_cbbo1s, read_cbbo1m, read_tcbbo, read_bbo1s, read_bbo1m # Consolidated/BBO readers -export Metadata, DBNHeader, RecordHeader, DBNTimestamp -export MBOMsg, TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg, StatusMsg, ImbalanceMsg, StatMsg -export CMBP1Msg, CBBO1sMsg, CBBO1mMsg, TCBBOMsg, BBO1sMsg, BBO1mMsg -export ErrorMsg, SymbolMappingMsg, SystemMsg, InstrumentDefMsg -export DBNStream, DBNStreamWriter, write_record!, close_writer! -export foreach_record, foreach_record_with_control, foreach_trade, foreach_mbo, foreach_mbp1, foreach_mbp10, foreach_tbbo # Market depth streaming -export record_type_for_dbn_schema # Schema -> concrete record type -export foreach_ohlcv, foreach_ohlcv_1s, foreach_ohlcv_1m, foreach_ohlcv_1h, foreach_ohlcv_1d # OHLCV streaming -export foreach_cmbp1, foreach_cbbo1s, foreach_cbbo1m, foreach_tcbbo, foreach_bbo1s, foreach_bbo1m # Consolidated/BBO streaming -export replay_dbn, replay_records # Timestamp-paced replay -export compress_dbn_file, compress_daily_files -export Schema, Compression, Encoding, SType, RType, Action, Side, InstrumentClass -export StatType, TradingEvent, safe_stat_type, safe_trading_event -export F_LAST, F_TOB, F_SNAPSHOT, F_MBP, F_BAD_TS_RECV, F_MAYBE_BAD_BOOK, F_PUBLISHER_SPECIFIC, has_flag -export price_to_float, float_to_price, ts_to_datetime, datetime_to_ts, ts_to_date_time, date_time_to_ts, to_nanoseconds -export record_length_bytes -export DBN_VERSION, FIXED_PRICE_SCALE, UNDEF_PRICE, UNDEF_ORDER_SIZE, UNDEF_TIMESTAMP -export BidAskPair, VersionUpgradePolicy, DatasetCondition -export write_header, read_header!, write_record, read_record, finalize_encoder -export dbn_to_csv, dbn_to_json, dbn_to_parquet, records_to_dataframe -export symbol_map, symbol_for, add_symbol_column! -export json_to_dbn, parquet_to_dbn, csv_to_dbn - -end # module DatabentoBinaryEncoding +""" + DatabentoBinaryEncoding + +Julia implementation of the Databento Binary Encoding (DBN) format for normalized market data. + +# Overview + +DatabentoBinaryEncoding.jl provides complete support for reading and writing DBN v3 format files with: +- Efficient streaming support for large files +- Automatic Zstd compression/decompression +- All DBN v3 message types +- Timestamp utilities with nanosecond precision +- Price conversion utilities with fixed-point arithmetic + +# Main Functions + +## Reading Data +- `read_dbn(filename)`: Read entire file into memory +- `DBNStream(filename)`: Memory-efficient streaming iterator +- `DBNDecoder(filename)`: Low-level decoder with manual control + +## Writing Data +- `write_dbn(filename, metadata, records)`: Write complete file +- `DBNStreamWriter(filename, dataset, schema)`: Real-time streaming writer +- `DBNEncoder(io, metadata)`: Low-level encoder + +## Compression +- `compress_dbn_file(input, output)`: Compress single file +- `compress_daily_files(date, directory)`: Batch compress files + +## Format Conversion +- `dbn_to_csv(input, output)`: Convert DBN to CSV +- `dbn_to_json(input, output)`: Convert DBN to JSON +- `dbn_to_parquet(input, output)`: Convert DBN to Parquet +- `json_to_dbn(input, output)`: Convert JSON to DBN +- `parquet_to_dbn(input, output)`: Convert Parquet to DBN +- `csv_to_dbn(input, output)`: Convert CSV to DBN + +## Utilities +- `price_to_float(price)` / `float_to_price(value)`: Price conversions +- `ts_to_datetime(ts)` / `datetime_to_ts(dt)`: Timestamp conversions +- `DBNTimestamp(ns)`: High-precision timestamp handling + +# Example Usage + +```julia +using DatabentoBinaryEncoding +# Or, for terser internal references: `import DatabentoBinaryEncoding as DBN` + +# Reading data +records = read_dbn("data.dbn") +for record in DBNStream("large_file.dbn.zst") + process(record) +end + +# Writing data +metadata = Metadata(3, "XNAS", Schema.TRADES, start_ts, end_ts, + length(records), SType.RAW_SYMBOL, SType.RAW_SYMBOL, + false, symbols, [], [], []) +write_dbn("output.dbn", metadata, records) + +# Streaming writer +writer = DBNStreamWriter("live.dbn", "XNAS", Schema.TRADES) +write_record!(writer, trade_msg) +close_writer!(writer) +``` + +# Supported Message Types + +- Market Data: `MBOMsg`, `TradeMsg`, `MBP1Msg`, `MBP10Msg`, `OHLCVMsg` +- Consolidated: `CMBP1Msg`, `CBBO1sMsg`, `CBBO1mMsg`, `TCBBOMsg`, `BBO1sMsg`, `BBO1mMsg` +- Status: `StatusMsg`, `ImbalanceMsg`, `StatMsg` +- System: `ErrorMsg`, `SymbolMappingMsg`, `SystemMsg` +- Definition: `InstrumentDefMsg` + +See the [DBN specification](https://databento.com/docs/standards-and-conventions/databento-binary-encoding) +for complete format documentation. +""" +module DatabentoBinaryEncoding + +# All using statements at the top +using Dates +using CodecZstd +using TranscodingStreams +using EnumX +using DataFrames +using CSV +using DuckDB +using DBInterface +using JSON3 +using StructTypes + + +# Include all the component files +include("types.jl") +include("messages.jl") +include("show.jl") +include("buffered_io.jl") +include("decode.jl") +include("encode.jl") +include("streaming.jl") +include("replay.jl") +include("export.jl") +include("symbols.jl") +include("import.jl") + +# Exports +export DBNDecoder, DBNEncoder, read_dbn, read_dbn_with_metadata, read_dbn_typed, write_dbn +export read_trades, read_mbo, read_mbp1, read_mbp10, read_tbbo # Market depth readers +export read_ohlcv, read_ohlcv_1s, read_ohlcv_1m, read_ohlcv_1h, read_ohlcv_1d # OHLCV readers +export read_cmbp1, read_cbbo1s, read_cbbo1m, read_tcbbo, read_bbo1s, read_bbo1m # Consolidated/BBO readers +export Metadata, DBNHeader, RecordHeader, DBNTimestamp +export MBOMsg, TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg, StatusMsg, ImbalanceMsg, StatMsg +export CMBP1Msg, CBBO1sMsg, CBBO1mMsg, TCBBOMsg, BBO1sMsg, BBO1mMsg +export ErrorMsg, SymbolMappingMsg, SystemMsg, InstrumentDefMsg +export DBNStream, DBNStreamWriter, write_record!, close_writer! +export foreach_record, foreach_record_with_control, foreach_trade, foreach_mbo, foreach_mbp1, foreach_mbp10, foreach_tbbo # Market depth streaming +export record_type_for_dbn_schema # Schema -> concrete record type +export foreach_ohlcv, foreach_ohlcv_1s, foreach_ohlcv_1m, foreach_ohlcv_1h, foreach_ohlcv_1d # OHLCV streaming +export foreach_cmbp1, foreach_cbbo1s, foreach_cbbo1m, foreach_tcbbo, foreach_bbo1s, foreach_bbo1m # Consolidated/BBO streaming +export replay_dbn, replay_records # Timestamp-paced replay +export compress_dbn_file, compress_daily_files +export Schema, Compression, Encoding, SType, RType, Action, Side, InstrumentClass +export StatType, TradingEvent, safe_stat_type, safe_trading_event +export F_LAST, F_TOB, F_SNAPSHOT, F_MBP, F_BAD_TS_RECV, F_MAYBE_BAD_BOOK, F_PUBLISHER_SPECIFIC, has_flag +export price_to_float, float_to_price, ts_to_datetime, datetime_to_ts, ts_to_date_time, date_time_to_ts, to_nanoseconds +export record_length_bytes +export DBN_VERSION, FIXED_PRICE_SCALE, UNDEF_PRICE, UNDEF_ORDER_SIZE, UNDEF_TIMESTAMP +export BidAskPair, VersionUpgradePolicy, DatasetCondition +export write_header, read_header!, write_record, read_record, finalize_encoder +export dbn_to_csv, dbn_to_json, dbn_to_parquet, records_to_dataframe +export symbol_map, symbol_for, add_symbol_column! +export json_to_dbn, parquet_to_dbn, csv_to_dbn + +end # module DatabentoBinaryEncoding diff --git a/src/encode.jl b/src/encode.jl index 35d2202e..c49070b5 100644 --- a/src/encode.jl +++ b/src/encode.jl @@ -1,917 +1,917 @@ -# DBN encoding functionality - -""" - DBNEncoder - -Encoder for writing DBN (Databento Binary Encoding) files with optional compression. - -# Fields -- `io::IO`: Current IO stream (may be wrapped with compression) -- `base_io::IO`: Original IO stream before any compression wrapper -- `metadata::Metadata`: Metadata for the DBN file -- `compressed_buffer::Union{IOBuffer,Nothing}`: Buffer for compressed data (if applicable) -""" -mutable struct DBNEncoder - io::IO - base_io::IO # Original IO before compression wrapper - metadata::Metadata - compressed_buffer::Union{IOBuffer,Nothing} -end - -""" - DBNEncoder(io::IO, metadata::Metadata) - -Construct a DBNEncoder for writing to an IO stream. - -# Arguments -- `io::IO`: Output stream to write to -- `metadata::Metadata`: Metadata information for the DBN file - -# Returns -- `DBNEncoder`: Encoder instance ready for writing -""" -DBNEncoder(io::IO, metadata::Metadata) = DBNEncoder(io, io, metadata, nothing) - -""" - write_header(encoder::DBNEncoder) - -Write the DBN file header including magic bytes, version, and metadata. - -# Arguments -- `encoder::DBNEncoder`: Encoder instance containing metadata to write - -# Details -Writes the complete DBN header in the correct binary format: -- Magic bytes "DBN" -- Version number -- Metadata length -- Complete metadata section with all fields - -Writes to the current IO stream (encoder.io), which may be compressed or uncompressed. -For compressed files (.zst), the entire file including the header is compressed. -""" -function write_header(encoder::DBNEncoder) - # Write header to the current IO stream (compressed or uncompressed) - io = encoder.io - - # Write magic bytes "DBN" - write(io, b"DBN") - - # Write version - write(io, UInt8(DBN_VERSION)) - - # Create metadata buffer to calculate size - metadata_buf = IOBuffer() - - # Write metadata fields in the exact format that read_header! expects - - # Dataset (16 bytes fixed-length C string) - dataset_bytes = Vector{UInt8}(undef, 16) - fill!(dataset_bytes, 0) - dataset_str_bytes = Vector{UInt8}(encoder.metadata.dataset) - copy_len = min(length(dataset_str_bytes), 15) # Leave room for null terminator - if copy_len > 0 - dataset_bytes[1:copy_len] = dataset_str_bytes[1:copy_len] - end - write(metadata_buf, dataset_bytes) - - # Schema (2 bytes) - write(metadata_buf, htol(UInt16(encoder.metadata.schema))) - - # Start timestamp (8 bytes) - write(metadata_buf, htol(UInt64(encoder.metadata.start_ts))) - - # End timestamp (8 bytes) - end_ts = encoder.metadata.end_ts === nothing ? 0 : UInt64(encoder.metadata.end_ts) - write(metadata_buf, htol(end_ts)) - - # Limit (8 bytes) - limit = encoder.metadata.limit === nothing ? 0 : encoder.metadata.limit - write(metadata_buf, htol(UInt64(limit))) - - # NOTE: For version > 1, we DON'T write record_count (8 bytes) here - # This is skipped in the read function for version > 1 - - # SType in (1 byte) - stype_in_val = encoder.metadata.stype_in === nothing ? 0xFF : UInt8(encoder.metadata.stype_in) - write(metadata_buf, stype_in_val) - - # SType out (1 byte) - write(metadata_buf, UInt8(encoder.metadata.stype_out)) - - # TS out (1 byte boolean) - write(metadata_buf, encoder.metadata.ts_out ? UInt8(1) : UInt8(0)) - - # Symbol string length (2 bytes) - only for version > 1 - # DBN v2/v3 use 71-byte symbol strings. - symbol_cstr_len = UInt16(SYMBOL_CSTR_LEN) - write(metadata_buf, htol(symbol_cstr_len)) - - # Reserved padding (53 bytes for v3) - write(metadata_buf, zeros(UInt8, 53)) - - # Schema definition length (4 bytes) - always 0 for now - write(metadata_buf, htol(UInt32(0))) - - # Variable-length sections - - # Symbols - write(metadata_buf, htol(UInt32(length(encoder.metadata.symbols)))) - for sym in encoder.metadata.symbols - sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) - fill!(sym_bytes, 0) - sym_str_bytes = Vector{UInt8}(sym) - copy_len = min(length(sym_str_bytes), symbol_cstr_len - 1) - if copy_len > 0 - sym_bytes[1:copy_len] = sym_str_bytes[1:copy_len] - end - write(metadata_buf, sym_bytes) - end - - # Partial symbols - write(metadata_buf, htol(UInt32(length(encoder.metadata.partial)))) - for sym in encoder.metadata.partial - sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) - fill!(sym_bytes, 0) - sym_str_bytes = Vector{UInt8}(sym) - copy_len = min(length(sym_str_bytes), symbol_cstr_len - 1) - if copy_len > 0 - sym_bytes[1:copy_len] = sym_str_bytes[1:copy_len] - end - write(metadata_buf, sym_bytes) - end - - # Not found symbols - write(metadata_buf, htol(UInt32(length(encoder.metadata.not_found)))) - for sym in encoder.metadata.not_found - sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) - fill!(sym_bytes, 0) - sym_str_bytes = Vector{UInt8}(sym) - copy_len = min(length(sym_str_bytes), symbol_cstr_len - 1) - if copy_len > 0 - sym_bytes[1:copy_len] = sym_str_bytes[1:copy_len] - end - write(metadata_buf, sym_bytes) - end - - # Symbol mappings - mappings holds one tuple per (raw symbol, interval), so - # group consecutive tuples sharing a raw symbol into one entry with N intervals - mapping_groups = Tuple{String,Vector{Tuple{String,Int64,Int64}}}[] - for (raw_symbol, mapped_symbol, start_date, end_date) in encoder.metadata.mappings - if isempty(mapping_groups) || mapping_groups[end][1] != raw_symbol - push!(mapping_groups, (raw_symbol, Tuple{String,Int64,Int64}[])) - end - push!(mapping_groups[end][2], (mapped_symbol, start_date, end_date)) - end - write(metadata_buf, htol(UInt32(length(mapping_groups)))) - for (raw_symbol, intervals) in mapping_groups - # Raw symbol (fixed length) - raw_sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) - fill!(raw_sym_bytes, 0) - raw_str_bytes = Vector{UInt8}(raw_symbol) - copy_len = min(length(raw_str_bytes), symbol_cstr_len - 1) - if copy_len > 0 - raw_sym_bytes[1:copy_len] = raw_str_bytes[1:copy_len] - end - write(metadata_buf, raw_sym_bytes) - - # Intervals count - write(metadata_buf, htol(UInt32(length(intervals)))) - - for (mapped_symbol, start_date, end_date) in intervals - # Start date (4 bytes) - write(metadata_buf, htol(UInt32(start_date))) - - # End date (4 bytes) - write(metadata_buf, htol(UInt32(end_date))) - - # Mapped symbol (fixed length) - mapped_sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) - fill!(mapped_sym_bytes, 0) - mapped_str_bytes = Vector{UInt8}(mapped_symbol) - copy_len = min(length(mapped_str_bytes), symbol_cstr_len - 1) - if copy_len > 0 - mapped_sym_bytes[1:copy_len] = mapped_str_bytes[1:copy_len] - end - write(metadata_buf, mapped_sym_bytes) - end - end - - # Get metadata bytes and write length + metadata - metadata_bytes = take!(metadata_buf) - write(io, htol(UInt32(length(metadata_bytes)))) - write(io, metadata_bytes) -end - -""" - write_record_header(io::IO, hd::RecordHeader) - -Write a record header to the output stream. - -# Arguments -- `io::IO`: Output stream -- `hd::RecordHeader`: Record header to write - -# Details -Writes the standard DBN record header fields in binary format: -- Length (1 byte) -- Record type (1 byte) -- Publisher ID (2 bytes) -- Instrument ID (4 bytes) -- Event timestamp (8 bytes) -""" -function write_record_header(io::IO, hd::RecordHeader) - # Length field is already in 4-byte units, write directly - write(io, hd.length) - write(io, UInt8(hd.rtype)) - write(io, hd.publisher_id) - write(io, hd.instrument_id) - write(io, hd.ts_event) -end - -""" - write_fixed_string(io::IO, s::String, len::Int) - -Write a fixed-length string with null padding to the output stream. - -# Arguments -- `io::IO`: Output stream -- `s::String`: String to write -- `len::Int`: Fixed length to write (in bytes) - -# Details -Writes exactly `len` bytes, truncating the string if too long or padding -with null bytes if too short. This ensures fixed-width fields in the -binary format. -""" -function write_fixed_string(io::IO, s::String, len::Int) - bytes = Vector{UInt8}(undef, len) - fill!(bytes, 0) # Fill with null bytes - s_bytes = Vector{UInt8}(s) - copy_len = min(length(s_bytes), len) - if copy_len > 0 - bytes[1:copy_len] = s_bytes[1:copy_len] - end - write(io, bytes) -end - -""" - write_record(encoder::DBNEncoder, record) - -Write a complete record to the DBN stream. - -# Arguments -- `encoder::DBNEncoder`: Encoder instance -- `record`: Record to write (any DBN message type) - -# Details -Writes the complete record including header and body based on the record type. -Supports all DBN v3 record types: -- Market data: MBOMsg, TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg -- Status: StatusMsg, ImbalanceMsg, StatMsg -- System: ErrorMsg, SymbolMappingMsg, SystemMsg -- Definition: InstrumentDefMsg -- Consolidated: CMBP1Msg, CBBO1sMsg, CBBO1mMsg, TCBBOMsg, BBO1sMsg, BBO1mMsg - -Each record type is serialized according to its specific binary layout. -""" -# Optimized write for simple bitstypes - direct memory write -@inline function write_record(encoder::DBNEncoder, record::Union{TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg, StatusMsg, ImbalanceMsg}) - unsafe_write(encoder.io, Ref(record), sizeof(record)) -end - -# Specialized optimized write for MBOMsg -@inline function write_record(encoder::DBNEncoder, record::MBOMsg) - # Wire order == struct order (official DBN MboMsg): hd -> order_id -> price -> size -> flags - # -> channel_id -> action -> side -> ts_recv -> ts_in_delta -> sequence. (Through 0.1.6 this - # writer mirrored the decoder's swapped order_id/ts_recv/price layout; see CHANGELOG.) - # - # Performance: This IOBuffer approach achieves 1.4x speedup (40% faster) compared to field-by-field write() - # by batching all fields into a buffer and performing a single write operation (2.1M vs 1.5M records/sec). - # The reduction in IO syscalls more than compensates for the temporary 56-byte allocation per record. - - # Batch all fields into a buffer, then write in one operation - buffer = IOBuffer() - - # Write header (16 bytes) - write(buffer, record.hd.length) - write(buffer, UInt8(record.hd.rtype)) - write(buffer, record.hd.publisher_id) - write(buffer, record.hd.instrument_id) - write(buffer, record.hd.ts_event) - - # Write body in binary order (40 bytes) - write(buffer, record.order_id) - write(buffer, record.price) - write(buffer, record.size) - write(buffer, record.flags) - write(buffer, record.channel_id) - write(buffer, UInt8(record.action)) - write(buffer, UInt8(record.side)) - write(buffer, record.ts_recv) - write(buffer, record.ts_in_delta) - write(buffer, record.sequence) - - # Write entire buffer in one operation to the encoder's IO - bytes = take!(buffer) - write(encoder.io, bytes) -end - -# Catch-all for types with variable-length fields (strings, etc.) -function write_record(encoder::DBNEncoder, record) - write_record_complex(encoder, record) -end - -# Write a Julia String into a fixed-width null-padded char array of `n` bytes. -# Truncates if too long; pads with NULs if too short. Used for SymbolMappingMsg -# and other records with fixed-length C-string fields. -@inline function _write_fixed_string(io::IO, s::AbstractString, n::Int) - bytes = Vector{UInt8}(String(s)) - if length(bytes) >= n - write(io, view(bytes, 1:n)) - else - write(io, bytes) - write(io, zeros(UInt8, n - length(bytes))) - end - return nothing -end - -# Fallback for complex types that need field-by-field writing -function write_record_complex(encoder::DBNEncoder, record) - io = encoder.io - - if isa(record, TradeMsg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - elseif isa(record, MBP1Msg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write level - write(io, record.levels.bid_px) - write(io, record.levels.ask_px) - write(io, record.levels.bid_sz) - write(io, record.levels.ask_sz) - write(io, record.levels.bid_ct) - write(io, record.levels.ask_ct) - - elseif isa(record, MBP10Msg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write levels - for level in record.levels - write(io, level.bid_px) - write(io, level.ask_px) - write(io, level.bid_sz) - write(io, level.ask_sz) - write(io, level.bid_ct) - write(io, level.ask_ct) - end - - elseif isa(record, OHLCVMsg) - write_record_header(io, record.hd) - write(io, record.open) - write(io, record.high) - write(io, record.low) - write(io, record.close) - write(io, record.volume) - - elseif isa(record, StatusMsg) - write_record_header(io, record.hd) - write(io, record.ts_recv) - write(io, record.action) - write(io, record.reason) - write(io, record.trading_event) - write(io, record.is_trading) - write(io, record.is_quoting) - write(io, record.is_short_sell_restricted) - write(io, zeros(UInt8, 7)) # Reserved (adjusted) - - elseif isa(record, InstrumentDefMsg) - write_record_header(io, record.hd) - # V2 and V3 have COMPLETELY different structures! - - if encoder.metadata.version == 2 - # ===== DBN V2 InstrumentDefMsg ===== - # V2 has encode_order for: ts_recv(0), raw_symbol(2), security_update_action(3), instrument_class(4), strike_price(46) - - # encode_order 0: ts_recv - write(io, record.ts_recv) - - # encode_order 2: raw_symbol (19 bytes in v2) - write_fixed_string(io, record.raw_symbol, 19) - - # encode_order 3: security_update_action - write(io, UInt8(record.security_update_action)) - - # encode_order 4: instrument_class - write(io, UInt8(record.instrument_class)) - - # encode_order 46: strike_price - write(io, record.strike_price) - - # All remaining fields in struct declaration order (no more encode_order) - write(io, record.min_price_increment) - write(io, record.display_factor) - write(io, record.expiration) - write(io, record.activation) - write(io, record.high_limit_price) - write(io, record.low_limit_price) - write(io, record.max_price_variation) - write(io, record.trading_reference_price) # v2 only - write(io, record.unit_of_measure_qty) - write(io, record.min_price_increment_amount) - write(io, record.price_ratio) - - write(io, record.inst_attrib_value) - write(io, record.underlying_id) - write(io, UInt32(record.raw_instrument_id)) # u32 in v2, u64 in v3! - write(io, record.market_depth_implied) - write(io, record.market_depth) - write(io, record.market_segment_id) - write(io, record.max_trade_vol) - write(io, record.min_lot_size) - write(io, record.min_lot_size_block) - write(io, record.min_lot_size_round_lot) - write(io, record.min_trade_vol) - write(io, record.contract_multiplier) - write(io, record.decay_quantity) - write(io, record.original_contract_size) - - write(io, record.trading_reference_date) # v2 only - write(io, record.appl_id) - write(io, record.maturity_year) - write(io, record.decay_start_date) - write(io, record.channel_id) - - # String fields (in struct order, but raw_symbol already written with encode_order(2)) - write_fixed_string(io, record.currency, 4) - write_fixed_string(io, record.settl_currency, 4) - write_fixed_string(io, record.secsubtype, 6) - # raw_symbol already written with encode_order(2) - write_fixed_string(io, record.group, 21) - write_fixed_string(io, record.exchange, 5) - write_fixed_string(io, record.asset, 7) # 7 bytes in v2, 11 in v3! - write_fixed_string(io, record.cfi, 7) - write_fixed_string(io, record.security_type, 7) - write_fixed_string(io, record.unit_of_measure, 31) - write_fixed_string(io, record.underlying, 21) - write_fixed_string(io, record.strike_price_currency, 4) - - # instrument_class and strike_price already written with encode_order(4) and encode_order(46) - - # Single-byte fields (in struct order, but security_update_action and instrument_class already written) - write(io, UInt8(record.match_algorithm)) - write(io, record.md_security_trading_status) # v2 only - write(io, record.main_fraction) - write(io, record.price_display_format) - write(io, record.settl_price_type) # v2 only - write(io, record.sub_fraction) - write(io, record.underlying_product) - # security_update_action already written with encode_order(3) - write(io, record.maturity_month) - write(io, record.maturity_day) - write(io, record.maturity_week) - write(io, record.user_defined_instrument ? UInt8('Y') : UInt8('N')) - write(io, record.contract_multiplier_unit) - write(io, record.flow_schedule_type) - write(io, record.tick_rule) - - # v2: 62 bytes _reserved (322 bytes written, 384 total, 62 remaining) - for _ in 1:62 - write(io, UInt8(0)) - end - - else # v3 - # ===== DBN V3 InstrumentDefMsg ===== - # DBN binary records are encoded in the fixed binary field layout. - # `encode_order` controls text/field ordering, not the binary layout. - write(io, record.ts_recv) - write(io, record.min_price_increment) - write(io, record.display_factor) - write(io, record.expiration) - write(io, record.activation) - write(io, record.high_limit_price) - write(io, record.low_limit_price) - write(io, record.max_price_variation) - write(io, record.unit_of_measure_qty) - write(io, record.min_price_increment_amount) - write(io, record.price_ratio) - write(io, record.strike_price) - write(io, record.raw_instrument_id) - write(io, record.leg_price) - write(io, record.leg_delta) - - write(io, record.inst_attrib_value) - write(io, record.underlying_id) - write(io, record.market_depth_implied) - write(io, record.market_depth) - write(io, record.market_segment_id) - write(io, record.max_trade_vol) - write(io, record.min_lot_size) - write(io, record.min_lot_size_block) - write(io, record.min_lot_size_round_lot) - write(io, record.min_trade_vol) - write(io, record.contract_multiplier) - write(io, record.decay_quantity) - write(io, record.original_contract_size) - write(io, record.leg_instrument_id) - write(io, record.leg_ratio_price_numerator) - write(io, record.leg_ratio_price_denominator) - write(io, record.leg_ratio_qty_numerator) - write(io, record.leg_ratio_qty_denominator) - write(io, record.leg_underlying_id) - - write(io, record.appl_id) - write(io, record.maturity_year) - write(io, record.decay_start_date) - write(io, record.channel_id) - write(io, record.leg_count) - write(io, record.leg_index) - - # String fields in struct declaration order. - write_fixed_string(io, record.currency, 4) - write_fixed_string(io, record.settl_currency, 4) - write_fixed_string(io, record.secsubtype, 6) - write_fixed_string(io, record.raw_symbol, SYMBOL_CSTR_LEN) - write_fixed_string(io, record.group, 21) - write_fixed_string(io, record.exchange, 5) - write_fixed_string(io, record.asset, 11) # 11 bytes in v3! - write_fixed_string(io, record.cfi, 7) - write_fixed_string(io, record.security_type, 7) - write_fixed_string(io, record.unit_of_measure, 31) - write_fixed_string(io, record.underlying, 21) - write_fixed_string(io, record.strike_price_currency, 4) - write_fixed_string(io, record.leg_raw_symbol, SYMBOL_CSTR_LEN) - - # Single-byte fields without encode_order - write(io, UInt8(record.instrument_class)) - write(io, UInt8(record.match_algorithm)) - write(io, record.main_fraction) - write(io, record.price_display_format) - write(io, record.sub_fraction) - write(io, record.underlying_product) - write(io, UInt8(record.security_update_action)) - write(io, record.maturity_month) - write(io, record.maturity_day) - write(io, record.maturity_week) - write(io, record.user_defined_instrument ? UInt8('Y') : UInt8('N')) - write(io, record.contract_multiplier_unit) - write(io, record.flow_schedule_type) - write(io, record.tick_rule) - write(io, UInt8(record.leg_instrument_class)) - write(io, UInt8(record.leg_side)) - - # v3: 17 bytes _reserved - for _ in 1:17 - write(io, UInt8(0)) - end - end - - elseif isa(record, ImbalanceMsg) - write_record_header(io, record.hd) - write(io, record.ts_recv) - write(io, record.ref_price) - write(io, record.auction_time) - write(io, record.cont_book_clr_price) - write(io, record.auct_interest_clr_price) - write(io, record.ssr_filling_price) - write(io, record.ind_match_price) - write(io, record.upper_collar) - write(io, record.lower_collar) - write(io, record.paired_qty) - write(io, record.total_imbalance_qty) - write(io, record.market_imbalance_qty) - write(io, record.unpaired_qty) - write(io, record.auction_type) - write(io, UInt8(record.side)) - write(io, record.auction_status) - write(io, record.freeze_status) - write(io, record.num_extensions) - write(io, record.unpaired_side) - write(io, record.significant_imbalance) - write(io, zeros(UInt8, 1)) # Reserved - - elseif isa(record, StatMsg) - write_record_header(io, record.hd) - write(io, record.ts_recv) - write(io, record.ts_ref) - write(io, record.price) - # quantity is a signed Int64 on the wire; the v3 UNDEF sentinel is typemax(Int64) - # (0x7fff...). Through 0.1.6 this wrote 0xffffffffffffffff (-1) for UNDEF instead. - write(io, record.quantity) - write(io, record.sequence) - write(io, record.ts_in_delta) - write(io, record.stat_type) - write(io, record.channel_id) - write(io, record.update_action) - write(io, record.stat_flags) - write(io, zeros(UInt8, 18)) # Reserved (adjusted for field size changes) - - elseif isa(record, CMBP1Msg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write level - write(io, record.levels.bid_px) - write(io, record.levels.ask_px) - write(io, record.levels.bid_sz) - write(io, record.levels.ask_sz) - write(io, record.levels.bid_ct) - write(io, record.levels.ask_ct) - - elseif isa(record, CBBO1sMsg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write level - write(io, record.levels.bid_px) - write(io, record.levels.ask_px) - write(io, record.levels.bid_sz) - write(io, record.levels.ask_sz) - write(io, record.levels.bid_ct) - write(io, record.levels.ask_ct) - - elseif isa(record, CBBO1mMsg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write level - write(io, record.levels.bid_px) - write(io, record.levels.ask_px) - write(io, record.levels.bid_sz) - write(io, record.levels.ask_sz) - write(io, record.levels.bid_ct) - write(io, record.levels.ask_ct) - - elseif isa(record, TCBBOMsg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write level - write(io, record.levels.bid_px) - write(io, record.levels.ask_px) - write(io, record.levels.bid_sz) - write(io, record.levels.ask_sz) - write(io, record.levels.bid_ct) - write(io, record.levels.ask_ct) - - elseif isa(record, BBO1sMsg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write level - write(io, record.levels.bid_px) - write(io, record.levels.ask_px) - write(io, record.levels.bid_sz) - write(io, record.levels.ask_sz) - write(io, record.levels.bid_ct) - write(io, record.levels.ask_ct) - - elseif isa(record, BBO1mMsg) - write_record_header(io, record.hd) - write(io, record.price) - write(io, record.size) - write(io, UInt8(record.action)) - write(io, UInt8(record.side)) - write(io, record.flags) - write(io, record.depth) - write(io, record.ts_recv) - write(io, record.ts_in_delta) - write(io, record.sequence) - - # Write level - write(io, record.levels.bid_px) - write(io, record.levels.ask_px) - write(io, record.levels.bid_sz) - write(io, record.levels.ask_sz) - write(io, record.levels.bid_ct) - write(io, record.levels.ask_ct) - - elseif isa(record, ErrorMsg) - write_record_header(io, record.hd) - # Write error message string with null terminator, padding to fill the - # payload size implied by hd.length (in 4-byte units). - target = Int(record.hd.length) * LENGTH_MULTIPLIER - 16 - err_bytes = Vector{UInt8}(record.err) - written = 0 - write(io, err_bytes); written += length(err_bytes) - if length(err_bytes) == 0 || err_bytes[end] != 0 - write(io, UInt8(0)); written += 1 - end - if written < target - write(io, zeros(UInt8, target - written)) - end - - - elseif isa(record, SymbolMappingMsg) - # Spec-compliant SymbolMappingMsg layout (depends on DBN version): - # v1: stype_in_symbol[22] | stype_out_symbol[22] | pad(4) | start_ts(8) | end_ts(8) (body 64, total 80, hd.length 20) - # v2+: stype_in(1) | stype_in_symbol[71] | stype_out(1) | stype_out_symbol[71] | - # start_ts(8) | end_ts(8) (body 160, total 176, hd.length 44) - # - # The record's hd.length may reflect a different on-wire version than - # the file we're writing into — the Databento Live gateway emits v1 - # layout (hd.length = 20) even when the consumer is writing a v3 file. - # Re-derive the length from the layout we are about to write so the - # resulting record header matches the bytes that follow it. - body_bytes = encoder.metadata.version == 1 ? 64 : 160 - fixed_length = UInt8((16 + body_bytes) ÷ LENGTH_MULTIPLIER) - hd = record.hd - out_hd = hd.length == fixed_length ? hd : - RecordHeader(fixed_length, hd.rtype, hd.publisher_id, hd.instrument_id, hd.ts_event) - write_record_header(io, out_hd) - if encoder.metadata.version == 1 - _write_fixed_string(io, record.stype_in_symbol, 22) - _write_fixed_string(io, record.stype_out_symbol, 22) - write(io, zeros(UInt8, 4)) # padding for 8-byte ts alignment - write(io, record.start_ts) - write(io, record.end_ts) - else - sym_len = 71 - write(io, UInt8(record.stype_in)) - _write_fixed_string(io, record.stype_in_symbol, sym_len) - write(io, UInt8(record.stype_out)) - _write_fixed_string(io, record.stype_out_symbol, sym_len) - write(io, record.start_ts) - write(io, record.end_ts) - end - - - elseif isa(record, SystemMsg) - write_record_header(io, record.hd) - # Write msg + null + code + null, padded to the payload size implied by - # hd.length (in 4-byte units). - target = Int(record.hd.length) * LENGTH_MULTIPLIER - 16 - msg_bytes = Vector{UInt8}(record.msg) - code_bytes = Vector{UInt8}(record.code) - write(io, msg_bytes); written = length(msg_bytes) - write(io, UInt8(0)); written += 1 - write(io, code_bytes); written += length(code_bytes) - if length(code_bytes) == 0 || code_bytes[end] != 0 - write(io, UInt8(0)); written += 1 - end - if written < target - write(io, zeros(UInt8, target - written)) - end - end -end - -# Add finalize function for encoder -""" - finalize_encoder(encoder::DBNEncoder) - -Finalize the encoder and flush any remaining data. - -# Arguments -- `encoder::DBNEncoder`: Encoder to finalize - -# Details -Ensures all buffered data is written to the output stream. -Should be called when finished writing all records. -""" -function finalize_encoder(encoder::DBNEncoder) - # For now, we don't use compression in write mode for simplicity - # In the future, compression support could be added here -end - -# Convenience function -""" - write_dbn(filename::String, metadata::Metadata, records) - -Convenience function to write a complete DBN file with automatic compression support. - -# Arguments -- `filename::String`: Output file path (use .zst extension for compression) -- `metadata::Metadata`: File metadata -- `records`: Collection of records to write - -# Details -Creates a complete DBN file with header and all records. -Automatically handles: -- File creation and management -- Zstd compression (when filename ends with .zst) -- Header writing (uncompressed for format detection) -- Record serialization -- Resource cleanup - -# Example -```julia -metadata = Metadata(3, "TEST", Schema.TRADES, start_ts, end_ts, length(records), - SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, symbols, [], [], []) -# Uncompressed -write_dbn("output.dbn", metadata, records) -# Compressed -write_dbn("output.dbn.zst", metadata, records) -``` -""" -function write_dbn(filename::String, metadata::Metadata, records) - # Check if compression is needed - use_compression = endswith(filename, ".zst") - - base_io = open(filename, "w") - - try - if use_compression - # Wrap the entire stream with compression (including header) - compressed_io = TranscodingStream(ZstdCompressor(), base_io) - - try - encoder = DBNEncoder(compressed_io, base_io, metadata, nothing) - write_header(encoder) - - # Write all records - for record in records - write_record(encoder, record) - end - - finalize_encoder(encoder) - finally - # Close the compression stream - close(compressed_io) - end - else - # Uncompressed: write directly to file - encoder = DBNEncoder(base_io, metadata) - write_header(encoder) - - # Write all records - for record in records - write_record(encoder, record) - end - - finalize_encoder(encoder) - end - finally - # Always close the base IO - if isopen(base_io) - close(base_io) - end - end -end +# DBN encoding functionality + +""" + DBNEncoder + +Encoder for writing DBN (Databento Binary Encoding) files with optional compression. + +# Fields +- `io::IO`: Current IO stream (may be wrapped with compression) +- `base_io::IO`: Original IO stream before any compression wrapper +- `metadata::Metadata`: Metadata for the DBN file +- `compressed_buffer::Union{IOBuffer,Nothing}`: Buffer for compressed data (if applicable) +""" +mutable struct DBNEncoder + io::IO + base_io::IO # Original IO before compression wrapper + metadata::Metadata + compressed_buffer::Union{IOBuffer,Nothing} +end + +""" + DBNEncoder(io::IO, metadata::Metadata) + +Construct a DBNEncoder for writing to an IO stream. + +# Arguments +- `io::IO`: Output stream to write to +- `metadata::Metadata`: Metadata information for the DBN file + +# Returns +- `DBNEncoder`: Encoder instance ready for writing +""" +DBNEncoder(io::IO, metadata::Metadata) = DBNEncoder(io, io, metadata, nothing) + +""" + write_header(encoder::DBNEncoder) + +Write the DBN file header including magic bytes, version, and metadata. + +# Arguments +- `encoder::DBNEncoder`: Encoder instance containing metadata to write + +# Details +Writes the complete DBN header in the correct binary format: +- Magic bytes "DBN" +- Version number +- Metadata length +- Complete metadata section with all fields + +Writes to the current IO stream (encoder.io), which may be compressed or uncompressed. +For compressed files (.zst), the entire file including the header is compressed. +""" +function write_header(encoder::DBNEncoder) + # Write header to the current IO stream (compressed or uncompressed) + io = encoder.io + + # Write magic bytes "DBN" + write(io, b"DBN") + + # Write version + write(io, UInt8(DBN_VERSION)) + + # Create metadata buffer to calculate size + metadata_buf = IOBuffer() + + # Write metadata fields in the exact format that read_header! expects + + # Dataset (16 bytes fixed-length C string) + dataset_bytes = Vector{UInt8}(undef, 16) + fill!(dataset_bytes, 0) + dataset_str_bytes = Vector{UInt8}(encoder.metadata.dataset) + copy_len = min(length(dataset_str_bytes), 15) # Leave room for null terminator + if copy_len > 0 + dataset_bytes[1:copy_len] = dataset_str_bytes[1:copy_len] + end + write(metadata_buf, dataset_bytes) + + # Schema (2 bytes) + write(metadata_buf, htol(UInt16(encoder.metadata.schema))) + + # Start timestamp (8 bytes) + write(metadata_buf, htol(UInt64(encoder.metadata.start_ts))) + + # End timestamp (8 bytes) + end_ts = encoder.metadata.end_ts === nothing ? 0 : UInt64(encoder.metadata.end_ts) + write(metadata_buf, htol(end_ts)) + + # Limit (8 bytes) + limit = encoder.metadata.limit === nothing ? 0 : encoder.metadata.limit + write(metadata_buf, htol(UInt64(limit))) + + # NOTE: For version > 1, we DON'T write record_count (8 bytes) here + # This is skipped in the read function for version > 1 + + # SType in (1 byte) + stype_in_val = encoder.metadata.stype_in === nothing ? 0xFF : UInt8(encoder.metadata.stype_in) + write(metadata_buf, stype_in_val) + + # SType out (1 byte) + write(metadata_buf, UInt8(encoder.metadata.stype_out)) + + # TS out (1 byte boolean) + write(metadata_buf, encoder.metadata.ts_out ? UInt8(1) : UInt8(0)) + + # Symbol string length (2 bytes) - only for version > 1 + # DBN v2/v3 use 71-byte symbol strings. + symbol_cstr_len = UInt16(SYMBOL_CSTR_LEN) + write(metadata_buf, htol(symbol_cstr_len)) + + # Reserved padding (53 bytes for v3) + write(metadata_buf, zeros(UInt8, 53)) + + # Schema definition length (4 bytes) - always 0 for now + write(metadata_buf, htol(UInt32(0))) + + # Variable-length sections + + # Symbols + write(metadata_buf, htol(UInt32(length(encoder.metadata.symbols)))) + for sym in encoder.metadata.symbols + sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) + fill!(sym_bytes, 0) + sym_str_bytes = Vector{UInt8}(sym) + copy_len = min(length(sym_str_bytes), symbol_cstr_len - 1) + if copy_len > 0 + sym_bytes[1:copy_len] = sym_str_bytes[1:copy_len] + end + write(metadata_buf, sym_bytes) + end + + # Partial symbols + write(metadata_buf, htol(UInt32(length(encoder.metadata.partial)))) + for sym in encoder.metadata.partial + sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) + fill!(sym_bytes, 0) + sym_str_bytes = Vector{UInt8}(sym) + copy_len = min(length(sym_str_bytes), symbol_cstr_len - 1) + if copy_len > 0 + sym_bytes[1:copy_len] = sym_str_bytes[1:copy_len] + end + write(metadata_buf, sym_bytes) + end + + # Not found symbols + write(metadata_buf, htol(UInt32(length(encoder.metadata.not_found)))) + for sym in encoder.metadata.not_found + sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) + fill!(sym_bytes, 0) + sym_str_bytes = Vector{UInt8}(sym) + copy_len = min(length(sym_str_bytes), symbol_cstr_len - 1) + if copy_len > 0 + sym_bytes[1:copy_len] = sym_str_bytes[1:copy_len] + end + write(metadata_buf, sym_bytes) + end + + # Symbol mappings - mappings holds one tuple per (raw symbol, interval), so + # group consecutive tuples sharing a raw symbol into one entry with N intervals + mapping_groups = Tuple{String,Vector{Tuple{String,Int64,Int64}}}[] + for (raw_symbol, mapped_symbol, start_date, end_date) in encoder.metadata.mappings + if isempty(mapping_groups) || mapping_groups[end][1] != raw_symbol + push!(mapping_groups, (raw_symbol, Tuple{String,Int64,Int64}[])) + end + push!(mapping_groups[end][2], (mapped_symbol, start_date, end_date)) + end + write(metadata_buf, htol(UInt32(length(mapping_groups)))) + for (raw_symbol, intervals) in mapping_groups + # Raw symbol (fixed length) + raw_sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) + fill!(raw_sym_bytes, 0) + raw_str_bytes = Vector{UInt8}(raw_symbol) + copy_len = min(length(raw_str_bytes), symbol_cstr_len - 1) + if copy_len > 0 + raw_sym_bytes[1:copy_len] = raw_str_bytes[1:copy_len] + end + write(metadata_buf, raw_sym_bytes) + + # Intervals count + write(metadata_buf, htol(UInt32(length(intervals)))) + + for (mapped_symbol, start_date, end_date) in intervals + # Start date (4 bytes) + write(metadata_buf, htol(UInt32(start_date))) + + # End date (4 bytes) + write(metadata_buf, htol(UInt32(end_date))) + + # Mapped symbol (fixed length) + mapped_sym_bytes = Vector{UInt8}(undef, symbol_cstr_len) + fill!(mapped_sym_bytes, 0) + mapped_str_bytes = Vector{UInt8}(mapped_symbol) + copy_len = min(length(mapped_str_bytes), symbol_cstr_len - 1) + if copy_len > 0 + mapped_sym_bytes[1:copy_len] = mapped_str_bytes[1:copy_len] + end + write(metadata_buf, mapped_sym_bytes) + end + end + + # Get metadata bytes and write length + metadata + metadata_bytes = take!(metadata_buf) + write(io, htol(UInt32(length(metadata_bytes)))) + write(io, metadata_bytes) +end + +""" + write_record_header(io::IO, hd::RecordHeader) + +Write a record header to the output stream. + +# Arguments +- `io::IO`: Output stream +- `hd::RecordHeader`: Record header to write + +# Details +Writes the standard DBN record header fields in binary format: +- Length (1 byte) +- Record type (1 byte) +- Publisher ID (2 bytes) +- Instrument ID (4 bytes) +- Event timestamp (8 bytes) +""" +function write_record_header(io::IO, hd::RecordHeader) + # Length field is already in 4-byte units, write directly + write(io, hd.length) + write(io, UInt8(hd.rtype)) + write(io, hd.publisher_id) + write(io, hd.instrument_id) + write(io, hd.ts_event) +end + +""" + write_fixed_string(io::IO, s::String, len::Int) + +Write a fixed-length string with null padding to the output stream. + +# Arguments +- `io::IO`: Output stream +- `s::String`: String to write +- `len::Int`: Fixed length to write (in bytes) + +# Details +Writes exactly `len` bytes, truncating the string if too long or padding +with null bytes if too short. This ensures fixed-width fields in the +binary format. +""" +function write_fixed_string(io::IO, s::String, len::Int) + bytes = Vector{UInt8}(undef, len) + fill!(bytes, 0) # Fill with null bytes + s_bytes = Vector{UInt8}(s) + copy_len = min(length(s_bytes), len) + if copy_len > 0 + bytes[1:copy_len] = s_bytes[1:copy_len] + end + write(io, bytes) +end + +""" + write_record(encoder::DBNEncoder, record) + +Write a complete record to the DBN stream. + +# Arguments +- `encoder::DBNEncoder`: Encoder instance +- `record`: Record to write (any DBN message type) + +# Details +Writes the complete record including header and body based on the record type. +Supports all DBN v3 record types: +- Market data: MBOMsg, TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg +- Status: StatusMsg, ImbalanceMsg, StatMsg +- System: ErrorMsg, SymbolMappingMsg, SystemMsg +- Definition: InstrumentDefMsg +- Consolidated: CMBP1Msg, CBBO1sMsg, CBBO1mMsg, TCBBOMsg, BBO1sMsg, BBO1mMsg + +Each record type is serialized according to its specific binary layout. +""" +# Optimized write for simple bitstypes - direct memory write +@inline function write_record(encoder::DBNEncoder, record::Union{TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg, StatusMsg, ImbalanceMsg}) + unsafe_write(encoder.io, Ref(record), sizeof(record)) +end + +# Specialized optimized write for MBOMsg +@inline function write_record(encoder::DBNEncoder, record::MBOMsg) + # Wire order == struct order (official DBN MboMsg): hd -> order_id -> price -> size -> flags + # -> channel_id -> action -> side -> ts_recv -> ts_in_delta -> sequence. (Through 0.1.6 this + # writer mirrored the decoder's swapped order_id/ts_recv/price layout; see CHANGELOG.) + # + # Performance: This IOBuffer approach achieves 1.4x speedup (40% faster) compared to field-by-field write() + # by batching all fields into a buffer and performing a single write operation (2.1M vs 1.5M records/sec). + # The reduction in IO syscalls more than compensates for the temporary 56-byte allocation per record. + + # Batch all fields into a buffer, then write in one operation + buffer = IOBuffer() + + # Write header (16 bytes) + write(buffer, record.hd.length) + write(buffer, UInt8(record.hd.rtype)) + write(buffer, record.hd.publisher_id) + write(buffer, record.hd.instrument_id) + write(buffer, record.hd.ts_event) + + # Write body in binary order (40 bytes) + write(buffer, record.order_id) + write(buffer, record.price) + write(buffer, record.size) + write(buffer, record.flags) + write(buffer, record.channel_id) + write(buffer, UInt8(record.action)) + write(buffer, UInt8(record.side)) + write(buffer, record.ts_recv) + write(buffer, record.ts_in_delta) + write(buffer, record.sequence) + + # Write entire buffer in one operation to the encoder's IO + bytes = take!(buffer) + write(encoder.io, bytes) +end + +# Catch-all for types with variable-length fields (strings, etc.) +function write_record(encoder::DBNEncoder, record) + write_record_complex(encoder, record) +end + +# Write a Julia String into a fixed-width null-padded char array of `n` bytes. +# Truncates if too long; pads with NULs if too short. Used for SymbolMappingMsg +# and other records with fixed-length C-string fields. +@inline function _write_fixed_string(io::IO, s::AbstractString, n::Int) + bytes = Vector{UInt8}(String(s)) + if length(bytes) >= n + write(io, view(bytes, 1:n)) + else + write(io, bytes) + write(io, zeros(UInt8, n - length(bytes))) + end + return nothing +end + +# Fallback for complex types that need field-by-field writing +function write_record_complex(encoder::DBNEncoder, record) + io = encoder.io + + if isa(record, TradeMsg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + elseif isa(record, MBP1Msg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write level + write(io, record.levels.bid_px) + write(io, record.levels.ask_px) + write(io, record.levels.bid_sz) + write(io, record.levels.ask_sz) + write(io, record.levels.bid_ct) + write(io, record.levels.ask_ct) + + elseif isa(record, MBP10Msg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write levels + for level in record.levels + write(io, level.bid_px) + write(io, level.ask_px) + write(io, level.bid_sz) + write(io, level.ask_sz) + write(io, level.bid_ct) + write(io, level.ask_ct) + end + + elseif isa(record, OHLCVMsg) + write_record_header(io, record.hd) + write(io, record.open) + write(io, record.high) + write(io, record.low) + write(io, record.close) + write(io, record.volume) + + elseif isa(record, StatusMsg) + write_record_header(io, record.hd) + write(io, record.ts_recv) + write(io, record.action) + write(io, record.reason) + write(io, record.trading_event) + write(io, record.is_trading) + write(io, record.is_quoting) + write(io, record.is_short_sell_restricted) + write(io, zeros(UInt8, 7)) # Reserved (adjusted) + + elseif isa(record, InstrumentDefMsg) + write_record_header(io, record.hd) + # V2 and V3 have COMPLETELY different structures! + + if encoder.metadata.version == 2 + # ===== DBN V2 InstrumentDefMsg ===== + # V2 has encode_order for: ts_recv(0), raw_symbol(2), security_update_action(3), instrument_class(4), strike_price(46) + + # encode_order 0: ts_recv + write(io, record.ts_recv) + + # encode_order 2: raw_symbol (19 bytes in v2) + write_fixed_string(io, record.raw_symbol, 19) + + # encode_order 3: security_update_action + write(io, UInt8(record.security_update_action)) + + # encode_order 4: instrument_class + write(io, UInt8(record.instrument_class)) + + # encode_order 46: strike_price + write(io, record.strike_price) + + # All remaining fields in struct declaration order (no more encode_order) + write(io, record.min_price_increment) + write(io, record.display_factor) + write(io, record.expiration) + write(io, record.activation) + write(io, record.high_limit_price) + write(io, record.low_limit_price) + write(io, record.max_price_variation) + write(io, record.trading_reference_price) # v2 only + write(io, record.unit_of_measure_qty) + write(io, record.min_price_increment_amount) + write(io, record.price_ratio) + + write(io, record.inst_attrib_value) + write(io, record.underlying_id) + write(io, UInt32(record.raw_instrument_id)) # u32 in v2, u64 in v3! + write(io, record.market_depth_implied) + write(io, record.market_depth) + write(io, record.market_segment_id) + write(io, record.max_trade_vol) + write(io, record.min_lot_size) + write(io, record.min_lot_size_block) + write(io, record.min_lot_size_round_lot) + write(io, record.min_trade_vol) + write(io, record.contract_multiplier) + write(io, record.decay_quantity) + write(io, record.original_contract_size) + + write(io, record.trading_reference_date) # v2 only + write(io, record.appl_id) + write(io, record.maturity_year) + write(io, record.decay_start_date) + write(io, record.channel_id) + + # String fields (in struct order, but raw_symbol already written with encode_order(2)) + write_fixed_string(io, record.currency, 4) + write_fixed_string(io, record.settl_currency, 4) + write_fixed_string(io, record.secsubtype, 6) + # raw_symbol already written with encode_order(2) + write_fixed_string(io, record.group, 21) + write_fixed_string(io, record.exchange, 5) + write_fixed_string(io, record.asset, 7) # 7 bytes in v2, 11 in v3! + write_fixed_string(io, record.cfi, 7) + write_fixed_string(io, record.security_type, 7) + write_fixed_string(io, record.unit_of_measure, 31) + write_fixed_string(io, record.underlying, 21) + write_fixed_string(io, record.strike_price_currency, 4) + + # instrument_class and strike_price already written with encode_order(4) and encode_order(46) + + # Single-byte fields (in struct order, but security_update_action and instrument_class already written) + write(io, UInt8(record.match_algorithm)) + write(io, record.md_security_trading_status) # v2 only + write(io, record.main_fraction) + write(io, record.price_display_format) + write(io, record.settl_price_type) # v2 only + write(io, record.sub_fraction) + write(io, record.underlying_product) + # security_update_action already written with encode_order(3) + write(io, record.maturity_month) + write(io, record.maturity_day) + write(io, record.maturity_week) + write(io, record.user_defined_instrument ? UInt8('Y') : UInt8('N')) + write(io, record.contract_multiplier_unit) + write(io, record.flow_schedule_type) + write(io, record.tick_rule) + + # v2: 62 bytes _reserved (322 bytes written, 384 total, 62 remaining) + for _ in 1:62 + write(io, UInt8(0)) + end + + else # v3 + # ===== DBN V3 InstrumentDefMsg ===== + # DBN binary records are encoded in the fixed binary field layout. + # `encode_order` controls text/field ordering, not the binary layout. + write(io, record.ts_recv) + write(io, record.min_price_increment) + write(io, record.display_factor) + write(io, record.expiration) + write(io, record.activation) + write(io, record.high_limit_price) + write(io, record.low_limit_price) + write(io, record.max_price_variation) + write(io, record.unit_of_measure_qty) + write(io, record.min_price_increment_amount) + write(io, record.price_ratio) + write(io, record.strike_price) + write(io, record.raw_instrument_id) + write(io, record.leg_price) + write(io, record.leg_delta) + + write(io, record.inst_attrib_value) + write(io, record.underlying_id) + write(io, record.market_depth_implied) + write(io, record.market_depth) + write(io, record.market_segment_id) + write(io, record.max_trade_vol) + write(io, record.min_lot_size) + write(io, record.min_lot_size_block) + write(io, record.min_lot_size_round_lot) + write(io, record.min_trade_vol) + write(io, record.contract_multiplier) + write(io, record.decay_quantity) + write(io, record.original_contract_size) + write(io, record.leg_instrument_id) + write(io, record.leg_ratio_price_numerator) + write(io, record.leg_ratio_price_denominator) + write(io, record.leg_ratio_qty_numerator) + write(io, record.leg_ratio_qty_denominator) + write(io, record.leg_underlying_id) + + write(io, record.appl_id) + write(io, record.maturity_year) + write(io, record.decay_start_date) + write(io, record.channel_id) + write(io, record.leg_count) + write(io, record.leg_index) + + # String fields in struct declaration order. + write_fixed_string(io, record.currency, 4) + write_fixed_string(io, record.settl_currency, 4) + write_fixed_string(io, record.secsubtype, 6) + write_fixed_string(io, record.raw_symbol, SYMBOL_CSTR_LEN) + write_fixed_string(io, record.group, 21) + write_fixed_string(io, record.exchange, 5) + write_fixed_string(io, record.asset, 11) # 11 bytes in v3! + write_fixed_string(io, record.cfi, 7) + write_fixed_string(io, record.security_type, 7) + write_fixed_string(io, record.unit_of_measure, 31) + write_fixed_string(io, record.underlying, 21) + write_fixed_string(io, record.strike_price_currency, 4) + write_fixed_string(io, record.leg_raw_symbol, SYMBOL_CSTR_LEN) + + # Single-byte fields without encode_order + write(io, UInt8(record.instrument_class)) + write(io, UInt8(record.match_algorithm)) + write(io, record.main_fraction) + write(io, record.price_display_format) + write(io, record.sub_fraction) + write(io, record.underlying_product) + write(io, UInt8(record.security_update_action)) + write(io, record.maturity_month) + write(io, record.maturity_day) + write(io, record.maturity_week) + write(io, record.user_defined_instrument ? UInt8('Y') : UInt8('N')) + write(io, record.contract_multiplier_unit) + write(io, record.flow_schedule_type) + write(io, record.tick_rule) + write(io, UInt8(record.leg_instrument_class)) + write(io, UInt8(record.leg_side)) + + # v3: 17 bytes _reserved + for _ in 1:17 + write(io, UInt8(0)) + end + end + + elseif isa(record, ImbalanceMsg) + write_record_header(io, record.hd) + write(io, record.ts_recv) + write(io, record.ref_price) + write(io, record.auction_time) + write(io, record.cont_book_clr_price) + write(io, record.auct_interest_clr_price) + write(io, record.ssr_filling_price) + write(io, record.ind_match_price) + write(io, record.upper_collar) + write(io, record.lower_collar) + write(io, record.paired_qty) + write(io, record.total_imbalance_qty) + write(io, record.market_imbalance_qty) + write(io, record.unpaired_qty) + write(io, record.auction_type) + write(io, UInt8(record.side)) + write(io, record.auction_status) + write(io, record.freeze_status) + write(io, record.num_extensions) + write(io, record.unpaired_side) + write(io, record.significant_imbalance) + write(io, zeros(UInt8, 1)) # Reserved + + elseif isa(record, StatMsg) + write_record_header(io, record.hd) + write(io, record.ts_recv) + write(io, record.ts_ref) + write(io, record.price) + # quantity is a signed Int64 on the wire; the v3 UNDEF sentinel is typemax(Int64) + # (0x7fff...). Through 0.1.6 this wrote 0xffffffffffffffff (-1) for UNDEF instead. + write(io, record.quantity) + write(io, record.sequence) + write(io, record.ts_in_delta) + write(io, record.stat_type) + write(io, record.channel_id) + write(io, record.update_action) + write(io, record.stat_flags) + write(io, zeros(UInt8, 18)) # Reserved (adjusted for field size changes) + + elseif isa(record, CMBP1Msg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write level + write(io, record.levels.bid_px) + write(io, record.levels.ask_px) + write(io, record.levels.bid_sz) + write(io, record.levels.ask_sz) + write(io, record.levels.bid_ct) + write(io, record.levels.ask_ct) + + elseif isa(record, CBBO1sMsg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write level + write(io, record.levels.bid_px) + write(io, record.levels.ask_px) + write(io, record.levels.bid_sz) + write(io, record.levels.ask_sz) + write(io, record.levels.bid_ct) + write(io, record.levels.ask_ct) + + elseif isa(record, CBBO1mMsg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write level + write(io, record.levels.bid_px) + write(io, record.levels.ask_px) + write(io, record.levels.bid_sz) + write(io, record.levels.ask_sz) + write(io, record.levels.bid_ct) + write(io, record.levels.ask_ct) + + elseif isa(record, TCBBOMsg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write level + write(io, record.levels.bid_px) + write(io, record.levels.ask_px) + write(io, record.levels.bid_sz) + write(io, record.levels.ask_sz) + write(io, record.levels.bid_ct) + write(io, record.levels.ask_ct) + + elseif isa(record, BBO1sMsg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write level + write(io, record.levels.bid_px) + write(io, record.levels.ask_px) + write(io, record.levels.bid_sz) + write(io, record.levels.ask_sz) + write(io, record.levels.bid_ct) + write(io, record.levels.ask_ct) + + elseif isa(record, BBO1mMsg) + write_record_header(io, record.hd) + write(io, record.price) + write(io, record.size) + write(io, UInt8(record.action)) + write(io, UInt8(record.side)) + write(io, record.flags) + write(io, record.depth) + write(io, record.ts_recv) + write(io, record.ts_in_delta) + write(io, record.sequence) + + # Write level + write(io, record.levels.bid_px) + write(io, record.levels.ask_px) + write(io, record.levels.bid_sz) + write(io, record.levels.ask_sz) + write(io, record.levels.bid_ct) + write(io, record.levels.ask_ct) + + elseif isa(record, ErrorMsg) + write_record_header(io, record.hd) + # Write error message string with null terminator, padding to fill the + # payload size implied by hd.length (in 4-byte units). + target = Int(record.hd.length) * LENGTH_MULTIPLIER - 16 + err_bytes = Vector{UInt8}(record.err) + written = 0 + write(io, err_bytes); written += length(err_bytes) + if length(err_bytes) == 0 || err_bytes[end] != 0 + write(io, UInt8(0)); written += 1 + end + if written < target + write(io, zeros(UInt8, target - written)) + end + + + elseif isa(record, SymbolMappingMsg) + # Spec-compliant SymbolMappingMsg layout (depends on DBN version): + # v1: stype_in_symbol[22] | stype_out_symbol[22] | pad(4) | start_ts(8) | end_ts(8) (body 64, total 80, hd.length 20) + # v2+: stype_in(1) | stype_in_symbol[71] | stype_out(1) | stype_out_symbol[71] | + # start_ts(8) | end_ts(8) (body 160, total 176, hd.length 44) + # + # The record's hd.length may reflect a different on-wire version than + # the file we're writing into — the Databento Live gateway emits v1 + # layout (hd.length = 20) even when the consumer is writing a v3 file. + # Re-derive the length from the layout we are about to write so the + # resulting record header matches the bytes that follow it. + body_bytes = encoder.metadata.version == 1 ? 64 : 160 + fixed_length = UInt8((16 + body_bytes) ÷ LENGTH_MULTIPLIER) + hd = record.hd + out_hd = hd.length == fixed_length ? hd : + RecordHeader(fixed_length, hd.rtype, hd.publisher_id, hd.instrument_id, hd.ts_event) + write_record_header(io, out_hd) + if encoder.metadata.version == 1 + _write_fixed_string(io, record.stype_in_symbol, 22) + _write_fixed_string(io, record.stype_out_symbol, 22) + write(io, zeros(UInt8, 4)) # padding for 8-byte ts alignment + write(io, record.start_ts) + write(io, record.end_ts) + else + sym_len = 71 + write(io, UInt8(record.stype_in)) + _write_fixed_string(io, record.stype_in_symbol, sym_len) + write(io, UInt8(record.stype_out)) + _write_fixed_string(io, record.stype_out_symbol, sym_len) + write(io, record.start_ts) + write(io, record.end_ts) + end + + + elseif isa(record, SystemMsg) + write_record_header(io, record.hd) + # Write msg + null + code + null, padded to the payload size implied by + # hd.length (in 4-byte units). + target = Int(record.hd.length) * LENGTH_MULTIPLIER - 16 + msg_bytes = Vector{UInt8}(record.msg) + code_bytes = Vector{UInt8}(record.code) + write(io, msg_bytes); written = length(msg_bytes) + write(io, UInt8(0)); written += 1 + write(io, code_bytes); written += length(code_bytes) + if length(code_bytes) == 0 || code_bytes[end] != 0 + write(io, UInt8(0)); written += 1 + end + if written < target + write(io, zeros(UInt8, target - written)) + end + end +end + +# Add finalize function for encoder +""" + finalize_encoder(encoder::DBNEncoder) + +Finalize the encoder and flush any remaining data. + +# Arguments +- `encoder::DBNEncoder`: Encoder to finalize + +# Details +Ensures all buffered data is written to the output stream. +Should be called when finished writing all records. +""" +function finalize_encoder(encoder::DBNEncoder) + # For now, we don't use compression in write mode for simplicity + # In the future, compression support could be added here +end + +# Convenience function +""" + write_dbn(filename::String, metadata::Metadata, records) + +Convenience function to write a complete DBN file with automatic compression support. + +# Arguments +- `filename::String`: Output file path (use .zst extension for compression) +- `metadata::Metadata`: File metadata +- `records`: Collection of records to write + +# Details +Creates a complete DBN file with header and all records. +Automatically handles: +- File creation and management +- Zstd compression (when filename ends with .zst) +- Header writing (uncompressed for format detection) +- Record serialization +- Resource cleanup + +# Example +```julia +metadata = Metadata(3, "TEST", Schema.TRADES, start_ts, end_ts, length(records), + SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, symbols, [], [], []) +# Uncompressed +write_dbn("output.dbn", metadata, records) +# Compressed +write_dbn("output.dbn.zst", metadata, records) +``` +""" +function write_dbn(filename::String, metadata::Metadata, records) + # Check if compression is needed + use_compression = endswith(filename, ".zst") + + base_io = open(filename, "w") + + try + if use_compression + # Wrap the entire stream with compression (including header) + compressed_io = TranscodingStream(ZstdCompressor(), base_io) + + try + encoder = DBNEncoder(compressed_io, base_io, metadata, nothing) + write_header(encoder) + + # Write all records + for record in records + write_record(encoder, record) + end + + finalize_encoder(encoder) + finally + # Close the compression stream + close(compressed_io) + end + else + # Uncompressed: write directly to file + encoder = DBNEncoder(base_io, metadata) + write_header(encoder) + + # Write all records + for record in records + write_record(encoder, record) + end + + finalize_encoder(encoder) + end + finally + # Always close the base IO + if isopen(base_io) + close(base_io) + end + end +end diff --git a/src/export.jl b/src/export.jl index 5e1f82ce..a256dddd 100644 --- a/src/export.jl +++ b/src/export.jl @@ -1,21 +1,21 @@ -""" -Export functionality for converting DBN data to other formats. -""" - -using CSV -using JSON3 -using DuckDB -using DBInterface -using DataFrames - -# --- Parquet I/O backend (DuckDB) ------------------------------------------ -# Parquet read/write is backed by DuckDB so the output is reliably readable by -# every Parquet consumer (DuckDB, Arrow, pandas/pyarrow, ...). These private -# helpers own the connection lifecycle and SQL path handling for both the -# export (`dbn_to_parquet`) and import (`parquet_to_dbn`) directions. - -# Escape a path for a DuckDB SQL string literal: double single quotes, and on -# Windows use forward slashes (accepted by DuckDB, avoids backslash ambiguity). +""" +Export functionality for converting DBN data to other formats. +""" + +using CSV +using JSON3 +using DuckDB +using DBInterface +using DataFrames + +# --- Parquet I/O backend (DuckDB) ------------------------------------------ +# Parquet read/write is backed by DuckDB so the output is reliably readable by +# every Parquet consumer (DuckDB, Arrow, pandas/pyarrow, ...). These private +# helpers own the connection lifecycle and SQL path handling for both the +# export (`dbn_to_parquet`) and import (`parquet_to_dbn`) directions. + +# Escape a path for a DuckDB SQL string literal: double single quotes, and on +# Windows use forward slashes (accepted by DuckDB, avoids backslash ambiguity). _duckdb_sql_path(p::AbstractString) = replace(replace(p, '\\' => '/'), "'" => "''") const _PARQUET_COMPRESSION_CODECS = ("ZSTD", "SNAPPY", "GZIP", "UNCOMPRESSED") @@ -32,526 +32,526 @@ function _write_parquet(df::DataFrame, output_file::AbstractString; compression= # Validate before interpolating the codec into DuckDB SQL. Unlike the path, # this token cannot be parameter-bound or quoted in COPY options. comp = _parquet_compression(compression) - # DuckDB cannot COPY a zero-column frame, which is what `records_to_dataframe` - # returns for an empty record set (e.g. a schema with no rows in the window). - # Fall back to a minimal header-only schema so we still write a valid, empty - # Parquet file that reads back as zero records. - if ncol(df) == 0 - df = DataFrame(ts_event = Int64[], instrument_id = UInt32[], publisher_id = UInt16[]) - end - con = DBInterface.connect(DuckDB.DB) - try - DuckDB.register_data_frame(con, df, "_dbn_export") - DBInterface.execute(con, - "COPY _dbn_export TO '$(_duckdb_sql_path(output_file))' " * - "(FORMAT PARQUET, COMPRESSION $comp)") - finally - DBInterface.close!(con) - end - return output_file -end - -function _read_parquet(input_file::AbstractString) - con = DBInterface.connect(DuckDB.DB) - try - return DBInterface.execute(con, - "SELECT * FROM read_parquet('$(_duckdb_sql_path(input_file))')") |> DataFrame - finally - DBInterface.close!(con) - end -end - -""" - dbn_to_csv(input_file, output_file) - -Convert a DBN file to CSV format. - -# Arguments -- `input_file::String`: Path to input DBN file -- `output_file::String`: Path to output CSV file - -# Example -```julia -dbn_to_csv("data.dbn", "data.csv") -``` -""" -function dbn_to_csv(input_file::String, output_file::String) - metadata, records = read_dbn_with_metadata(input_file) - df = records_to_dataframe(records) - CSV.write(output_file, df) - return df -end - -""" - dbn_to_json(input_file, output_file; pretty=false) - -Convert a DBN file to JSON format. - -# Arguments -- `input_file::String`: Path to input DBN file -- `output_file::String`: Path to output JSON file -- `pretty::Bool`: Whether to pretty-print the JSON (default: false) - -# Example -```julia -dbn_to_json("data.dbn", "data.json", pretty=true) -``` -""" -function dbn_to_json(input_file::String, output_file::String; pretty=false) - metadata, records = read_dbn_with_metadata(input_file) - - # Convert records to JSON-serializable format - json_records = [] - for record in records - push!(json_records, record_to_dict(record)) - end - - # Create output structure - output = Dict( - "metadata" => metadata_to_dict(metadata), - "records" => json_records - ) - - # Write JSON - if pretty - JSON3.pretty(open(output_file, "w"), output) - else - JSON3.write(output_file, output) - end - - return output -end - -""" - dbn_to_parquet(input_file, output_file; compression="zstd") - -Convert a DBN file to Parquet format. - -The Parquet file is written via DuckDB so the output is reliably readable by every -Parquet consumer (DuckDB, Arrow, pandas/pyarrow, ...). A single Parquet file is written -at `output_file`. - -# Arguments -- `input_file::String`: Path to input DBN file -- `output_file::String`: Path to output Parquet file -- `compression`: Parquet compression codec — one of `"zstd"` (default), `"snappy"`, - `"gzip"`, or `"uncompressed"`. - -# Example -```julia -dbn_to_parquet("data.dbn", "data.parquet") -dbn_to_parquet("data.dbn", "data.parquet", compression="snappy") -``` -""" -function dbn_to_parquet(input_file::String, output_file::String; compression="zstd") - metadata, records = read_dbn_with_metadata(input_file) - df = records_to_dataframe(records) - # An empty record set yields a column-less DataFrame; rebuild it from the file's - # schema so the Parquet keeps the real columns and round-trips to zero records. - if ncol(df) == 0 - df = empty_dataframe_for_schema(metadata.schema) - end - _write_parquet(df, output_file; compression=compression) - return df -end - -""" - records_to_dataframe(records) - -Convert DBN records to a DataFrame. -""" -function records_to_dataframe(records::Vector) - if isempty(records) - return DataFrame() - end - - # Get the type of the first record to determine schema - record_type = typeof(records[1]) - - # Check if all records are the same type - all_same_type = all(r -> typeof(r) == record_type, records) - - if all_same_type - # Create type-specific conversion - if record_type <: TradeMsg - return trades_to_dataframe(convert(Vector{TradeMsg}, records)) - elseif record_type <: MBOMsg - return mbo_to_dataframe(convert(Vector{MBOMsg}, records)) - elseif record_type <: TopOfBookMsg - # MBP-1 and the consolidated/BBO family (CMBP-1, TBBO, CBBO, BBO) all - # share the MBP-1 record layout: scalar fields plus a single - # top-of-book `levels::BidAskPair`. One converter handles them all. - return mbp1_to_dataframe(convert(Vector{record_type}, records)) - elseif record_type <: MBP10Msg - return mbp10_to_dataframe(convert(Vector{MBP10Msg}, records)) - elseif record_type <: OHLCVMsg - return ohlcv_to_dataframe(convert(Vector{OHLCVMsg}, records)) - elseif record_type <: StatusMsg - return status_to_dataframe(convert(Vector{StatusMsg}, records)) - elseif record_type <: ImbalanceMsg - return imbalance_to_dataframe(convert(Vector{ImbalanceMsg}, records)) - elseif record_type <: StatMsg - return stat_to_dataframe(convert(Vector{StatMsg}, records)) - elseif record_type <: InstrumentDefMsg - return instrument_def_to_dataframe(convert(Vector{InstrumentDefMsg}, records)) - else - # For unknown types, use generic mixed approach - return mixed_records_to_dataframe(records) - end - else - # For mixed record types, create a generic structure - return mixed_records_to_dataframe(records) - end -end - -""" - empty_dataframe_for_schema(schema) - -Build a zero-row DataFrame with the columns of `schema`'s record type. - -`records_to_dataframe` returns a column-less `DataFrame()` for an empty record set -because it has no record to dispatch on. When the originating `Metadata.schema` is known -(e.g. in `dbn_to_parquet`), use it here instead so the exported file keeps the real schema -and round-trips back to zero records. Unknown/mixed schemas fall back to a minimal -header-only frame. -""" -function empty_dataframe_for_schema(schema) - if schema == Schema.TRADES - return trades_to_dataframe(TradeMsg[]) - elseif schema == Schema.MBO - return mbo_to_dataframe(MBOMsg[]) - elseif schema == Schema.MBP_1 || schema == Schema.TBBO - return mbp1_to_dataframe(MBP1Msg[]) - elseif schema == Schema.CMBP_1 - return mbp1_to_dataframe(CMBP1Msg[]) - elseif schema == Schema.CBBO_1S - return mbp1_to_dataframe(CBBO1sMsg[]) - elseif schema == Schema.CBBO_1M - return mbp1_to_dataframe(CBBO1mMsg[]) - elseif schema == Schema.TCBBO - return mbp1_to_dataframe(TCBBOMsg[]) - elseif schema == Schema.BBO_1S - return mbp1_to_dataframe(BBO1sMsg[]) - elseif schema == Schema.BBO_1M - return mbp1_to_dataframe(BBO1mMsg[]) - elseif schema == Schema.MBP_10 - return mbp10_to_dataframe(MBP10Msg[]) - elseif schema == Schema.OHLCV_1S || schema == Schema.OHLCV_1M || - schema == Schema.OHLCV_1H || schema == Schema.OHLCV_1D - return ohlcv_to_dataframe(OHLCVMsg[]) - elseif schema == Schema.STATUS - return status_to_dataframe(StatusMsg[]) - elseif schema == Schema.IMBALANCE - return imbalance_to_dataframe(ImbalanceMsg[]) - elseif schema == Schema.STATISTICS - return stat_to_dataframe(StatMsg[]) - elseif schema == Schema.DEFINITION - return instrument_def_to_dataframe(InstrumentDefMsg[]) - else - # Unknown / mixed schema: minimal header-only frame. - return DataFrame(ts_event = Int64[], instrument_id = UInt32[], publisher_id = UInt16[]) - end -end - -function trades_to_dataframe(records::Vector{TradeMsg}) - DataFrame( - ts_event = [r.hd.ts_event for r in records], - ts_recv = [r.ts_recv for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - price = [price_to_float(r.price) for r in records], - size = [r.size for r in records], - action = [string(r.action) for r in records], - side = [string(r.side) for r in records], - flags = [r.flags for r in records], - depth = [r.depth for r in records], - ts_in_delta = [r.ts_in_delta for r in records], - sequence = [r.sequence for r in records] - ) -end - -function mbo_to_dataframe(records::Vector{MBOMsg}) - DataFrame( - ts_event = [r.hd.ts_event for r in records], - ts_recv = [r.ts_recv for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - order_id = [r.order_id for r in records], - price = [price_to_float(r.price) for r in records], - size = [r.size for r in records], - flags = [r.flags for r in records], - channel_id = [r.channel_id for r in records], - action = [string(r.action) for r in records], - side = [string(r.side) for r in records], - ts_in_delta = [r.ts_in_delta for r in records], - sequence = [r.sequence for r in records] - ) -end - -# MBP-1 and the consolidated/BBO family share an identical record layout: the -# scalar trade fields plus a single top-of-book `levels::BidAskPair`. They differ -# only in which fields the gateway populates, so one DataFrame converter serves -# all of them. -const TopOfBookMsg = Union{MBP1Msg,CMBP1Msg,TCBBOMsg,CBBO1sMsg,CBBO1mMsg,BBO1sMsg,BBO1mMsg} - -function mbp1_to_dataframe(records::Vector{<:TopOfBookMsg}) - DataFrame( - ts_event = [r.hd.ts_event for r in records], - ts_recv = [r.ts_recv for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - bid_price = [price_to_float(r.levels.bid_px) for r in records], - ask_price = [price_to_float(r.levels.ask_px) for r in records], - bid_size = [r.levels.bid_sz for r in records], - ask_size = [r.levels.ask_sz for r in records], - bid_ct = [r.levels.bid_ct for r in records], - ask_ct = [r.levels.ask_ct for r in records], - flags = [r.flags for r in records], - ts_in_delta = [r.ts_in_delta for r in records], - sequence = [r.sequence for r in records], - action = [string(r.action) for r in records], - side = [string(r.side) for r in records] - ) -end - -function mbp10_to_dataframe(records::Vector{MBP10Msg}) - # MBP-10 carries 10 price levels in `levels::NTuple{10,BidAskPair}`; expand - # each record into one row per level (0-indexed in the output). - # The row-by-row builder below would collapse to a 0-column frame on empty - # input, so return an explicitly-typed zero-row frame in that case. - if isempty(records) - return DataFrame( - ts_event = Int64[], ts_recv = Int64[], instrument_id = UInt32[], - publisher_id = UInt16[], level = Int64[], bid_price = Float64[], - ask_price = Float64[], bid_size = UInt32[], ask_size = UInt32[], - bid_ct = UInt32[], ask_ct = UInt32[], flags = UInt8[], - ts_in_delta = Int32[], sequence = UInt32[], action = String[], side = String[] - ) - end - rows = [] - for record in records - for (level, pair) in enumerate(record.levels) - push!(rows, ( - ts_event = record.hd.ts_event, - ts_recv = record.ts_recv, - instrument_id = record.hd.instrument_id, - publisher_id = record.hd.publisher_id, - level = level - 1, # 0-indexed - bid_price = price_to_float(pair.bid_px), - ask_price = price_to_float(pair.ask_px), - bid_size = pair.bid_sz, - ask_size = pair.ask_sz, - bid_ct = pair.bid_ct, - ask_ct = pair.ask_ct, - flags = record.flags, - ts_in_delta = record.ts_in_delta, - sequence = record.sequence, - action = string(record.action), - side = string(record.side) - )) - end - end - - return DataFrame(rows) -end - -function ohlcv_to_dataframe(records::Vector{OHLCVMsg}) - # OHLCVMsg has no ts_recv: it carries only hd, open/high/low/close, and volume. - DataFrame( - ts_event = [r.hd.ts_event for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - open = [price_to_float(r.open) for r in records], - high = [price_to_float(r.high) for r in records], - low = [price_to_float(r.low) for r in records], - close = [price_to_float(r.close) for r in records], - volume = [r.volume for r in records] - ) -end - -function status_to_dataframe(records::Vector{StatusMsg}) - # StatusMsg has no ts_in_delta/sequence; `action` is a status action code - # (UInt16), not a trading Action enum, so emit it raw alongside the flags. - DataFrame( - ts_event = [r.hd.ts_event for r in records], - ts_recv = [r.ts_recv for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - action = [r.action for r in records], - reason = [r.reason for r in records], - trading_event = [r.trading_event for r in records], - is_trading = [r.is_trading for r in records], - is_quoting = [r.is_quoting for r in records], - is_short_sell_restricted = [r.is_short_sell_restricted for r in records] - ) -end - -function imbalance_to_dataframe(records::Vector{ImbalanceMsg}) - # ImbalanceMsg has no auction_price / ts_in_delta / sequence fields; it carries - # the auction price set below plus auction_time, and side codes are raw bytes. - DataFrame( - ts_event = [r.hd.ts_event for r in records], - ts_recv = [r.ts_recv for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - ref_price = [price_to_float(r.ref_price) for r in records], - auction_time = [r.auction_time for r in records], - cont_book_clr_price = [price_to_float(r.cont_book_clr_price) for r in records], - auct_interest_clr_price = [price_to_float(r.auct_interest_clr_price) for r in records], - ssr_filling_price = [price_to_float(r.ssr_filling_price) for r in records], - ind_match_price = [price_to_float(r.ind_match_price) for r in records], - upper_collar = [price_to_float(r.upper_collar) for r in records], - lower_collar = [price_to_float(r.lower_collar) for r in records], - paired_qty = [r.paired_qty for r in records], - total_imbalance_qty = [r.total_imbalance_qty for r in records], - market_imbalance_qty = [r.market_imbalance_qty for r in records], - unpaired_qty = [r.unpaired_qty for r in records], - auction_type = [r.auction_type for r in records], - side = [string(r.side) for r in records], - auction_status = [r.auction_status for r in records], - freeze_status = [r.freeze_status for r in records], - num_extensions = [r.num_extensions for r in records], - unpaired_side = [r.unpaired_side for r in records], - significant_imbalance = [r.significant_imbalance for r in records] - ) -end - -function stat_to_dataframe(records::Vector{StatMsg}) - DataFrame( - ts_event = [r.hd.ts_event for r in records], - ts_recv = [r.ts_recv for r in records], - ts_ref = [r.ts_ref for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - stat_type = [r.stat_type for r in records], - channel_id = [r.channel_id for r in records], - update_action = [r.update_action for r in records], - price = [price_to_float(r.price) for r in records], - quantity = [r.quantity for r in records], - flags = [r.stat_flags for r in records], - ts_in_delta = [r.ts_in_delta for r in records], - sequence = [r.sequence for r in records] - ) -end - -function instrument_def_to_dataframe(records::Vector{InstrumentDefMsg}) - DataFrame( - ts_event = [r.hd.ts_event for r in records], - ts_recv = [r.ts_recv for r in records], - instrument_id = [r.hd.instrument_id for r in records], - publisher_id = [r.hd.publisher_id for r in records], - raw_symbol = [strip_nulls(String(r.raw_symbol)) for r in records], - group = [strip_nulls(String(r.group)) for r in records], - exchange = [strip_nulls(String(r.exchange)) for r in records], - asset = [strip_nulls(String(r.asset)) for r in records], - cfi = [strip_nulls(String(r.cfi)) for r in records], - security_type = [strip_nulls(String(r.security_type)) for r in records], - currency = [strip_nulls(String(r.currency)) for r in records], - instrument_class = [string(r.instrument_class) for r in records], - strike_price = [price_to_float(r.strike_price) for r in records], - expiration = [r.expiration for r in records], - activation = [r.activation for r in records], - high_limit_price = [price_to_float(r.high_limit_price) for r in records], - low_limit_price = [price_to_float(r.low_limit_price) for r in records], - max_price_variation = [price_to_float(r.max_price_variation) for r in records], - trading_reference_price = [price_to_float(r.trading_reference_price) for r in records], - unit_of_measure_qty = [price_to_float(r.unit_of_measure_qty) for r in records], - min_price_increment = [price_to_float(r.min_price_increment) for r in records], - min_price_increment_amount = [price_to_float(r.min_price_increment_amount) for r in records], - price_ratio = [price_to_float(r.price_ratio) for r in records], - inst_attrib_value = [r.inst_attrib_value for r in records], - underlying_id = [r.underlying_id for r in records], - raw_instrument_id = [r.raw_instrument_id for r in records], - market_depth_implied = [r.market_depth_implied for r in records], - market_depth = [r.market_depth for r in records], - market_segment_id = [r.market_segment_id for r in records], - max_trade_vol = [r.max_trade_vol for r in records], - min_lot_size = [r.min_lot_size for r in records], - min_lot_size_block = [r.min_lot_size_block for r in records], - min_lot_size_round_lot = [r.min_lot_size_round_lot for r in records], - min_trade_vol = [r.min_trade_vol for r in records], - contract_multiplier = [r.contract_multiplier for r in records], - contract_multiplier_unit = [r.contract_multiplier_unit for r in records], - flow_schedule_type = [r.flow_schedule_type for r in records], - tick_rule = [r.tick_rule for r in records], - user_defined_instrument = [r.user_defined_instrument for r in records], - trading_reference_date = [r.trading_reference_date for r in records] - ) -end - -function mixed_records_to_dataframe(records::Vector) - # For mixed record types, create a generic structure with common fields - DataFrame( - record_type = [string(typeof(r)) for r in records], - ts_event = [hasproperty(r, :hd) ? r.hd.ts_event : missing for r in records], - instrument_id = [hasproperty(r, :hd) ? r.hd.instrument_id : missing for r in records], - publisher_id = [hasproperty(r, :hd) ? r.hd.publisher_id : missing for r in records] - ) -end - -""" - record_to_dict(record) - -Convert a DBN record to a dictionary for JSON serialization. -""" -function record_to_dict(record) - dict = Dict{String, Any}() - dict["record_type"] = string(typeof(record)) - - # Use reflection to get all fields - for field in fieldnames(typeof(record)) - value = getfield(record, field) - if isa(value, RecordHeader) - dict[string(field)] = record_header_to_dict(value) - elseif isa(value, NTuple{N, UInt8} where N) - # Convert byte arrays to strings - dict[string(field)] = strip_nulls(String(collect(value))) - else - dict[string(field)] = value - end - end - - return dict -end - -function record_header_to_dict(hd::RecordHeader) - return Dict( - "length" => hd.length, - "rtype" => hd.rtype, - "publisher_id" => hd.publisher_id, - "instrument_id" => hd.instrument_id, - "ts_event" => hd.ts_event - ) -end - -""" - metadata_to_dict(metadata) - -Convert metadata to a dictionary for JSON serialization. -""" -function metadata_to_dict(metadata::Metadata) - return Dict( - "version" => metadata.version, - "dataset" => metadata.dataset, - "schema" => string(metadata.schema), - "start_ts" => metadata.start_ts, - "end_ts" => metadata.end_ts, - "limit" => metadata.limit, - "stype_in" => string(metadata.stype_in), - "stype_out" => string(metadata.stype_out), - "ts_out" => metadata.ts_out, - "symbols" => metadata.symbols, - "partial" => metadata.partial, - "not_found" => metadata.not_found, - "mappings" => metadata.mappings - ) -end - -""" - strip_nulls(s) - -Remove null bytes from a string. -""" -function strip_nulls(s::String) - return replace(s, '\0' => "") + # DuckDB cannot COPY a zero-column frame, which is what `records_to_dataframe` + # returns for an empty record set (e.g. a schema with no rows in the window). + # Fall back to a minimal header-only schema so we still write a valid, empty + # Parquet file that reads back as zero records. + if ncol(df) == 0 + df = DataFrame(ts_event = Int64[], instrument_id = UInt32[], publisher_id = UInt16[]) + end + con = DBInterface.connect(DuckDB.DB) + try + DuckDB.register_data_frame(con, df, "_dbn_export") + DBInterface.execute(con, + "COPY _dbn_export TO '$(_duckdb_sql_path(output_file))' " * + "(FORMAT PARQUET, COMPRESSION $comp)") + finally + DBInterface.close!(con) + end + return output_file +end + +function _read_parquet(input_file::AbstractString) + con = DBInterface.connect(DuckDB.DB) + try + return DBInterface.execute(con, + "SELECT * FROM read_parquet('$(_duckdb_sql_path(input_file))')") |> DataFrame + finally + DBInterface.close!(con) + end +end + +""" + dbn_to_csv(input_file, output_file) + +Convert a DBN file to CSV format. + +# Arguments +- `input_file::String`: Path to input DBN file +- `output_file::String`: Path to output CSV file + +# Example +```julia +dbn_to_csv("data.dbn", "data.csv") +``` +""" +function dbn_to_csv(input_file::String, output_file::String) + metadata, records = read_dbn_with_metadata(input_file) + df = records_to_dataframe(records) + CSV.write(output_file, df) + return df +end + +""" + dbn_to_json(input_file, output_file; pretty=false) + +Convert a DBN file to JSON format. + +# Arguments +- `input_file::String`: Path to input DBN file +- `output_file::String`: Path to output JSON file +- `pretty::Bool`: Whether to pretty-print the JSON (default: false) + +# Example +```julia +dbn_to_json("data.dbn", "data.json", pretty=true) +``` +""" +function dbn_to_json(input_file::String, output_file::String; pretty=false) + metadata, records = read_dbn_with_metadata(input_file) + + # Convert records to JSON-serializable format + json_records = [] + for record in records + push!(json_records, record_to_dict(record)) + end + + # Create output structure + output = Dict( + "metadata" => metadata_to_dict(metadata), + "records" => json_records + ) + + # Write JSON + if pretty + JSON3.pretty(open(output_file, "w"), output) + else + JSON3.write(output_file, output) + end + + return output +end + +""" + dbn_to_parquet(input_file, output_file; compression="zstd") + +Convert a DBN file to Parquet format. + +The Parquet file is written via DuckDB so the output is reliably readable by every +Parquet consumer (DuckDB, Arrow, pandas/pyarrow, ...). A single Parquet file is written +at `output_file`. + +# Arguments +- `input_file::String`: Path to input DBN file +- `output_file::String`: Path to output Parquet file +- `compression`: Parquet compression codec — one of `"zstd"` (default), `"snappy"`, + `"gzip"`, or `"uncompressed"`. + +# Example +```julia +dbn_to_parquet("data.dbn", "data.parquet") +dbn_to_parquet("data.dbn", "data.parquet", compression="snappy") +``` +""" +function dbn_to_parquet(input_file::String, output_file::String; compression="zstd") + metadata, records = read_dbn_with_metadata(input_file) + df = records_to_dataframe(records) + # An empty record set yields a column-less DataFrame; rebuild it from the file's + # schema so the Parquet keeps the real columns and round-trips to zero records. + if ncol(df) == 0 + df = empty_dataframe_for_schema(metadata.schema) + end + _write_parquet(df, output_file; compression=compression) + return df +end + +""" + records_to_dataframe(records) + +Convert DBN records to a DataFrame. +""" +function records_to_dataframe(records::Vector) + if isempty(records) + return DataFrame() + end + + # Get the type of the first record to determine schema + record_type = typeof(records[1]) + + # Check if all records are the same type + all_same_type = all(r -> typeof(r) == record_type, records) + + if all_same_type + # Create type-specific conversion + if record_type <: TradeMsg + return trades_to_dataframe(convert(Vector{TradeMsg}, records)) + elseif record_type <: MBOMsg + return mbo_to_dataframe(convert(Vector{MBOMsg}, records)) + elseif record_type <: TopOfBookMsg + # MBP-1 and the consolidated/BBO family (CMBP-1, TBBO, CBBO, BBO) all + # share the MBP-1 record layout: scalar fields plus a single + # top-of-book `levels::BidAskPair`. One converter handles them all. + return mbp1_to_dataframe(convert(Vector{record_type}, records)) + elseif record_type <: MBP10Msg + return mbp10_to_dataframe(convert(Vector{MBP10Msg}, records)) + elseif record_type <: OHLCVMsg + return ohlcv_to_dataframe(convert(Vector{OHLCVMsg}, records)) + elseif record_type <: StatusMsg + return status_to_dataframe(convert(Vector{StatusMsg}, records)) + elseif record_type <: ImbalanceMsg + return imbalance_to_dataframe(convert(Vector{ImbalanceMsg}, records)) + elseif record_type <: StatMsg + return stat_to_dataframe(convert(Vector{StatMsg}, records)) + elseif record_type <: InstrumentDefMsg + return instrument_def_to_dataframe(convert(Vector{InstrumentDefMsg}, records)) + else + # For unknown types, use generic mixed approach + return mixed_records_to_dataframe(records) + end + else + # For mixed record types, create a generic structure + return mixed_records_to_dataframe(records) + end +end + +""" + empty_dataframe_for_schema(schema) + +Build a zero-row DataFrame with the columns of `schema`'s record type. + +`records_to_dataframe` returns a column-less `DataFrame()` for an empty record set +because it has no record to dispatch on. When the originating `Metadata.schema` is known +(e.g. in `dbn_to_parquet`), use it here instead so the exported file keeps the real schema +and round-trips back to zero records. Unknown/mixed schemas fall back to a minimal +header-only frame. +""" +function empty_dataframe_for_schema(schema) + if schema == Schema.TRADES + return trades_to_dataframe(TradeMsg[]) + elseif schema == Schema.MBO + return mbo_to_dataframe(MBOMsg[]) + elseif schema == Schema.MBP_1 || schema == Schema.TBBO + return mbp1_to_dataframe(MBP1Msg[]) + elseif schema == Schema.CMBP_1 + return mbp1_to_dataframe(CMBP1Msg[]) + elseif schema == Schema.CBBO_1S + return mbp1_to_dataframe(CBBO1sMsg[]) + elseif schema == Schema.CBBO_1M + return mbp1_to_dataframe(CBBO1mMsg[]) + elseif schema == Schema.TCBBO + return mbp1_to_dataframe(TCBBOMsg[]) + elseif schema == Schema.BBO_1S + return mbp1_to_dataframe(BBO1sMsg[]) + elseif schema == Schema.BBO_1M + return mbp1_to_dataframe(BBO1mMsg[]) + elseif schema == Schema.MBP_10 + return mbp10_to_dataframe(MBP10Msg[]) + elseif schema == Schema.OHLCV_1S || schema == Schema.OHLCV_1M || + schema == Schema.OHLCV_1H || schema == Schema.OHLCV_1D + return ohlcv_to_dataframe(OHLCVMsg[]) + elseif schema == Schema.STATUS + return status_to_dataframe(StatusMsg[]) + elseif schema == Schema.IMBALANCE + return imbalance_to_dataframe(ImbalanceMsg[]) + elseif schema == Schema.STATISTICS + return stat_to_dataframe(StatMsg[]) + elseif schema == Schema.DEFINITION + return instrument_def_to_dataframe(InstrumentDefMsg[]) + else + # Unknown / mixed schema: minimal header-only frame. + return DataFrame(ts_event = Int64[], instrument_id = UInt32[], publisher_id = UInt16[]) + end +end + +function trades_to_dataframe(records::Vector{TradeMsg}) + DataFrame( + ts_event = [r.hd.ts_event for r in records], + ts_recv = [r.ts_recv for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + price = [price_to_float(r.price) for r in records], + size = [r.size for r in records], + action = [string(r.action) for r in records], + side = [string(r.side) for r in records], + flags = [r.flags for r in records], + depth = [r.depth for r in records], + ts_in_delta = [r.ts_in_delta for r in records], + sequence = [r.sequence for r in records] + ) +end + +function mbo_to_dataframe(records::Vector{MBOMsg}) + DataFrame( + ts_event = [r.hd.ts_event for r in records], + ts_recv = [r.ts_recv for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + order_id = [r.order_id for r in records], + price = [price_to_float(r.price) for r in records], + size = [r.size for r in records], + flags = [r.flags for r in records], + channel_id = [r.channel_id for r in records], + action = [string(r.action) for r in records], + side = [string(r.side) for r in records], + ts_in_delta = [r.ts_in_delta for r in records], + sequence = [r.sequence for r in records] + ) +end + +# MBP-1 and the consolidated/BBO family share an identical record layout: the +# scalar trade fields plus a single top-of-book `levels::BidAskPair`. They differ +# only in which fields the gateway populates, so one DataFrame converter serves +# all of them. +const TopOfBookMsg = Union{MBP1Msg,CMBP1Msg,TCBBOMsg,CBBO1sMsg,CBBO1mMsg,BBO1sMsg,BBO1mMsg} + +function mbp1_to_dataframe(records::Vector{<:TopOfBookMsg}) + DataFrame( + ts_event = [r.hd.ts_event for r in records], + ts_recv = [r.ts_recv for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + bid_price = [price_to_float(r.levels.bid_px) for r in records], + ask_price = [price_to_float(r.levels.ask_px) for r in records], + bid_size = [r.levels.bid_sz for r in records], + ask_size = [r.levels.ask_sz for r in records], + bid_ct = [r.levels.bid_ct for r in records], + ask_ct = [r.levels.ask_ct for r in records], + flags = [r.flags for r in records], + ts_in_delta = [r.ts_in_delta for r in records], + sequence = [r.sequence for r in records], + action = [string(r.action) for r in records], + side = [string(r.side) for r in records] + ) +end + +function mbp10_to_dataframe(records::Vector{MBP10Msg}) + # MBP-10 carries 10 price levels in `levels::NTuple{10,BidAskPair}`; expand + # each record into one row per level (0-indexed in the output). + # The row-by-row builder below would collapse to a 0-column frame on empty + # input, so return an explicitly-typed zero-row frame in that case. + if isempty(records) + return DataFrame( + ts_event = Int64[], ts_recv = Int64[], instrument_id = UInt32[], + publisher_id = UInt16[], level = Int64[], bid_price = Float64[], + ask_price = Float64[], bid_size = UInt32[], ask_size = UInt32[], + bid_ct = UInt32[], ask_ct = UInt32[], flags = UInt8[], + ts_in_delta = Int32[], sequence = UInt32[], action = String[], side = String[] + ) + end + rows = [] + for record in records + for (level, pair) in enumerate(record.levels) + push!(rows, ( + ts_event = record.hd.ts_event, + ts_recv = record.ts_recv, + instrument_id = record.hd.instrument_id, + publisher_id = record.hd.publisher_id, + level = level - 1, # 0-indexed + bid_price = price_to_float(pair.bid_px), + ask_price = price_to_float(pair.ask_px), + bid_size = pair.bid_sz, + ask_size = pair.ask_sz, + bid_ct = pair.bid_ct, + ask_ct = pair.ask_ct, + flags = record.flags, + ts_in_delta = record.ts_in_delta, + sequence = record.sequence, + action = string(record.action), + side = string(record.side) + )) + end + end + + return DataFrame(rows) +end + +function ohlcv_to_dataframe(records::Vector{OHLCVMsg}) + # OHLCVMsg has no ts_recv: it carries only hd, open/high/low/close, and volume. + DataFrame( + ts_event = [r.hd.ts_event for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + open = [price_to_float(r.open) for r in records], + high = [price_to_float(r.high) for r in records], + low = [price_to_float(r.low) for r in records], + close = [price_to_float(r.close) for r in records], + volume = [r.volume for r in records] + ) +end + +function status_to_dataframe(records::Vector{StatusMsg}) + # StatusMsg has no ts_in_delta/sequence; `action` is a status action code + # (UInt16), not a trading Action enum, so emit it raw alongside the flags. + DataFrame( + ts_event = [r.hd.ts_event for r in records], + ts_recv = [r.ts_recv for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + action = [r.action for r in records], + reason = [r.reason for r in records], + trading_event = [r.trading_event for r in records], + is_trading = [r.is_trading for r in records], + is_quoting = [r.is_quoting for r in records], + is_short_sell_restricted = [r.is_short_sell_restricted for r in records] + ) +end + +function imbalance_to_dataframe(records::Vector{ImbalanceMsg}) + # ImbalanceMsg has no auction_price / ts_in_delta / sequence fields; it carries + # the auction price set below plus auction_time, and side codes are raw bytes. + DataFrame( + ts_event = [r.hd.ts_event for r in records], + ts_recv = [r.ts_recv for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + ref_price = [price_to_float(r.ref_price) for r in records], + auction_time = [r.auction_time for r in records], + cont_book_clr_price = [price_to_float(r.cont_book_clr_price) for r in records], + auct_interest_clr_price = [price_to_float(r.auct_interest_clr_price) for r in records], + ssr_filling_price = [price_to_float(r.ssr_filling_price) for r in records], + ind_match_price = [price_to_float(r.ind_match_price) for r in records], + upper_collar = [price_to_float(r.upper_collar) for r in records], + lower_collar = [price_to_float(r.lower_collar) for r in records], + paired_qty = [r.paired_qty for r in records], + total_imbalance_qty = [r.total_imbalance_qty for r in records], + market_imbalance_qty = [r.market_imbalance_qty for r in records], + unpaired_qty = [r.unpaired_qty for r in records], + auction_type = [r.auction_type for r in records], + side = [string(r.side) for r in records], + auction_status = [r.auction_status for r in records], + freeze_status = [r.freeze_status for r in records], + num_extensions = [r.num_extensions for r in records], + unpaired_side = [r.unpaired_side for r in records], + significant_imbalance = [r.significant_imbalance for r in records] + ) +end + +function stat_to_dataframe(records::Vector{StatMsg}) + DataFrame( + ts_event = [r.hd.ts_event for r in records], + ts_recv = [r.ts_recv for r in records], + ts_ref = [r.ts_ref for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + stat_type = [r.stat_type for r in records], + channel_id = [r.channel_id for r in records], + update_action = [r.update_action for r in records], + price = [price_to_float(r.price) for r in records], + quantity = [r.quantity for r in records], + flags = [r.stat_flags for r in records], + ts_in_delta = [r.ts_in_delta for r in records], + sequence = [r.sequence for r in records] + ) +end + +function instrument_def_to_dataframe(records::Vector{InstrumentDefMsg}) + DataFrame( + ts_event = [r.hd.ts_event for r in records], + ts_recv = [r.ts_recv for r in records], + instrument_id = [r.hd.instrument_id for r in records], + publisher_id = [r.hd.publisher_id for r in records], + raw_symbol = [strip_nulls(String(r.raw_symbol)) for r in records], + group = [strip_nulls(String(r.group)) for r in records], + exchange = [strip_nulls(String(r.exchange)) for r in records], + asset = [strip_nulls(String(r.asset)) for r in records], + cfi = [strip_nulls(String(r.cfi)) for r in records], + security_type = [strip_nulls(String(r.security_type)) for r in records], + currency = [strip_nulls(String(r.currency)) for r in records], + instrument_class = [string(r.instrument_class) for r in records], + strike_price = [price_to_float(r.strike_price) for r in records], + expiration = [r.expiration for r in records], + activation = [r.activation for r in records], + high_limit_price = [price_to_float(r.high_limit_price) for r in records], + low_limit_price = [price_to_float(r.low_limit_price) for r in records], + max_price_variation = [price_to_float(r.max_price_variation) for r in records], + trading_reference_price = [price_to_float(r.trading_reference_price) for r in records], + unit_of_measure_qty = [price_to_float(r.unit_of_measure_qty) for r in records], + min_price_increment = [price_to_float(r.min_price_increment) for r in records], + min_price_increment_amount = [price_to_float(r.min_price_increment_amount) for r in records], + price_ratio = [price_to_float(r.price_ratio) for r in records], + inst_attrib_value = [r.inst_attrib_value for r in records], + underlying_id = [r.underlying_id for r in records], + raw_instrument_id = [r.raw_instrument_id for r in records], + market_depth_implied = [r.market_depth_implied for r in records], + market_depth = [r.market_depth for r in records], + market_segment_id = [r.market_segment_id for r in records], + max_trade_vol = [r.max_trade_vol for r in records], + min_lot_size = [r.min_lot_size for r in records], + min_lot_size_block = [r.min_lot_size_block for r in records], + min_lot_size_round_lot = [r.min_lot_size_round_lot for r in records], + min_trade_vol = [r.min_trade_vol for r in records], + contract_multiplier = [r.contract_multiplier for r in records], + contract_multiplier_unit = [r.contract_multiplier_unit for r in records], + flow_schedule_type = [r.flow_schedule_type for r in records], + tick_rule = [r.tick_rule for r in records], + user_defined_instrument = [r.user_defined_instrument for r in records], + trading_reference_date = [r.trading_reference_date for r in records] + ) +end + +function mixed_records_to_dataframe(records::Vector) + # For mixed record types, create a generic structure with common fields + DataFrame( + record_type = [string(typeof(r)) for r in records], + ts_event = [hasproperty(r, :hd) ? r.hd.ts_event : missing for r in records], + instrument_id = [hasproperty(r, :hd) ? r.hd.instrument_id : missing for r in records], + publisher_id = [hasproperty(r, :hd) ? r.hd.publisher_id : missing for r in records] + ) +end + +""" + record_to_dict(record) + +Convert a DBN record to a dictionary for JSON serialization. +""" +function record_to_dict(record) + dict = Dict{String, Any}() + dict["record_type"] = string(typeof(record)) + + # Use reflection to get all fields + for field in fieldnames(typeof(record)) + value = getfield(record, field) + if isa(value, RecordHeader) + dict[string(field)] = record_header_to_dict(value) + elseif isa(value, NTuple{N, UInt8} where N) + # Convert byte arrays to strings + dict[string(field)] = strip_nulls(String(collect(value))) + else + dict[string(field)] = value + end + end + + return dict +end + +function record_header_to_dict(hd::RecordHeader) + return Dict( + "length" => hd.length, + "rtype" => hd.rtype, + "publisher_id" => hd.publisher_id, + "instrument_id" => hd.instrument_id, + "ts_event" => hd.ts_event + ) +end + +""" + metadata_to_dict(metadata) + +Convert metadata to a dictionary for JSON serialization. +""" +function metadata_to_dict(metadata::Metadata) + return Dict( + "version" => metadata.version, + "dataset" => metadata.dataset, + "schema" => string(metadata.schema), + "start_ts" => metadata.start_ts, + "end_ts" => metadata.end_ts, + "limit" => metadata.limit, + "stype_in" => string(metadata.stype_in), + "stype_out" => string(metadata.stype_out), + "ts_out" => metadata.ts_out, + "symbols" => metadata.symbols, + "partial" => metadata.partial, + "not_found" => metadata.not_found, + "mappings" => metadata.mappings + ) +end + +""" + strip_nulls(s) + +Remove null bytes from a string. +""" +function strip_nulls(s::String) + return replace(s, '\0' => "") end diff --git a/src/import.jl b/src/import.jl index 3dc6fb35..db57578a 100644 --- a/src/import.jl +++ b/src/import.jl @@ -1,772 +1,772 @@ -""" -Import functionality for converting other formats to DBN. -""" - -using JSON3 -using DataFrames -using CSV - -""" - json_to_dbn(input_file, output_file) - -Convert a JSON file to DBN format. - -# Arguments -- `input_file::String`: Path to input JSON file -- `output_file::String`: Path to output DBN file - -# JSON Format -Expects JSON with structure: -```json -{ - "metadata": { ... }, - "records": [ ... ] -} -``` - -Or JSONL format (one record per line). - -# Example -```julia -json_to_dbn("data.json", "data.dbn") -``` -""" -function json_to_dbn(input_file::String, output_file::String) - # Read JSON file - content = strip(read(input_file, String)) - - # Try to parse as structured JSON first - try - data = JSON3.read(content, Dict{String, Any}) - if haskey(data, "metadata") && haskey(data, "records") - return structured_json_to_dbn(data, output_file) - end - catch e - @debug "Failed to parse as structured JSON: $e" - end - - # Try JSONL format (one record per line) - return jsonl_to_dbn(String(content), output_file) -end - -""" - structured_json_to_dbn(data, output_file) - -Convert structured JSON (with metadata and records) to DBN. -""" -function structured_json_to_dbn(data::Dict, output_file::String) - # Parse metadata - metadata = dict_to_metadata(data["metadata"]) - - # Parse records - records = [] - for record_data in data["records"] - record = parse_json_record(record_data) - push!(records, record) - end - - # Write DBN file - write_dbn(output_file, metadata, records) - return length(records) -end - -""" - jsonl_to_dbn(content, output_file; schema=nothing, dataset="", start_ts=0, end_ts=0) - -Convert JSONL content (one record per line) to DBN. -""" -function jsonl_to_dbn(content::String, output_file::String; - schema=nothing, dataset="", start_ts=0, end_ts=0) - # Check if content is already a complete JSON object - content = strip(content) - if startswith(content, "{") && endswith(content, "}") - try - data = JSON3.read(content, Dict{String, Any}) - if haskey(data, "metadata") && haskey(data, "records") - return structured_json_to_dbn(data, output_file) - end - catch e - @debug "Not a structured JSON: $e" - end - end - - # Split into lines for JSONL processing - lines = filter(line -> !isempty(strip(line)), split(content, '\n')) - - # Parse records - records = [] - for line in lines - line = strip(line) - if isempty(line) - continue - end - try - # Try to parse each line as a complete JSON record - record_data = JSON3.read(line, Dict{String, Any}) - record = parse_json_record(JSON3.write(record_data)) - push!(records, record) - catch e - @debug "Failed to parse JSON record: $e" - @debug "Line: $line" - end - end - - if isempty(records) - error("No valid records found in JSON input") - end - - # Infer metadata if not provided - if schema === nothing - schema = infer_schema_from_records(records) - end - - # Create minimal metadata - metadata = Metadata( - UInt8(3), # DBN version - dataset, # dataset - schema, # schema - Int64(start_ts == 0 ? records[1].hd.ts_event : start_ts), # start_ts - Int64(end_ts == 0 ? records[end].hd.ts_event : end_ts), # end_ts - UInt64(length(records)), # limit - SType.RAW_SYMBOL, # stype_in - SType.RAW_SYMBOL, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String, String, Int64, Int64}[] # mappings - ) - - # Write DBN file - write_dbn(output_file, metadata, records) - return length(records) -end - -""" - parse_json_record(json_str) - -Parse a JSON record string into the appropriate DBN struct. -This reuses the parsing logic from compatibility testing. -""" -function parse_json_record(json_dict::Dict) - - # Extract header info - hd_dict = json_dict["hd"] - rtype_val = hd_dict["rtype"] - rtype = rtype_from_value(rtype_val) - - # Determine record size - record_size = get_record_size_for_rtype(rtype) - length = UInt8(record_size ÷ LENGTH_MULTIPLIER) - - # Create RecordHeader - hd = RecordHeader( - length, - rtype, - UInt16(hd_dict["publisher_id"]), - UInt32(hd_dict["instrument_id"]), - parse_timestamp(hd_dict["ts_event"]) - ) - - # Parse based on record type - if rtype == RType.MBP_0_MSG - return TradeMsg( - hd, - parse_price(json_dict["price"]), - UInt32(json_dict["size"]), - action_from_value(json_dict["action"]), - side_from_value(json_dict["side"]), - UInt8(json_dict["flags"]), - UInt8(json_dict["depth"]), - parse_timestamp(json_dict["ts_recv"]), - Int32(json_dict["ts_in_delta"]), - UInt32(json_dict["sequence"]) - ) - elseif rtype == RType.MBP_1_MSG || rtype == RType.CMBP_1_MSG - # Parse levels array for MBP-1 messages - levels_dict = json_dict["levels"][1] # First level - levels = BidAskPair( - parse_price(levels_dict["bid_px"]), - parse_price(levels_dict["ask_px"]), - UInt32(levels_dict["bid_sz"]), - UInt32(levels_dict["ask_sz"]), - UInt32(get(levels_dict, "bid_ct", get(levels_dict, "bid_pb", 0))), - UInt32(get(levels_dict, "ask_ct", get(levels_dict, "ask_pb", 0))) - ) - - return MBP1Msg( - hd, - parse_price(json_dict["price"]), - UInt32(json_dict["size"]), - action_from_value(json_dict["action"]), - side_from_value(json_dict["side"]), - UInt8(json_dict["flags"]), - UInt8(get(json_dict, "depth", 0)), - parse_timestamp(json_dict["ts_recv"]), - Int32(json_dict["ts_in_delta"]), - UInt32(get(json_dict, "sequence", 0)), - levels - ) - elseif rtype == RType.MBP_10_MSG - # Parse all 10 levels if available - levels_array = get(json_dict, "levels", []) - levels_tuple = create_mbp10_levels(levels_array) - - return MBP10Msg( - hd, - parse_price(json_dict["price"]), - UInt32(json_dict["size"]), - action_from_value(json_dict["action"]), - side_from_value(json_dict["side"]), - UInt8(json_dict["flags"]), - UInt8(json_dict["depth"]), - parse_timestamp(json_dict["ts_recv"]), - Int32(json_dict["ts_in_delta"]), - UInt32(get(json_dict, "sequence", 0)), - levels_tuple - ) - elseif rtype == RType.OHLCV_1S_MSG || rtype == RType.OHLCV_1M_MSG || - rtype == RType.OHLCV_1H_MSG || rtype == RType.OHLCV_1D_MSG - return OHLCVMsg( - hd, - parse_price(json_dict["open"]), - parse_price(json_dict["high"]), - parse_price(json_dict["low"]), - parse_price(json_dict["close"]), - UInt64(json_dict["volume"]) - ) - elseif rtype == RType.STATUS_MSG - return StatusMsg( - hd, - parse_timestamp(json_dict["ts_recv"]), - UInt16(json_dict["action"]), - UInt16(json_dict["reason"]), - UInt16(json_dict["trading_event"]), - parse_char_field(json_dict["is_trading"]), - parse_char_field(json_dict["is_quoting"]), - parse_char_field(json_dict["is_short_sell_restricted"]) - ) - elseif rtype == RType.MBO_MSG - return MBOMsg( - hd, - parse_uint64(json_dict["order_id"]), - parse_price(json_dict["price"]), - UInt32(json_dict["size"]), - UInt8(json_dict["flags"]), - UInt8(json_dict["channel_id"]), - action_from_value(json_dict["action"]), - side_from_value(json_dict["side"]), - parse_timestamp(json_dict["ts_recv"]), - Int32(json_dict["ts_in_delta"]), - UInt32(json_dict["sequence"]) - ) - else - error("Unsupported record type for JSON parsing: $rtype ($(UInt8(rtype)))") - end -end - -""" - parquet_to_dbn(input_file, output_file; schema=nothing, dataset="") - -Convert a Parquet file to DBN format. - -# Arguments -- `input_file::String`: Path to input Parquet file -- `output_file::String`: Path to output DBN file -- `schema`: DBN schema (will be inferred if not provided) -- `dataset::String`: Dataset name for metadata - -# Example -```julia -parquet_to_dbn("data.parquet", "data.dbn", schema=Schema.TRADES, dataset="XNAS") -``` -""" -function parquet_to_dbn(input_file::String, output_file::String; - schema=nothing, dataset="") - # Read Parquet file (via DuckDB) - df = _read_parquet(input_file) - - # Convert DataFrame to records - records = dataframe_to_records(df, schema) - - # Infer schema if not provided - if schema === nothing - schema = infer_schema_from_records(records) - end - - # Create metadata - metadata = create_metadata_from_dataframe(df, schema, dataset) - - # Write DBN file - write_dbn(output_file, metadata, records) - return length(records) -end - -function parse_json_record(json_str::String) - json_dict = JSON3.read(json_str, Dict{String, Any}) - return parse_json_record(json_dict) -end - -""" - csv_to_dbn(input_file, output_file; schema=nothing, dataset="") - -Convert a CSV file to DBN format. - -# Arguments -- `input_file::String`: Path to input CSV file -- `output_file::String`: Path to output DBN file -- `schema`: DBN schema (will be inferred if not provided) -- `dataset::String`: Dataset name for metadata - -# Example -```julia -csv_to_dbn("data.csv", "data.dbn", schema=Schema.TRADES, dataset="XNAS") -``` -""" -function csv_to_dbn(input_file::String, output_file::String; - schema=nothing, dataset="") - if schema === nothing - throw(ArgumentError("schema parameter is required for CSV conversion")) - end - if isempty(dataset) - throw(ArgumentError("dataset parameter is required for CSV conversion")) - end - - # Read CSV file - df = CSV.read(input_file, DataFrame;header=true,truestrings=["true","True","TRUE"], falsestrings=["false","False","FALSE"]) - - # Convert DataFrame to records - records = dataframe_to_records(df, schema) - - # Create metadata - metadata = create_metadata_from_dataframe(df, schema, dataset) - - # Write DBN file - write_dbn(output_file, metadata, records) - return length(records) -end - -# Helper functions - -function parse_timestamp(ts) - if isa(ts, String) - return parse(Int64, ts) - else - return Int64(ts) - end -end - -function parse_price(price) - if isa(price, String) - return parse(Int64, price) - else - return Int64(price) - end -end - -function parse_uint64(val) - if isa(val, String) - return parse(UInt64, val) - else - return UInt64(val) - end -end - -function parse_char_field(field) - if isa(field, String) && !isempty(field) - return UInt8(field[1]) - else - return UInt8(field) - end -end - -function rtype_from_value(val) - if isa(val, String) - return rtype_from_string(val) - else - return RType.T(UInt8(val)) - end -end - -function rtype_from_string(s::String) - if s == "MBP_0_MSG" - return RType.MBP_0_MSG - elseif s == "MBP_1_MSG" - return RType.MBP_1_MSG - elseif s == "MBP_10_MSG" - return RType.MBP_10_MSG - elseif s == "MBO_MSG" - return RType.MBO_MSG - elseif s == "STATUS_MSG" - return RType.STATUS_MSG - elseif s == "OHLCV_1S_MSG" - return RType.OHLCV_1S_MSG - elseif s == "OHLCV_1M_MSG" - return RType.OHLCV_1M_MSG - elseif s == "OHLCV_1H_MSG" - return RType.OHLCV_1H_MSG - elseif s == "OHLCV_1D_MSG" - return RType.OHLCV_1D_MSG - elseif s == "INSTRUMENT_DEF_MSG" - return RType.INSTRUMENT_DEF_MSG - elseif s == "IMBALANCE_MSG" - return RType.IMBALANCE_MSG - elseif s == "ERROR_MSG" - return RType.ERROR_MSG - elseif s == "SYMBOL_MAPPING_MSG" - return RType.SYMBOL_MAPPING_MSG - elseif s == "SYSTEM_MSG" - return RType.SYSTEM_MSG - elseif s == "STAT_MSG" - return RType.STAT_MSG - elseif s == "CMBP_1_MSG" - return RType.CMBP_1_MSG - elseif s == "CBBO_1S_MSG" - return RType.CBBO_1S_MSG - elseif s == "CBBO_1M_MSG" - return RType.CBBO_1M_MSG - elseif s == "TCBBO_MSG" - return RType.TCBBO_MSG - elseif s == "BBO_1S_MSG" - return RType.BBO_1S_MSG - elseif s == "BBO_1M_MSG" - return RType.BBO_1M_MSG - else - error("Unknown record type: $s") - end -end - -function action_from_string(s::AbstractString) - if s == "A" || s == "ADD" - return Action.ADD - elseif s == "C" || s == "CANCEL" - return Action.CANCEL - elseif s == "M" || s == "MODIFY" - return Action.MODIFY - elseif s == "T" || s == "TRADE" - return Action.TRADE - elseif s == "F" || s == "FILL" - return Action.FILL - else - error("Unknown action: $s") - end -end - -function action_from_value(val) - if isa(val, String) - return action_from_string(val) - else - return Action.T(UInt8(val)) - end -end - -function side_from_string(s::AbstractString) - if s == "A" || s == "ASK" - return Side.ASK - elseif s == "B" || s == "BID" - return Side.BID - elseif s == "N" || s == "NONE" - return Side.NONE - else - error("Unknown side: $s") - end -end - -function side_from_value(val) - if isa(val, String) - return side_from_string(val) - else - return Side.T(UInt8(val)) - end -end - -function create_mbp10_levels(levels_array) - # Create 10 levels, padding with zeros if needed - padded_levels = [] - for i in 1:10 - if i <= length(levels_array) - level = levels_array[i] - push!(padded_levels, BidAskPair( - parse_price(level["bid_px"]), - parse_price(level["ask_px"]), - UInt32(level["bid_sz"]), - UInt32(level["ask_sz"]), - UInt32(get(level, "bid_ct", get(level, "bid_pb", 0))), - UInt32(get(level, "ask_ct", get(level, "ask_pb", 0))) - )) - else - push!(padded_levels, BidAskPair(0, 0, 0, 0, 0, 0)) - end - end - return tuple(padded_levels...) -end - -function get_record_size_for_rtype(rtype::RType.T) - if rtype == RType.MBP_0_MSG - return sizeof(TradeMsg) - elseif rtype == RType.MBP_1_MSG || rtype == RType.CMBP_1_MSG - return sizeof(MBP1Msg) - elseif rtype == RType.MBP_10_MSG - return sizeof(MBP10Msg) - elseif rtype == RType.MBO_MSG - return sizeof(MBOMsg) - elseif rtype == RType.OHLCV_1S_MSG || rtype == RType.OHLCV_1M_MSG || - rtype == RType.OHLCV_1H_MSG || rtype == RType.OHLCV_1D_MSG - return sizeof(OHLCVMsg) - elseif rtype == RType.STATUS_MSG - return sizeof(StatusMsg) - else - return 0 - end -end - -function infer_schema_from_records(records) - if isempty(records) - return Schema.MIX - end - - record_type = typeof(records[1]) - if record_type <: TradeMsg - return Schema.TRADES - elseif record_type <: MBP1Msg - return Schema.MBP_1 - elseif record_type <: MBP10Msg - return Schema.MBP_10 - elseif record_type <: MBOMsg - return Schema.MBO - elseif record_type <: OHLCVMsg - return Schema.OHLCV_1S # Default to 1S, could be more sophisticated - elseif record_type <: StatusMsg - return Schema.STATUS - else - return Schema.MIX - end -end - -function dataframe_to_records(df::DataFrame, schema=nothing) - # Use schema if provided - if schema == Schema.TRADES - return dataframe_to_trade_records(df) - elseif schema == Schema.MBO - return dataframe_to_mbo_records(df) - elseif schema in [Schema.MBP_1, Schema.TBBO] - return dataframe_to_mbp1_records(df) - elseif schema == Schema.MBP_10 - return dataframe_to_mbp10_records(df) - elseif schema in [Schema.OHLCV_1S, Schema.OHLCV_1M, Schema.OHLCV_1H, Schema.OHLCV_1D] - return dataframe_to_ohlcv_records(df) - end - - # Infer record type from DataFrame columns - col_names = names(df) - if "order_id" in col_names - return dataframe_to_mbo_records(df) - elseif "bid_price" in col_names && "ask_price" in col_names - if "level" in col_names - return dataframe_to_mbp10_records(df) - else - return dataframe_to_mbp1_records(df) - end - elseif "open" in col_names && "high" in col_names && "low" in col_names && "close" in col_names - return dataframe_to_ohlcv_records(df) - elseif "price" in col_names && "size" in col_names - return dataframe_to_trade_records(df) - else - error("Cannot infer record type from DataFrame columns: $(col_names)") - end -end - -function dataframe_to_trade_records(df::DataFrame) - records = TradeMsg[] - # println(df) - for row in eachrow(df) - # println(row) - # Handle nested column names like "hd.ts_event" - ts_event = haskey(row, "hd.ts_event") ? row["hd.ts_event"] : get(row, :ts_event, 0) - publisher_id = haskey(row, "hd.publisher_id") ? row["hd.publisher_id"] : get(row, :publisher_id, 1) - instrument_id = haskey(row, "hd.instrument_id") ? row["hd.instrument_id"] : get(row, :instrument_id, 0) - - hd = RecordHeader( - UInt8(sizeof(TradeMsg) ÷ LENGTH_MULTIPLIER), - RType.MBP_0_MSG, - UInt16(publisher_id), - UInt32(instrument_id), - Int64(ts_event) - ) - - # Get record fields - price_val = isa(row.price, String) ? parse(Float64, row.price) : Float64(row.price) - ts_recv = haskey(row, :ts_recv) ? Int64(row.ts_recv) : Int64(ts_event) - - record = TradeMsg( - hd, - float_to_price(price_val), - UInt32(row.size), - action_from_string(row.action), - side_from_string(row.side), - UInt8(get(row, :flags, 0)), - UInt8(get(row, :depth, 0)), - ts_recv, - Int32(get(row, :ts_in_delta, 0)), - UInt32(get(row, :sequence, 0)) - ) - push!(records, record) - end - return records -end - -function dataframe_to_mbo_records(df::DataFrame) - records = MBOMsg[] - for row in eachrow(df) - hd = RecordHeader( - UInt8(sizeof(MBOMsg) ÷ LENGTH_MULTIPLIER), - RType.MBO_MSG, - UInt16(get(row, :publisher_id, 1)), - UInt32(row.instrument_id), - Int64(row.ts_event) - ) - - record = MBOMsg( - hd, - UInt64(row.order_id), - float_to_price(Float64(row.price)), - UInt32(row.size), - UInt8(get(row, :flags, 0)), - UInt8(get(row, :channel_id, 0)), - action_from_string(string(row.action)), - side_from_string(string(row.side)), - Int64(row.ts_recv), - Int32(get(row, :ts_in_delta, 0)), - UInt32(get(row, :sequence, 0)) - ) - push!(records, record) - end - return records -end - -function dataframe_to_ohlcv_records(df::DataFrame) - records = OHLCVMsg[] - for row in eachrow(df) - hd = RecordHeader( - UInt8(sizeof(OHLCVMsg) ÷ LENGTH_MULTIPLIER), - RType.OHLCV_1S_MSG, # Default to 1S - UInt16(get(row, :publisher_id, 1)), - UInt32(row.instrument_id), - Int64(row.ts_event) - ) - - record = OHLCVMsg( - hd, - float_to_price(Float64(row.open)), - float_to_price(Float64(row.high)), - float_to_price(Float64(row.low)), - float_to_price(Float64(row.close)), - UInt64(row.volume) - ) - push!(records, record) - end - return records -end - -function dataframe_to_mbp1_records(df::DataFrame) - # Implementation for MBP1 records - error("MBP1 DataFrame conversion not yet implemented") -end - -function dataframe_to_mbp10_records(df::DataFrame) - # Implementation for MBP10 records - error("MBP10 DataFrame conversion not yet implemented") -end - -function create_metadata_from_dataframe(df::DataFrame, schema, dataset) - # Find timestamp column (could be "ts_event" or "hd.ts_event") - ts_col = if "hd.ts_event" in names(df) - "hd.ts_event" - elseif "ts_event" in names(df) - "ts_event" - else - error("No timestamp column found in DataFrame") - end - - # Guard the empty case: minimum/maximum throw on a zero-row column. - start_ts = isempty(df[!, ts_col]) ? Int64(0) : minimum(df[!, ts_col]) - end_ts = isempty(df[!, ts_col]) ? Int64(0) : maximum(df[!, ts_col]) - - return Metadata( - UInt8(3), # DBN version - dataset, # dataset - schema, # schema - start_ts, # start_ts - end_ts, # end_ts - UInt64(nrow(df)), # limit - SType.RAW_SYMBOL, # stype_in - SType.RAW_SYMBOL, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String, String, Int64, Int64}[] # mappings - ) -end - -function dict_to_metadata(dict::Dict) - return Metadata( - UInt8(dict["version"]), - string(dict["dataset"]), - schema_from_value(dict["schema"]), - parse_timestamp(dict["start_ts"]), - parse_timestamp(dict["end_ts"]), - UInt64(dict["limit"]), - stype_from_value(dict["stype_in"]), - stype_from_value(dict["stype_out"]), - Bool(dict["ts_out"]), - Vector{String}(dict["symbols"]), - Vector{String}(dict["partial"]), - Vector{String}(dict["not_found"]), - Vector{Tuple{String, String, Int64, Int64}}(dict["mappings"]) - ) -end - -function schema_from_string(s::String) - if s == "TRADES" - return Schema.TRADES - elseif s == "MBP_1" - return Schema.MBP_1 - elseif s == "MBP_10" - return Schema.MBP_10 - elseif s == "MBO" - return Schema.MBO - elseif s == "OHLCV_1S" - return Schema.OHLCV_1S - elseif s == "STATUS" - return Schema.STATUS - else - return Schema.MIX - end -end - -function stype_from_string(s::String) - if s == "RAW_SYMBOL" - return SType.RAW_SYMBOL - elseif s == "INSTRUMENT_ID" - return SType.INSTRUMENT_ID - else - return SType.RAW_SYMBOL - end -end - -function schema_from_value(val) - if isa(val, String) - return schema_from_string(val) - else - return Schema.T(UInt16(val)) - end -end - -function stype_from_value(val) - if isa(val, String) - return stype_from_string(val) - else - return SType.T(UInt8(val)) - end +""" +Import functionality for converting other formats to DBN. +""" + +using JSON3 +using DataFrames +using CSV + +""" + json_to_dbn(input_file, output_file) + +Convert a JSON file to DBN format. + +# Arguments +- `input_file::String`: Path to input JSON file +- `output_file::String`: Path to output DBN file + +# JSON Format +Expects JSON with structure: +```json +{ + "metadata": { ... }, + "records": [ ... ] +} +``` + +Or JSONL format (one record per line). + +# Example +```julia +json_to_dbn("data.json", "data.dbn") +``` +""" +function json_to_dbn(input_file::String, output_file::String) + # Read JSON file + content = strip(read(input_file, String)) + + # Try to parse as structured JSON first + try + data = JSON3.read(content, Dict{String, Any}) + if haskey(data, "metadata") && haskey(data, "records") + return structured_json_to_dbn(data, output_file) + end + catch e + @debug "Failed to parse as structured JSON: $e" + end + + # Try JSONL format (one record per line) + return jsonl_to_dbn(String(content), output_file) +end + +""" + structured_json_to_dbn(data, output_file) + +Convert structured JSON (with metadata and records) to DBN. +""" +function structured_json_to_dbn(data::Dict, output_file::String) + # Parse metadata + metadata = dict_to_metadata(data["metadata"]) + + # Parse records + records = [] + for record_data in data["records"] + record = parse_json_record(record_data) + push!(records, record) + end + + # Write DBN file + write_dbn(output_file, metadata, records) + return length(records) +end + +""" + jsonl_to_dbn(content, output_file; schema=nothing, dataset="", start_ts=0, end_ts=0) + +Convert JSONL content (one record per line) to DBN. +""" +function jsonl_to_dbn(content::String, output_file::String; + schema=nothing, dataset="", start_ts=0, end_ts=0) + # Check if content is already a complete JSON object + content = strip(content) + if startswith(content, "{") && endswith(content, "}") + try + data = JSON3.read(content, Dict{String, Any}) + if haskey(data, "metadata") && haskey(data, "records") + return structured_json_to_dbn(data, output_file) + end + catch e + @debug "Not a structured JSON: $e" + end + end + + # Split into lines for JSONL processing + lines = filter(line -> !isempty(strip(line)), split(content, '\n')) + + # Parse records + records = [] + for line in lines + line = strip(line) + if isempty(line) + continue + end + try + # Try to parse each line as a complete JSON record + record_data = JSON3.read(line, Dict{String, Any}) + record = parse_json_record(JSON3.write(record_data)) + push!(records, record) + catch e + @debug "Failed to parse JSON record: $e" + @debug "Line: $line" + end + end + + if isempty(records) + error("No valid records found in JSON input") + end + + # Infer metadata if not provided + if schema === nothing + schema = infer_schema_from_records(records) + end + + # Create minimal metadata + metadata = Metadata( + UInt8(3), # DBN version + dataset, # dataset + schema, # schema + Int64(start_ts == 0 ? records[1].hd.ts_event : start_ts), # start_ts + Int64(end_ts == 0 ? records[end].hd.ts_event : end_ts), # end_ts + UInt64(length(records)), # limit + SType.RAW_SYMBOL, # stype_in + SType.RAW_SYMBOL, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String, String, Int64, Int64}[] # mappings + ) + + # Write DBN file + write_dbn(output_file, metadata, records) + return length(records) +end + +""" + parse_json_record(json_str) + +Parse a JSON record string into the appropriate DBN struct. +This reuses the parsing logic from compatibility testing. +""" +function parse_json_record(json_dict::Dict) + + # Extract header info + hd_dict = json_dict["hd"] + rtype_val = hd_dict["rtype"] + rtype = rtype_from_value(rtype_val) + + # Determine record size + record_size = get_record_size_for_rtype(rtype) + length = UInt8(record_size ÷ LENGTH_MULTIPLIER) + + # Create RecordHeader + hd = RecordHeader( + length, + rtype, + UInt16(hd_dict["publisher_id"]), + UInt32(hd_dict["instrument_id"]), + parse_timestamp(hd_dict["ts_event"]) + ) + + # Parse based on record type + if rtype == RType.MBP_0_MSG + return TradeMsg( + hd, + parse_price(json_dict["price"]), + UInt32(json_dict["size"]), + action_from_value(json_dict["action"]), + side_from_value(json_dict["side"]), + UInt8(json_dict["flags"]), + UInt8(json_dict["depth"]), + parse_timestamp(json_dict["ts_recv"]), + Int32(json_dict["ts_in_delta"]), + UInt32(json_dict["sequence"]) + ) + elseif rtype == RType.MBP_1_MSG || rtype == RType.CMBP_1_MSG + # Parse levels array for MBP-1 messages + levels_dict = json_dict["levels"][1] # First level + levels = BidAskPair( + parse_price(levels_dict["bid_px"]), + parse_price(levels_dict["ask_px"]), + UInt32(levels_dict["bid_sz"]), + UInt32(levels_dict["ask_sz"]), + UInt32(get(levels_dict, "bid_ct", get(levels_dict, "bid_pb", 0))), + UInt32(get(levels_dict, "ask_ct", get(levels_dict, "ask_pb", 0))) + ) + + return MBP1Msg( + hd, + parse_price(json_dict["price"]), + UInt32(json_dict["size"]), + action_from_value(json_dict["action"]), + side_from_value(json_dict["side"]), + UInt8(json_dict["flags"]), + UInt8(get(json_dict, "depth", 0)), + parse_timestamp(json_dict["ts_recv"]), + Int32(json_dict["ts_in_delta"]), + UInt32(get(json_dict, "sequence", 0)), + levels + ) + elseif rtype == RType.MBP_10_MSG + # Parse all 10 levels if available + levels_array = get(json_dict, "levels", []) + levels_tuple = create_mbp10_levels(levels_array) + + return MBP10Msg( + hd, + parse_price(json_dict["price"]), + UInt32(json_dict["size"]), + action_from_value(json_dict["action"]), + side_from_value(json_dict["side"]), + UInt8(json_dict["flags"]), + UInt8(json_dict["depth"]), + parse_timestamp(json_dict["ts_recv"]), + Int32(json_dict["ts_in_delta"]), + UInt32(get(json_dict, "sequence", 0)), + levels_tuple + ) + elseif rtype == RType.OHLCV_1S_MSG || rtype == RType.OHLCV_1M_MSG || + rtype == RType.OHLCV_1H_MSG || rtype == RType.OHLCV_1D_MSG + return OHLCVMsg( + hd, + parse_price(json_dict["open"]), + parse_price(json_dict["high"]), + parse_price(json_dict["low"]), + parse_price(json_dict["close"]), + UInt64(json_dict["volume"]) + ) + elseif rtype == RType.STATUS_MSG + return StatusMsg( + hd, + parse_timestamp(json_dict["ts_recv"]), + UInt16(json_dict["action"]), + UInt16(json_dict["reason"]), + UInt16(json_dict["trading_event"]), + parse_char_field(json_dict["is_trading"]), + parse_char_field(json_dict["is_quoting"]), + parse_char_field(json_dict["is_short_sell_restricted"]) + ) + elseif rtype == RType.MBO_MSG + return MBOMsg( + hd, + parse_uint64(json_dict["order_id"]), + parse_price(json_dict["price"]), + UInt32(json_dict["size"]), + UInt8(json_dict["flags"]), + UInt8(json_dict["channel_id"]), + action_from_value(json_dict["action"]), + side_from_value(json_dict["side"]), + parse_timestamp(json_dict["ts_recv"]), + Int32(json_dict["ts_in_delta"]), + UInt32(json_dict["sequence"]) + ) + else + error("Unsupported record type for JSON parsing: $rtype ($(UInt8(rtype)))") + end +end + +""" + parquet_to_dbn(input_file, output_file; schema=nothing, dataset="") + +Convert a Parquet file to DBN format. + +# Arguments +- `input_file::String`: Path to input Parquet file +- `output_file::String`: Path to output DBN file +- `schema`: DBN schema (will be inferred if not provided) +- `dataset::String`: Dataset name for metadata + +# Example +```julia +parquet_to_dbn("data.parquet", "data.dbn", schema=Schema.TRADES, dataset="XNAS") +``` +""" +function parquet_to_dbn(input_file::String, output_file::String; + schema=nothing, dataset="") + # Read Parquet file (via DuckDB) + df = _read_parquet(input_file) + + # Convert DataFrame to records + records = dataframe_to_records(df, schema) + + # Infer schema if not provided + if schema === nothing + schema = infer_schema_from_records(records) + end + + # Create metadata + metadata = create_metadata_from_dataframe(df, schema, dataset) + + # Write DBN file + write_dbn(output_file, metadata, records) + return length(records) +end + +function parse_json_record(json_str::String) + json_dict = JSON3.read(json_str, Dict{String, Any}) + return parse_json_record(json_dict) +end + +""" + csv_to_dbn(input_file, output_file; schema=nothing, dataset="") + +Convert a CSV file to DBN format. + +# Arguments +- `input_file::String`: Path to input CSV file +- `output_file::String`: Path to output DBN file +- `schema`: DBN schema (will be inferred if not provided) +- `dataset::String`: Dataset name for metadata + +# Example +```julia +csv_to_dbn("data.csv", "data.dbn", schema=Schema.TRADES, dataset="XNAS") +``` +""" +function csv_to_dbn(input_file::String, output_file::String; + schema=nothing, dataset="") + if schema === nothing + throw(ArgumentError("schema parameter is required for CSV conversion")) + end + if isempty(dataset) + throw(ArgumentError("dataset parameter is required for CSV conversion")) + end + + # Read CSV file + df = CSV.read(input_file, DataFrame;header=true,truestrings=["true","True","TRUE"], falsestrings=["false","False","FALSE"]) + + # Convert DataFrame to records + records = dataframe_to_records(df, schema) + + # Create metadata + metadata = create_metadata_from_dataframe(df, schema, dataset) + + # Write DBN file + write_dbn(output_file, metadata, records) + return length(records) +end + +# Helper functions + +function parse_timestamp(ts) + if isa(ts, String) + return parse(Int64, ts) + else + return Int64(ts) + end +end + +function parse_price(price) + if isa(price, String) + return parse(Int64, price) + else + return Int64(price) + end +end + +function parse_uint64(val) + if isa(val, String) + return parse(UInt64, val) + else + return UInt64(val) + end +end + +function parse_char_field(field) + if isa(field, String) && !isempty(field) + return UInt8(field[1]) + else + return UInt8(field) + end +end + +function rtype_from_value(val) + if isa(val, String) + return rtype_from_string(val) + else + return RType.T(UInt8(val)) + end +end + +function rtype_from_string(s::String) + if s == "MBP_0_MSG" + return RType.MBP_0_MSG + elseif s == "MBP_1_MSG" + return RType.MBP_1_MSG + elseif s == "MBP_10_MSG" + return RType.MBP_10_MSG + elseif s == "MBO_MSG" + return RType.MBO_MSG + elseif s == "STATUS_MSG" + return RType.STATUS_MSG + elseif s == "OHLCV_1S_MSG" + return RType.OHLCV_1S_MSG + elseif s == "OHLCV_1M_MSG" + return RType.OHLCV_1M_MSG + elseif s == "OHLCV_1H_MSG" + return RType.OHLCV_1H_MSG + elseif s == "OHLCV_1D_MSG" + return RType.OHLCV_1D_MSG + elseif s == "INSTRUMENT_DEF_MSG" + return RType.INSTRUMENT_DEF_MSG + elseif s == "IMBALANCE_MSG" + return RType.IMBALANCE_MSG + elseif s == "ERROR_MSG" + return RType.ERROR_MSG + elseif s == "SYMBOL_MAPPING_MSG" + return RType.SYMBOL_MAPPING_MSG + elseif s == "SYSTEM_MSG" + return RType.SYSTEM_MSG + elseif s == "STAT_MSG" + return RType.STAT_MSG + elseif s == "CMBP_1_MSG" + return RType.CMBP_1_MSG + elseif s == "CBBO_1S_MSG" + return RType.CBBO_1S_MSG + elseif s == "CBBO_1M_MSG" + return RType.CBBO_1M_MSG + elseif s == "TCBBO_MSG" + return RType.TCBBO_MSG + elseif s == "BBO_1S_MSG" + return RType.BBO_1S_MSG + elseif s == "BBO_1M_MSG" + return RType.BBO_1M_MSG + else + error("Unknown record type: $s") + end +end + +function action_from_string(s::AbstractString) + if s == "A" || s == "ADD" + return Action.ADD + elseif s == "C" || s == "CANCEL" + return Action.CANCEL + elseif s == "M" || s == "MODIFY" + return Action.MODIFY + elseif s == "T" || s == "TRADE" + return Action.TRADE + elseif s == "F" || s == "FILL" + return Action.FILL + else + error("Unknown action: $s") + end +end + +function action_from_value(val) + if isa(val, String) + return action_from_string(val) + else + return Action.T(UInt8(val)) + end +end + +function side_from_string(s::AbstractString) + if s == "A" || s == "ASK" + return Side.ASK + elseif s == "B" || s == "BID" + return Side.BID + elseif s == "N" || s == "NONE" + return Side.NONE + else + error("Unknown side: $s") + end +end + +function side_from_value(val) + if isa(val, String) + return side_from_string(val) + else + return Side.T(UInt8(val)) + end +end + +function create_mbp10_levels(levels_array) + # Create 10 levels, padding with zeros if needed + padded_levels = [] + for i in 1:10 + if i <= length(levels_array) + level = levels_array[i] + push!(padded_levels, BidAskPair( + parse_price(level["bid_px"]), + parse_price(level["ask_px"]), + UInt32(level["bid_sz"]), + UInt32(level["ask_sz"]), + UInt32(get(level, "bid_ct", get(level, "bid_pb", 0))), + UInt32(get(level, "ask_ct", get(level, "ask_pb", 0))) + )) + else + push!(padded_levels, BidAskPair(0, 0, 0, 0, 0, 0)) + end + end + return tuple(padded_levels...) +end + +function get_record_size_for_rtype(rtype::RType.T) + if rtype == RType.MBP_0_MSG + return sizeof(TradeMsg) + elseif rtype == RType.MBP_1_MSG || rtype == RType.CMBP_1_MSG + return sizeof(MBP1Msg) + elseif rtype == RType.MBP_10_MSG + return sizeof(MBP10Msg) + elseif rtype == RType.MBO_MSG + return sizeof(MBOMsg) + elseif rtype == RType.OHLCV_1S_MSG || rtype == RType.OHLCV_1M_MSG || + rtype == RType.OHLCV_1H_MSG || rtype == RType.OHLCV_1D_MSG + return sizeof(OHLCVMsg) + elseif rtype == RType.STATUS_MSG + return sizeof(StatusMsg) + else + return 0 + end +end + +function infer_schema_from_records(records) + if isempty(records) + return Schema.MIX + end + + record_type = typeof(records[1]) + if record_type <: TradeMsg + return Schema.TRADES + elseif record_type <: MBP1Msg + return Schema.MBP_1 + elseif record_type <: MBP10Msg + return Schema.MBP_10 + elseif record_type <: MBOMsg + return Schema.MBO + elseif record_type <: OHLCVMsg + return Schema.OHLCV_1S # Default to 1S, could be more sophisticated + elseif record_type <: StatusMsg + return Schema.STATUS + else + return Schema.MIX + end +end + +function dataframe_to_records(df::DataFrame, schema=nothing) + # Use schema if provided + if schema == Schema.TRADES + return dataframe_to_trade_records(df) + elseif schema == Schema.MBO + return dataframe_to_mbo_records(df) + elseif schema in [Schema.MBP_1, Schema.TBBO] + return dataframe_to_mbp1_records(df) + elseif schema == Schema.MBP_10 + return dataframe_to_mbp10_records(df) + elseif schema in [Schema.OHLCV_1S, Schema.OHLCV_1M, Schema.OHLCV_1H, Schema.OHLCV_1D] + return dataframe_to_ohlcv_records(df) + end + + # Infer record type from DataFrame columns + col_names = names(df) + if "order_id" in col_names + return dataframe_to_mbo_records(df) + elseif "bid_price" in col_names && "ask_price" in col_names + if "level" in col_names + return dataframe_to_mbp10_records(df) + else + return dataframe_to_mbp1_records(df) + end + elseif "open" in col_names && "high" in col_names && "low" in col_names && "close" in col_names + return dataframe_to_ohlcv_records(df) + elseif "price" in col_names && "size" in col_names + return dataframe_to_trade_records(df) + else + error("Cannot infer record type from DataFrame columns: $(col_names)") + end +end + +function dataframe_to_trade_records(df::DataFrame) + records = TradeMsg[] + # println(df) + for row in eachrow(df) + # println(row) + # Handle nested column names like "hd.ts_event" + ts_event = haskey(row, "hd.ts_event") ? row["hd.ts_event"] : get(row, :ts_event, 0) + publisher_id = haskey(row, "hd.publisher_id") ? row["hd.publisher_id"] : get(row, :publisher_id, 1) + instrument_id = haskey(row, "hd.instrument_id") ? row["hd.instrument_id"] : get(row, :instrument_id, 0) + + hd = RecordHeader( + UInt8(sizeof(TradeMsg) ÷ LENGTH_MULTIPLIER), + RType.MBP_0_MSG, + UInt16(publisher_id), + UInt32(instrument_id), + Int64(ts_event) + ) + + # Get record fields + price_val = isa(row.price, String) ? parse(Float64, row.price) : Float64(row.price) + ts_recv = haskey(row, :ts_recv) ? Int64(row.ts_recv) : Int64(ts_event) + + record = TradeMsg( + hd, + float_to_price(price_val), + UInt32(row.size), + action_from_string(row.action), + side_from_string(row.side), + UInt8(get(row, :flags, 0)), + UInt8(get(row, :depth, 0)), + ts_recv, + Int32(get(row, :ts_in_delta, 0)), + UInt32(get(row, :sequence, 0)) + ) + push!(records, record) + end + return records +end + +function dataframe_to_mbo_records(df::DataFrame) + records = MBOMsg[] + for row in eachrow(df) + hd = RecordHeader( + UInt8(sizeof(MBOMsg) ÷ LENGTH_MULTIPLIER), + RType.MBO_MSG, + UInt16(get(row, :publisher_id, 1)), + UInt32(row.instrument_id), + Int64(row.ts_event) + ) + + record = MBOMsg( + hd, + UInt64(row.order_id), + float_to_price(Float64(row.price)), + UInt32(row.size), + UInt8(get(row, :flags, 0)), + UInt8(get(row, :channel_id, 0)), + action_from_string(string(row.action)), + side_from_string(string(row.side)), + Int64(row.ts_recv), + Int32(get(row, :ts_in_delta, 0)), + UInt32(get(row, :sequence, 0)) + ) + push!(records, record) + end + return records +end + +function dataframe_to_ohlcv_records(df::DataFrame) + records = OHLCVMsg[] + for row in eachrow(df) + hd = RecordHeader( + UInt8(sizeof(OHLCVMsg) ÷ LENGTH_MULTIPLIER), + RType.OHLCV_1S_MSG, # Default to 1S + UInt16(get(row, :publisher_id, 1)), + UInt32(row.instrument_id), + Int64(row.ts_event) + ) + + record = OHLCVMsg( + hd, + float_to_price(Float64(row.open)), + float_to_price(Float64(row.high)), + float_to_price(Float64(row.low)), + float_to_price(Float64(row.close)), + UInt64(row.volume) + ) + push!(records, record) + end + return records +end + +function dataframe_to_mbp1_records(df::DataFrame) + # Implementation for MBP1 records + error("MBP1 DataFrame conversion not yet implemented") +end + +function dataframe_to_mbp10_records(df::DataFrame) + # Implementation for MBP10 records + error("MBP10 DataFrame conversion not yet implemented") +end + +function create_metadata_from_dataframe(df::DataFrame, schema, dataset) + # Find timestamp column (could be "ts_event" or "hd.ts_event") + ts_col = if "hd.ts_event" in names(df) + "hd.ts_event" + elseif "ts_event" in names(df) + "ts_event" + else + error("No timestamp column found in DataFrame") + end + + # Guard the empty case: minimum/maximum throw on a zero-row column. + start_ts = isempty(df[!, ts_col]) ? Int64(0) : minimum(df[!, ts_col]) + end_ts = isempty(df[!, ts_col]) ? Int64(0) : maximum(df[!, ts_col]) + + return Metadata( + UInt8(3), # DBN version + dataset, # dataset + schema, # schema + start_ts, # start_ts + end_ts, # end_ts + UInt64(nrow(df)), # limit + SType.RAW_SYMBOL, # stype_in + SType.RAW_SYMBOL, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String, String, Int64, Int64}[] # mappings + ) +end + +function dict_to_metadata(dict::Dict) + return Metadata( + UInt8(dict["version"]), + string(dict["dataset"]), + schema_from_value(dict["schema"]), + parse_timestamp(dict["start_ts"]), + parse_timestamp(dict["end_ts"]), + UInt64(dict["limit"]), + stype_from_value(dict["stype_in"]), + stype_from_value(dict["stype_out"]), + Bool(dict["ts_out"]), + Vector{String}(dict["symbols"]), + Vector{String}(dict["partial"]), + Vector{String}(dict["not_found"]), + Vector{Tuple{String, String, Int64, Int64}}(dict["mappings"]) + ) +end + +function schema_from_string(s::String) + if s == "TRADES" + return Schema.TRADES + elseif s == "MBP_1" + return Schema.MBP_1 + elseif s == "MBP_10" + return Schema.MBP_10 + elseif s == "MBO" + return Schema.MBO + elseif s == "OHLCV_1S" + return Schema.OHLCV_1S + elseif s == "STATUS" + return Schema.STATUS + else + return Schema.MIX + end +end + +function stype_from_string(s::String) + if s == "RAW_SYMBOL" + return SType.RAW_SYMBOL + elseif s == "INSTRUMENT_ID" + return SType.INSTRUMENT_ID + else + return SType.RAW_SYMBOL + end +end + +function schema_from_value(val) + if isa(val, String) + return schema_from_string(val) + else + return Schema.T(UInt16(val)) + end +end + +function stype_from_value(val) + if isa(val, String) + return stype_from_string(val) + else + return SType.T(UInt8(val)) + end end \ No newline at end of file diff --git a/src/messages.jl b/src/messages.jl index 19cec666..d42db330 100644 --- a/src/messages.jl +++ b/src/messages.jl @@ -1,714 +1,714 @@ -# DBN message type definitions - -# Message Types - -""" - MBOMsg - -Market-by-order message containing individual order information. - -# Fields -- `hd::RecordHeader`: Standard record header -- `order_id::UInt64`: Unique order identifier -- `price::Int64`: Order price (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Order size/quantity -- `flags::UInt8`: Order flags -- `channel_id::UInt8`: Channel identifier -- `action::Action.T`: Order action (ADD, MODIFY, CANCEL, etc.) -- `side::Side.T`: Order side (BID or ASK) -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -""" -struct MBOMsg - hd::RecordHeader - order_id::UInt64 - price::Int64 - size::UInt32 - flags::UInt8 - channel_id::UInt8 - action::Action.T - side::Side.T - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 -end - -""" - TradeMsg - -Trade execution message. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Trade price (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Trade size/quantity -- `action::Action.T`: Trade action -- `side::Side.T`: Aggressor side -- `flags::UInt8`: Trade flags -- `depth::UInt8`: Book depth -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -""" -struct TradeMsg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 -end - -""" - MBP1Msg - -Market-by-price message with book depth 1 (top-of-book). - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth (always 1) -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::BidAskPair`: Best bid and ask information -""" -struct MBP1Msg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::BidAskPair -end - -""" - MBP10Msg - -Market-by-price message with book depth 10. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth (up to 10) -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::NTuple{10,BidAskPair}`: Up to 10 levels of bid/ask information -""" -struct MBP10Msg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::NTuple{10,BidAskPair} -end - -""" - OHLCVMsg - -Open, High, Low, Close, Volume (OHLCV) aggregated data message. - -# Fields -- `hd::RecordHeader`: Standard record header -- `open::Int64`: Opening price (scaled by FIXED_PRICE_SCALE) -- `high::Int64`: Highest price (scaled by FIXED_PRICE_SCALE) -- `low::Int64`: Lowest price (scaled by FIXED_PRICE_SCALE) -- `close::Int64`: Closing price (scaled by FIXED_PRICE_SCALE) -- `volume::UInt64`: Total volume traded -""" -struct OHLCVMsg - hd::RecordHeader - open::Int64 - high::Int64 - low::Int64 - close::Int64 - volume::UInt64 -end - -""" - StatusMsg - -Exchange status and trading state message. - -# Fields -- `hd::RecordHeader`: Standard record header -- `ts_recv::UInt64`: Timestamp when message was received -- `action::UInt16`: Status action code -- `reason::UInt16`: Reason for status change -- `trading_event::UInt16`: Trading event identifier -- `is_trading::UInt8`: Trading state (0=false, 1=true) -- `is_quoting::UInt8`: Quoting state (0=false, 1=true) -- `is_short_sell_restricted::UInt8`: Short sell restriction state (0=false, 1=true) -""" -struct StatusMsg - hd::RecordHeader - ts_recv::UInt64 - action::UInt16 - reason::UInt16 - trading_event::UInt16 - is_trading::UInt8 # c_char in Rust - is_quoting::UInt8 # c_char in Rust - is_short_sell_restricted::UInt8 # c_char in Rust -end - -""" - ImbalanceMsg - -Order imbalance information for auction periods. - -# Fields -- `hd::RecordHeader`: Standard record header -- `ts_recv::Int64`: Timestamp when message was received -- `ref_price::Int64`: Reference price (scaled by FIXED_PRICE_SCALE) -- `auction_time::UInt64`: Auction time -- `cont_book_clr_price::Int64`: Continuous book clearing price -- `auct_interest_clr_price::Int64`: Auction interest clearing price -- `ssr_filling_price::Int64`: Short sale restriction filling price -- `ind_match_price::Int64`: Indicative match price -- `upper_collar::Int64`: Upper price collar -- `lower_collar::Int64`: Lower price collar -- `paired_qty::UInt32`: Paired quantity -- `total_imbalance_qty::UInt32`: Total imbalance quantity -- `market_imbalance_qty::UInt32`: Market imbalance quantity -- `unpaired_qty::UInt32`: Unpaired quantity -- `auction_type::UInt8`: Type of auction -- `side::Side.T`: Imbalance side -- `auction_status::UInt8`: Auction status -- `freeze_status::UInt8`: Freeze status -- `num_extensions::UInt8`: Number of extensions -- `unpaired_side::UInt8`: Unpaired side -- `significant_imbalance::UInt8`: Significant imbalance indicator -""" -struct ImbalanceMsg - hd::RecordHeader - ts_recv::Int64 - ref_price::Int64 - auction_time::UInt64 - cont_book_clr_price::Int64 - auct_interest_clr_price::Int64 - ssr_filling_price::Int64 - ind_match_price::Int64 - upper_collar::Int64 - lower_collar::Int64 - paired_qty::UInt32 - total_imbalance_qty::UInt32 - market_imbalance_qty::UInt32 - unpaired_qty::UInt32 - auction_type::UInt8 - side::Side.T - auction_status::UInt8 - freeze_status::UInt8 - num_extensions::UInt8 - unpaired_side::UInt8 - significant_imbalance::UInt8 - # _reserved field for alignment -end - -""" - StatMsg - -Statistics message containing market statistics and derived data. - -# Fields -- `hd::RecordHeader`: Standard record header -- `ts_recv::UInt64`: Timestamp when message was received -- `ts_ref::UInt64`: Reference timestamp -- `price::Int64`: Statistical price (scaled by FIXED_PRICE_SCALE) -- `quantity::Int64`: Statistical quantity (expanded to 64 bits in DBN v3) -- `sequence::UInt32`: Message sequence number -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `stat_type::UInt16`: Type of statistic -- `channel_id::UInt16`: Channel identifier (changed to UInt16 to match Rust) -- `update_action::UInt8`: Update action -- `stat_flags::UInt8`: Statistical flags -""" -struct StatMsg - hd::RecordHeader - ts_recv::UInt64 - ts_ref::UInt64 - price::Int64 - quantity::Int64 # Expanded to 64 bits in DBN v3 - sequence::UInt32 - ts_in_delta::Int32 - stat_type::UInt16 - channel_id::UInt16 # Changed to UInt16 to match Rust - update_action::UInt8 - stat_flags::UInt8 -end - -""" - ErrorMsg - -Error message from live gateway. - -# Fields -- `hd::RecordHeader`: Standard record header -- `err::String`: Error message text -""" -struct ErrorMsg - hd::RecordHeader - err::String -end - -""" - SymbolMappingMsg - -Symbol mapping message from live gateway. - -# Fields -- `hd::RecordHeader`: Standard record header -- `stype_in::SType.T`: Input symbol type -- `stype_in_symbol::String`: Input symbol string -- `stype_out::SType.T`: Output symbol type -- `stype_out_symbol::String`: Output symbol string -- `start_ts::Int64`: Mapping start timestamp -- `end_ts::Int64`: Mapping end timestamp -""" -struct SymbolMappingMsg - hd::RecordHeader - stype_in::SType.T - stype_in_symbol::String - stype_out::SType.T - stype_out_symbol::String - start_ts::Int64 - end_ts::Int64 -end - -""" - SystemMsg - -System message from live gateway. - -# Fields -- `hd::RecordHeader`: Standard record header -- `msg::String`: System message text -- `code::String`: System message code -""" -struct SystemMsg - hd::RecordHeader - msg::String - code::String -end - -""" - InstrumentDefMsg - -Instrument definition message containing detailed information about financial instruments. - -Note: DBN v2 and v3 have different field sets. This struct supports both versions: -- v2-only fields (trading_reference_price, trading_reference_date, md_security_trading_status, settl_price_type) are set to 0 in v3 -- v3-only fields (all leg_* fields) are set to 0/empty in v2 -- raw_instrument_id is UInt32 in v2, UInt64 in v3 +# DBN message type definitions + +# Message Types + +""" + MBOMsg + +Market-by-order message containing individual order information. + +# Fields +- `hd::RecordHeader`: Standard record header +- `order_id::UInt64`: Unique order identifier +- `price::Int64`: Order price (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Order size/quantity +- `flags::UInt8`: Order flags +- `channel_id::UInt8`: Channel identifier +- `action::Action.T`: Order action (ADD, MODIFY, CANCEL, etc.) +- `side::Side.T`: Order side (BID or ASK) +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +""" +struct MBOMsg + hd::RecordHeader + order_id::UInt64 + price::Int64 + size::UInt32 + flags::UInt8 + channel_id::UInt8 + action::Action.T + side::Side.T + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 +end + +""" + TradeMsg + +Trade execution message. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Trade price (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Trade size/quantity +- `action::Action.T`: Trade action +- `side::Side.T`: Aggressor side +- `flags::UInt8`: Trade flags +- `depth::UInt8`: Book depth +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +""" +struct TradeMsg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 +end + +""" + MBP1Msg + +Market-by-price message with book depth 1 (top-of-book). + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth (always 1) +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::BidAskPair`: Best bid and ask information +""" +struct MBP1Msg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::BidAskPair +end + +""" + MBP10Msg + +Market-by-price message with book depth 10. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth (up to 10) +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::NTuple{10,BidAskPair}`: Up to 10 levels of bid/ask information +""" +struct MBP10Msg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::NTuple{10,BidAskPair} +end + +""" + OHLCVMsg + +Open, High, Low, Close, Volume (OHLCV) aggregated data message. + +# Fields +- `hd::RecordHeader`: Standard record header +- `open::Int64`: Opening price (scaled by FIXED_PRICE_SCALE) +- `high::Int64`: Highest price (scaled by FIXED_PRICE_SCALE) +- `low::Int64`: Lowest price (scaled by FIXED_PRICE_SCALE) +- `close::Int64`: Closing price (scaled by FIXED_PRICE_SCALE) +- `volume::UInt64`: Total volume traded +""" +struct OHLCVMsg + hd::RecordHeader + open::Int64 + high::Int64 + low::Int64 + close::Int64 + volume::UInt64 +end + +""" + StatusMsg + +Exchange status and trading state message. + +# Fields +- `hd::RecordHeader`: Standard record header +- `ts_recv::UInt64`: Timestamp when message was received +- `action::UInt16`: Status action code +- `reason::UInt16`: Reason for status change +- `trading_event::UInt16`: Trading event identifier +- `is_trading::UInt8`: Trading state (0=false, 1=true) +- `is_quoting::UInt8`: Quoting state (0=false, 1=true) +- `is_short_sell_restricted::UInt8`: Short sell restriction state (0=false, 1=true) +""" +struct StatusMsg + hd::RecordHeader + ts_recv::UInt64 + action::UInt16 + reason::UInt16 + trading_event::UInt16 + is_trading::UInt8 # c_char in Rust + is_quoting::UInt8 # c_char in Rust + is_short_sell_restricted::UInt8 # c_char in Rust +end + +""" + ImbalanceMsg + +Order imbalance information for auction periods. + +# Fields +- `hd::RecordHeader`: Standard record header +- `ts_recv::Int64`: Timestamp when message was received +- `ref_price::Int64`: Reference price (scaled by FIXED_PRICE_SCALE) +- `auction_time::UInt64`: Auction time +- `cont_book_clr_price::Int64`: Continuous book clearing price +- `auct_interest_clr_price::Int64`: Auction interest clearing price +- `ssr_filling_price::Int64`: Short sale restriction filling price +- `ind_match_price::Int64`: Indicative match price +- `upper_collar::Int64`: Upper price collar +- `lower_collar::Int64`: Lower price collar +- `paired_qty::UInt32`: Paired quantity +- `total_imbalance_qty::UInt32`: Total imbalance quantity +- `market_imbalance_qty::UInt32`: Market imbalance quantity +- `unpaired_qty::UInt32`: Unpaired quantity +- `auction_type::UInt8`: Type of auction +- `side::Side.T`: Imbalance side +- `auction_status::UInt8`: Auction status +- `freeze_status::UInt8`: Freeze status +- `num_extensions::UInt8`: Number of extensions +- `unpaired_side::UInt8`: Unpaired side +- `significant_imbalance::UInt8`: Significant imbalance indicator +""" +struct ImbalanceMsg + hd::RecordHeader + ts_recv::Int64 + ref_price::Int64 + auction_time::UInt64 + cont_book_clr_price::Int64 + auct_interest_clr_price::Int64 + ssr_filling_price::Int64 + ind_match_price::Int64 + upper_collar::Int64 + lower_collar::Int64 + paired_qty::UInt32 + total_imbalance_qty::UInt32 + market_imbalance_qty::UInt32 + unpaired_qty::UInt32 + auction_type::UInt8 + side::Side.T + auction_status::UInt8 + freeze_status::UInt8 + num_extensions::UInt8 + unpaired_side::UInt8 + significant_imbalance::UInt8 + # _reserved field for alignment +end + +""" + StatMsg + +Statistics message containing market statistics and derived data. + +# Fields +- `hd::RecordHeader`: Standard record header +- `ts_recv::UInt64`: Timestamp when message was received +- `ts_ref::UInt64`: Reference timestamp +- `price::Int64`: Statistical price (scaled by FIXED_PRICE_SCALE) +- `quantity::Int64`: Statistical quantity (expanded to 64 bits in DBN v3) +- `sequence::UInt32`: Message sequence number +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `stat_type::UInt16`: Type of statistic +- `channel_id::UInt16`: Channel identifier (changed to UInt16 to match Rust) +- `update_action::UInt8`: Update action +- `stat_flags::UInt8`: Statistical flags +""" +struct StatMsg + hd::RecordHeader + ts_recv::UInt64 + ts_ref::UInt64 + price::Int64 + quantity::Int64 # Expanded to 64 bits in DBN v3 + sequence::UInt32 + ts_in_delta::Int32 + stat_type::UInt16 + channel_id::UInt16 # Changed to UInt16 to match Rust + update_action::UInt8 + stat_flags::UInt8 +end + +""" + ErrorMsg + +Error message from live gateway. + +# Fields +- `hd::RecordHeader`: Standard record header +- `err::String`: Error message text +""" +struct ErrorMsg + hd::RecordHeader + err::String +end + +""" + SymbolMappingMsg + +Symbol mapping message from live gateway. + +# Fields +- `hd::RecordHeader`: Standard record header +- `stype_in::SType.T`: Input symbol type +- `stype_in_symbol::String`: Input symbol string +- `stype_out::SType.T`: Output symbol type +- `stype_out_symbol::String`: Output symbol string +- `start_ts::Int64`: Mapping start timestamp +- `end_ts::Int64`: Mapping end timestamp +""" +struct SymbolMappingMsg + hd::RecordHeader + stype_in::SType.T + stype_in_symbol::String + stype_out::SType.T + stype_out_symbol::String + start_ts::Int64 + end_ts::Int64 +end + +""" + SystemMsg + +System message from live gateway. + +# Fields +- `hd::RecordHeader`: Standard record header +- `msg::String`: System message text +- `code::String`: System message code +""" +struct SystemMsg + hd::RecordHeader + msg::String + code::String +end + +""" + InstrumentDefMsg + +Instrument definition message containing detailed information about financial instruments. + +Note: DBN v2 and v3 have different field sets. This struct supports both versions: +- v2-only fields (trading_reference_price, trading_reference_date, md_security_trading_status, settl_price_type) are set to 0 in v3 +- v3-only fields (all leg_* fields) are set to 0/empty in v2 +- raw_instrument_id is UInt32 in v2, UInt64 in v3 - raw_symbol is 71 bytes in DBN v2/v3 -- asset is 7 bytes in v2, 11 bytes in v3 - -# Fields -- `hd::RecordHeader`: Standard record header -- `ts_recv::Int64`: Timestamp when message was received -- `min_price_increment::Int64`: Minimum price increment -- `display_factor::Int64`: Price display factor -- `expiration::Int64`: Expiration timestamp -- `activation::Int64`: Activation timestamp -- `high_limit_price::Int64`: High limit price -- `low_limit_price::Int64`: Low limit price -- `max_price_variation::Int64`: Maximum price variation -- `trading_reference_price::Int64`: Trading reference price (DBN v2 only, 0 in v3) -- `unit_of_measure_qty::Int64`: Unit of measure quantity -- `min_price_increment_amount::Int64`: Minimum price increment amount -- `price_ratio::Int64`: Price ratio -- `inst_attrib_value::Int32`: Instrument attribute value -- `underlying_id::UInt32`: Underlying instrument ID -- `raw_instrument_id::UInt64`: Raw instrument ID (UInt32 in v2, UInt64 in v3) -- `market_depth_implied::Int32`: Market depth implied -- `market_depth::Int32`: Market depth -- `market_segment_id::UInt32`: Market segment ID -- `max_trade_vol::UInt32`: Maximum trade volume -- `min_lot_size::Int32`: Minimum lot size -- `min_lot_size_block::Int32`: Minimum lot size block -- `min_lot_size_round_lot::Int32`: Minimum lot size round lot -- `min_trade_vol::UInt32`: Minimum trade volume -- `contract_multiplier::Int32`: Contract multiplier -- `decay_quantity::Int32`: Decay quantity -- `original_contract_size::Int32`: Original contract size -- `trading_reference_date::UInt16`: Trading reference date (DBN v2 only, 0 in v3) -- `appl_id::Int16`: Application ID -- `maturity_year::UInt16`: Maturity year -- `decay_start_date::UInt16`: Decay start date -- `channel_id::UInt16`: Channel ID -- `currency::String`: Currency code -- `settl_currency::String`: Settlement currency -- `secsubtype::String`: Security subtype +- asset is 7 bytes in v2, 11 bytes in v3 + +# Fields +- `hd::RecordHeader`: Standard record header +- `ts_recv::Int64`: Timestamp when message was received +- `min_price_increment::Int64`: Minimum price increment +- `display_factor::Int64`: Price display factor +- `expiration::Int64`: Expiration timestamp +- `activation::Int64`: Activation timestamp +- `high_limit_price::Int64`: High limit price +- `low_limit_price::Int64`: Low limit price +- `max_price_variation::Int64`: Maximum price variation +- `trading_reference_price::Int64`: Trading reference price (DBN v2 only, 0 in v3) +- `unit_of_measure_qty::Int64`: Unit of measure quantity +- `min_price_increment_amount::Int64`: Minimum price increment amount +- `price_ratio::Int64`: Price ratio +- `inst_attrib_value::Int32`: Instrument attribute value +- `underlying_id::UInt32`: Underlying instrument ID +- `raw_instrument_id::UInt64`: Raw instrument ID (UInt32 in v2, UInt64 in v3) +- `market_depth_implied::Int32`: Market depth implied +- `market_depth::Int32`: Market depth +- `market_segment_id::UInt32`: Market segment ID +- `max_trade_vol::UInt32`: Maximum trade volume +- `min_lot_size::Int32`: Minimum lot size +- `min_lot_size_block::Int32`: Minimum lot size block +- `min_lot_size_round_lot::Int32`: Minimum lot size round lot +- `min_trade_vol::UInt32`: Minimum trade volume +- `contract_multiplier::Int32`: Contract multiplier +- `decay_quantity::Int32`: Decay quantity +- `original_contract_size::Int32`: Original contract size +- `trading_reference_date::UInt16`: Trading reference date (DBN v2 only, 0 in v3) +- `appl_id::Int16`: Application ID +- `maturity_year::UInt16`: Maturity year +- `decay_start_date::UInt16`: Decay start date +- `channel_id::UInt16`: Channel ID +- `currency::String`: Currency code +- `settl_currency::String`: Settlement currency +- `secsubtype::String`: Security subtype - `raw_symbol::String`: Raw symbol (71 bytes in DBN v2/v3) -- `group::String`: Group identifier -- `exchange::String`: Exchange identifier -- `asset::String`: Asset identifier (7 bytes in v2, 11 bytes in v3) -- `cfi::String`: CFI code -- `security_type::String`: Security type -- `unit_of_measure::String`: Unit of measure -- `underlying::String`: Underlying identifier -- `strike_price_currency::String`: Strike price currency -- `instrument_class::InstrumentClass.T`: Instrument class -- `strike_price::Int64`: Strike price -- `match_algorithm::Char`: Match algorithm -- `md_security_trading_status::UInt8`: MD security trading status (DBN v2 only, 0 in v3) -- `main_fraction::UInt8`: Main fraction -- `price_display_format::UInt8`: Price display format -- `settl_price_type::UInt8`: Settlement price type (DBN v2 only, 0 in v3) -- `sub_fraction::UInt8`: Sub fraction -- `underlying_product::UInt8`: Underlying product -- `security_update_action::Char`: Security update action -- `maturity_month::UInt8`: Maturity month -- `maturity_day::UInt8`: Maturity day -- `maturity_week::UInt8`: Maturity week -- `user_defined_instrument::Bool`: User defined instrument flag -- `contract_multiplier_unit::Int8`: Contract multiplier unit -- `flow_schedule_type::Int8`: Flow schedule type -- `tick_rule::UInt8`: Tick rule -- `leg_count::UInt16`: Number of legs (DBN v3 only, 0 in v2) -- `leg_index::UInt16`: Leg index (DBN v3 only, 0 in v2) -- `leg_instrument_id::UInt32`: Leg instrument ID (DBN v3 only, 0 in v2) +- `group::String`: Group identifier +- `exchange::String`: Exchange identifier +- `asset::String`: Asset identifier (7 bytes in v2, 11 bytes in v3) +- `cfi::String`: CFI code +- `security_type::String`: Security type +- `unit_of_measure::String`: Unit of measure +- `underlying::String`: Underlying identifier +- `strike_price_currency::String`: Strike price currency +- `instrument_class::InstrumentClass.T`: Instrument class +- `strike_price::Int64`: Strike price +- `match_algorithm::Char`: Match algorithm +- `md_security_trading_status::UInt8`: MD security trading status (DBN v2 only, 0 in v3) +- `main_fraction::UInt8`: Main fraction +- `price_display_format::UInt8`: Price display format +- `settl_price_type::UInt8`: Settlement price type (DBN v2 only, 0 in v3) +- `sub_fraction::UInt8`: Sub fraction +- `underlying_product::UInt8`: Underlying product +- `security_update_action::Char`: Security update action +- `maturity_month::UInt8`: Maturity month +- `maturity_day::UInt8`: Maturity day +- `maturity_week::UInt8`: Maturity week +- `user_defined_instrument::Bool`: User defined instrument flag +- `contract_multiplier_unit::Int8`: Contract multiplier unit +- `flow_schedule_type::Int8`: Flow schedule type +- `tick_rule::UInt8`: Tick rule +- `leg_count::UInt16`: Number of legs (DBN v3 only, 0 in v2) +- `leg_index::UInt16`: Leg index (DBN v3 only, 0 in v2) +- `leg_instrument_id::UInt32`: Leg instrument ID (DBN v3 only, 0 in v2) - `leg_raw_symbol::String`: Leg raw symbol (DBN v3 only, 71 bytes, empty in v2) -- `leg_side::Side.T`: Leg side (DBN v3 only, NONE in v2) -- `leg_underlying_id::UInt32`: Leg underlying ID (DBN v3 only, 0 in v2) -- `leg_instrument_class::InstrumentClass.T`: Leg instrument class (DBN v3 only, UNKNOWN_0 in v2) +- `leg_side::Side.T`: Leg side (DBN v3 only, NONE in v2) +- `leg_underlying_id::UInt32`: Leg underlying ID (DBN v3 only, 0 in v2) +- `leg_instrument_class::InstrumentClass.T`: Leg instrument class (DBN v3 only, UNKNOWN_0 in v2) - `leg_ratio_qty_numerator::Int32`: Leg ratio quantity numerator (DBN v3 only, 0 in v2) - `leg_ratio_qty_denominator::Int32`: Leg ratio quantity denominator (DBN v3 only, 0 in v2) - `leg_ratio_price_numerator::Int32`: Leg ratio price numerator (DBN v3 only, 0 in v2) - `leg_ratio_price_denominator::Int32`: Leg ratio price denominator (DBN v3 only, 0 in v2) -- `leg_price::Int64`: Leg price (DBN v3 only, 0 in v2) -- `leg_delta::Int64`: Leg delta (DBN v3 only, 0 in v2) -""" -struct InstrumentDefMsg - hd::RecordHeader - ts_recv::Int64 - min_price_increment::Int64 - display_factor::Int64 - expiration::Int64 - activation::Int64 - high_limit_price::Int64 - low_limit_price::Int64 - max_price_variation::Int64 - trading_reference_price::Int64 # DBN v2 only (0 in v3) - unit_of_measure_qty::Int64 - min_price_increment_amount::Int64 - price_ratio::Int64 - inst_attrib_value::Int32 - underlying_id::UInt32 - raw_instrument_id::UInt64 # u32 in v2, u64 in v3 - market_depth_implied::Int32 - market_depth::Int32 - market_segment_id::UInt32 - max_trade_vol::UInt32 - min_lot_size::Int32 - min_lot_size_block::Int32 - min_lot_size_round_lot::Int32 - min_trade_vol::UInt32 - contract_multiplier::Int32 - decay_quantity::Int32 - original_contract_size::Int32 - trading_reference_date::UInt16 # DBN v2 only (0 in v3) - appl_id::Int16 - maturity_year::UInt16 - decay_start_date::UInt16 - channel_id::UInt16 - currency::String - settl_currency::String - secsubtype::String - raw_symbol::String - group::String - exchange::String - asset::String # 7 bytes in v2, 11 bytes in v3 - cfi::String - security_type::String - unit_of_measure::String - underlying::String - strike_price_currency::String - instrument_class::InstrumentClass.T - strike_price::Int64 - match_algorithm::Char - md_security_trading_status::UInt8 # DBN v2 only (0 in v3) - main_fraction::UInt8 - price_display_format::UInt8 - settl_price_type::UInt8 # DBN v2 only (0 in v3) - sub_fraction::UInt8 - underlying_product::UInt8 - security_update_action::Char - maturity_month::UInt8 - maturity_day::UInt8 - maturity_week::UInt8 - user_defined_instrument::Bool - contract_multiplier_unit::Int8 - flow_schedule_type::Int8 - tick_rule::UInt8 - # New strategy leg fields in DBN v3 (all 0/empty in v2) - leg_count::UInt16 - leg_index::UInt16 - leg_instrument_id::UInt32 - leg_raw_symbol::String - leg_side::Side.T - leg_underlying_id::UInt32 - leg_instrument_class::InstrumentClass.T +- `leg_price::Int64`: Leg price (DBN v3 only, 0 in v2) +- `leg_delta::Int64`: Leg delta (DBN v3 only, 0 in v2) +""" +struct InstrumentDefMsg + hd::RecordHeader + ts_recv::Int64 + min_price_increment::Int64 + display_factor::Int64 + expiration::Int64 + activation::Int64 + high_limit_price::Int64 + low_limit_price::Int64 + max_price_variation::Int64 + trading_reference_price::Int64 # DBN v2 only (0 in v3) + unit_of_measure_qty::Int64 + min_price_increment_amount::Int64 + price_ratio::Int64 + inst_attrib_value::Int32 + underlying_id::UInt32 + raw_instrument_id::UInt64 # u32 in v2, u64 in v3 + market_depth_implied::Int32 + market_depth::Int32 + market_segment_id::UInt32 + max_trade_vol::UInt32 + min_lot_size::Int32 + min_lot_size_block::Int32 + min_lot_size_round_lot::Int32 + min_trade_vol::UInt32 + contract_multiplier::Int32 + decay_quantity::Int32 + original_contract_size::Int32 + trading_reference_date::UInt16 # DBN v2 only (0 in v3) + appl_id::Int16 + maturity_year::UInt16 + decay_start_date::UInt16 + channel_id::UInt16 + currency::String + settl_currency::String + secsubtype::String + raw_symbol::String + group::String + exchange::String + asset::String # 7 bytes in v2, 11 bytes in v3 + cfi::String + security_type::String + unit_of_measure::String + underlying::String + strike_price_currency::String + instrument_class::InstrumentClass.T + strike_price::Int64 + match_algorithm::Char + md_security_trading_status::UInt8 # DBN v2 only (0 in v3) + main_fraction::UInt8 + price_display_format::UInt8 + settl_price_type::UInt8 # DBN v2 only (0 in v3) + sub_fraction::UInt8 + underlying_product::UInt8 + security_update_action::Char + maturity_month::UInt8 + maturity_day::UInt8 + maturity_week::UInt8 + user_defined_instrument::Bool + contract_multiplier_unit::Int8 + flow_schedule_type::Int8 + tick_rule::UInt8 + # New strategy leg fields in DBN v3 (all 0/empty in v2) + leg_count::UInt16 + leg_index::UInt16 + leg_instrument_id::UInt32 + leg_raw_symbol::String + leg_side::Side.T + leg_underlying_id::UInt32 + leg_instrument_class::InstrumentClass.T leg_ratio_qty_numerator::Int32 leg_ratio_qty_denominator::Int32 leg_ratio_price_numerator::Int32 leg_ratio_price_denominator::Int32 - leg_price::Int64 - leg_delta::Int64 -end - -# Additional message structures for consolidated and BBO records - -""" - CMBP1Msg - -Consolidated market-by-price message with book depth 1. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth (always 1) -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::BidAskPair`: Consolidated best bid and ask information -""" -struct CMBP1Msg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::BidAskPair -end - -""" - CBBO1sMsg - -Consolidated best bid/offer message at 1-second intervals. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::BidAskPair`: Consolidated BBO information -""" -struct CBBO1sMsg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::BidAskPair -end - -""" - CBBO1mMsg - -Consolidated best bid/offer message at 1-minute intervals. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::BidAskPair`: Consolidated BBO information -""" -struct CBBO1mMsg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::BidAskPair -end - -""" - TCBBOMsg - -Trade-consolidated best bid/offer message. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::BidAskPair`: Trade-consolidated BBO information -""" -struct TCBBOMsg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::BidAskPair -end - -""" - BBO1sMsg - -Best bid/offer message at 1-second intervals. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::BidAskPair`: BBO information -""" -struct BBO1sMsg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::BidAskPair -end - -""" - BBO1mMsg - -Best bid/offer message at 1-minute intervals. - -# Fields -- `hd::RecordHeader`: Standard record header -- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) -- `size::UInt32`: Size at this price level -- `action::Action.T`: Price level action -- `side::Side.T`: Price level side -- `flags::UInt8`: Message flags -- `depth::UInt8`: Book depth -- `ts_recv::Int64`: Timestamp when message was received -- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion -- `sequence::UInt32`: Message sequence number -- `levels::BidAskPair`: BBO information -""" -struct BBO1mMsg - hd::RecordHeader - price::Int64 - size::UInt32 - action::Action.T - side::Side.T - flags::UInt8 - depth::UInt8 - ts_recv::Int64 - ts_in_delta::Int32 - sequence::UInt32 - levels::BidAskPair -end - - -# StructTypes definitions for JSON serialization -# RecordHeader excludes length field (implementation detail, not semantic data) -StructTypes.StructType(::Type{RecordHeader}) = StructTypes.Struct() -StructTypes.excludes(::Type{RecordHeader}) = (:length,) - - -StructTypes.StructType(::Type{MBOMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{TradeMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{MBP1Msg}) = StructTypes.Struct() -StructTypes.StructType(::Type{MBP10Msg}) = StructTypes.Struct() -StructTypes.StructType(::Type{OHLCVMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{StatusMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{ImbalanceMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{StatMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{CMBP1Msg}) = StructTypes.Struct() -StructTypes.StructType(::Type{CBBO1sMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{CBBO1mMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{TCBBOMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{BBO1sMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{BBO1mMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{ErrorMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{SymbolMappingMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{SystemMsg}) = StructTypes.Struct() -StructTypes.StructType(::Type{InstrumentDefMsg}) = StructTypes.Struct() - -# Union type for all DBN record types - enables type-stable containers -# Using a Union instead of Vector{Any} dramatically improves performance by: -# - Eliminating boxing/unboxing overhead -# - Enabling Julia's type inference and specialization -# - Reducing GC pressure (80-90% GC time → much lower) -const DBNRecord = Union{ - MBOMsg, TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg, - StatusMsg, ImbalanceMsg, StatMsg, ErrorMsg, SymbolMappingMsg, SystemMsg, - InstrumentDefMsg, CMBP1Msg, CBBO1sMsg, CBBO1mMsg, TCBBOMsg, BBO1sMsg, BBO1mMsg -} + leg_price::Int64 + leg_delta::Int64 +end + +# Additional message structures for consolidated and BBO records + +""" + CMBP1Msg + +Consolidated market-by-price message with book depth 1. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth (always 1) +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::BidAskPair`: Consolidated best bid and ask information +""" +struct CMBP1Msg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::BidAskPair +end + +""" + CBBO1sMsg + +Consolidated best bid/offer message at 1-second intervals. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::BidAskPair`: Consolidated BBO information +""" +struct CBBO1sMsg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::BidAskPair +end + +""" + CBBO1mMsg + +Consolidated best bid/offer message at 1-minute intervals. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::BidAskPair`: Consolidated BBO information +""" +struct CBBO1mMsg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::BidAskPair +end + +""" + TCBBOMsg + +Trade-consolidated best bid/offer message. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::BidAskPair`: Trade-consolidated BBO information +""" +struct TCBBOMsg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::BidAskPair +end + +""" + BBO1sMsg + +Best bid/offer message at 1-second intervals. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::BidAskPair`: BBO information +""" +struct BBO1sMsg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::BidAskPair +end + +""" + BBO1mMsg + +Best bid/offer message at 1-minute intervals. + +# Fields +- `hd::RecordHeader`: Standard record header +- `price::Int64`: Price level (scaled by FIXED_PRICE_SCALE) +- `size::UInt32`: Size at this price level +- `action::Action.T`: Price level action +- `side::Side.T`: Price level side +- `flags::UInt8`: Message flags +- `depth::UInt8`: Book depth +- `ts_recv::Int64`: Timestamp when message was received +- `ts_in_delta::Int32`: Delta from ts_event to gateway ingestion +- `sequence::UInt32`: Message sequence number +- `levels::BidAskPair`: BBO information +""" +struct BBO1mMsg + hd::RecordHeader + price::Int64 + size::UInt32 + action::Action.T + side::Side.T + flags::UInt8 + depth::UInt8 + ts_recv::Int64 + ts_in_delta::Int32 + sequence::UInt32 + levels::BidAskPair +end + + +# StructTypes definitions for JSON serialization +# RecordHeader excludes length field (implementation detail, not semantic data) +StructTypes.StructType(::Type{RecordHeader}) = StructTypes.Struct() +StructTypes.excludes(::Type{RecordHeader}) = (:length,) + + +StructTypes.StructType(::Type{MBOMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{TradeMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{MBP1Msg}) = StructTypes.Struct() +StructTypes.StructType(::Type{MBP10Msg}) = StructTypes.Struct() +StructTypes.StructType(::Type{OHLCVMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{StatusMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{ImbalanceMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{StatMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{CMBP1Msg}) = StructTypes.Struct() +StructTypes.StructType(::Type{CBBO1sMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{CBBO1mMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{TCBBOMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{BBO1sMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{BBO1mMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{ErrorMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{SymbolMappingMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{SystemMsg}) = StructTypes.Struct() +StructTypes.StructType(::Type{InstrumentDefMsg}) = StructTypes.Struct() + +# Union type for all DBN record types - enables type-stable containers +# Using a Union instead of Vector{Any} dramatically improves performance by: +# - Eliminating boxing/unboxing overhead +# - Enabling Julia's type inference and specialization +# - Reducing GC pressure (80-90% GC time → much lower) +const DBNRecord = Union{ + MBOMsg, TradeMsg, MBP1Msg, MBP10Msg, OHLCVMsg, + StatusMsg, ImbalanceMsg, StatMsg, ErrorMsg, SymbolMappingMsg, SystemMsg, + InstrumentDefMsg, CMBP1Msg, CBBO1sMsg, CBBO1mMsg, TCBBOMsg, BBO1sMsg, BBO1mMsg +} StructTypes.StructType(::Type{BidAskPair}) = StructTypes.Struct() diff --git a/src/streaming.jl b/src/streaming.jl index 7dae0de8..c0e5bd24 100644 --- a/src/streaming.jl +++ b/src/streaming.jl @@ -1,936 +1,936 @@ -# DBN streaming functionality - -""" - DBNStream - -Iterator for streaming DBN file reading with automatic compression support. - -# Fields -- `decoder::DBNDecoder`: Decoder instance (stored to avoid allocation overhead) -- `cleanup::Ref{Bool}`: Flag to track if cleanup has been done - -# Usage -```julia -for record in DBNStream("data.dbn") - println(typeof(record)) -end -``` - -# Details -Provides memory-efficient streaming access to DBN files without loading -the entire file into memory. Automatically detects and handles Zstd compression. -Gracefully skips unknown record types. - -# Performance -`DBNStream` returns the abstract `Union` element type, so each iteration boxes -the record and allocates roughly ~120 bytes per record. This is acceptable for -exploration / debugging but bottlenecks high-throughput workloads. For -performance-critical loops over a known-schema file, prefer -[`foreach_record`](@ref): - -```julia -foreach_record("data.dbn", DBN.TradeMsg) do rec - # near-zero allocation per record -end -``` -""" -mutable struct DBNStream - decoder::DBNDecoder - cleanup::Ref{Bool} - - function DBNStream(filename::String) - decoder = DBNDecoder(filename) - new(decoder, Ref(false)) - end -end - -""" - Base.iterate(stream::DBNStream) - -Initialize iteration over a DBN stream. - -# Arguments -- `stream::DBNStream`: Stream to iterate over - -# Returns -- `Tuple`: (first_record, nothing) or `nothing` if empty -""" -Base.iterate(stream::DBNStream) = iterate(stream, nothing) - -""" - Base.iterate(stream::DBNStream, state) - -Continue iteration over a DBN stream. - -# Arguments -- `stream::DBNStream`: Stream being iterated -- `state`: Unused (kept for API compatibility) - -# Returns -- `Tuple`: (next_record, nothing) or `nothing` if end reached -""" -Base.iterate(stream::DBNStream, state) = begin - decoder = stream.decoder - - # Check if already cleaned up - if stream.cleanup[] - return nothing - end - - if eof(decoder.io) - # Clean up resources once - if !stream.cleanup[] - if decoder.io !== decoder.base_io - close(decoder.io) - end - if isa(decoder.base_io, IOStream) - close(decoder.base_io) - end - stream.cleanup[] = true - end - return nothing - end - - record = read_record(decoder) - if record === nothing - return iterate(stream, state) # Skip unknown records - end - return (record, nothing) -end - -""" - Base.IteratorSize(::Type{DBNStream}) - -Indicates that DBNStream has unknown size (cannot determine record count without reading). -""" -Base.IteratorSize(::Type{DBNStream}) = Base.SizeUnknown() -""" - Base.eltype(::Type{DBNStream}) - -Element type for DBNStream iterator (Any, since different record types are possible). -""" -Base.eltype(::Type{DBNStream}) = Any - -# ============================================================================ -# Internal Helpers (Shared by Eager Read and Callback Streaming) -# ============================================================================ - -# Map type to RType for validation -_type_to_rtype_stream(::Type{TradeMsg}) = RType.MBP_0_MSG -_type_to_rtype_stream(::Type{MBOMsg}) = RType.MBO_MSG -_type_to_rtype_stream(::Type{MBP1Msg}) = RType.MBP_1_MSG -_type_to_rtype_stream(::Type{MBP10Msg}) = RType.MBP_10_MSG -_type_to_rtype_stream(::Type{OHLCVMsg}) = RType.OHLCV_1S_MSG # Default to 1s -_type_to_rtype_stream(::Type{StatusMsg}) = RType.STATUS_MSG -_type_to_rtype_stream(::Type{InstrumentDefMsg}) = RType.INSTRUMENT_DEF_MSG -_type_to_rtype_stream(::Type{ImbalanceMsg}) = RType.IMBALANCE_MSG -_type_to_rtype_stream(::Type{StatMsg}) = RType.STAT_MSG -_type_to_rtype_stream(::Type{CMBP1Msg}) = RType.CMBP_1_MSG -_type_to_rtype_stream(::Type{CBBO1sMsg}) = RType.CBBO_1S_MSG -_type_to_rtype_stream(::Type{CBBO1mMsg}) = RType.CBBO_1M_MSG -_type_to_rtype_stream(::Type{TCBBOMsg}) = RType.TCBBO_MSG -_type_to_rtype_stream(::Type{BBO1sMsg}) = RType.BBO_1S_MSG -_type_to_rtype_stream(::Type{BBO1mMsg}) = RType.BBO_1M_MSG - -""" - record_type_for_dbn_schema(schema::Schema.T) -> Union{Type, Nothing} - -Map a DBN `Schema` to the concrete record struct it produces, when the schema -is type-pure AND DBN.jl has a `_read_typed_record_stream` overload for that -type. Returns `nothing` otherwise (callers should fall back to the generic -`read_record` Union path). - -Used by `read_dbn` and `read_dbn_with_metadata` to dispatch to the typed -zero-allocation reader when possible. -""" -function record_type_for_dbn_schema(schema::Schema.T) - schema == Schema.TRADES && return TradeMsg - schema == Schema.MBO && return MBOMsg - schema == Schema.MBP_1 && return MBP1Msg - schema == Schema.MBP_10 && return MBP10Msg - schema == Schema.TBBO && return MBP1Msg # TBBO records are MBP1Msg layout - schema == Schema.OHLCV_1S && return OHLCVMsg - schema == Schema.OHLCV_1M && return OHLCVMsg - schema == Schema.OHLCV_1H && return OHLCVMsg - schema == Schema.OHLCV_1D && return OHLCVMsg - schema == Schema.DEFINITION && return InstrumentDefMsg - schema == Schema.STATUS && return StatusMsg - schema == Schema.IMBALANCE && return ImbalanceMsg - schema == Schema.STATISTICS && return StatMsg - schema == Schema.CMBP_1 && return CMBP1Msg - schema == Schema.CBBO_1S && return CBBO1sMsg - schema == Schema.CBBO_1M && return CBBO1mMsg - schema == Schema.TCBBO && return TCBBOMsg - schema == Schema.BBO_1S && return BBO1sMsg - schema == Schema.BBO_1M && return BBO1mMsg - return nothing -end - -# Type-stable record reading helpers (shared by eager read and callback API) -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{TradeMsg}, hd::RecordHeader) - return read_trade_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{MBOMsg}, hd::RecordHeader) - return read_mbo_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{MBP1Msg}, hd::RecordHeader) - return read_mbp1_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{MBP10Msg}, hd::RecordHeader) - return read_mbp10_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{OHLCVMsg}, hd::RecordHeader) - return read_ohlcv_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{StatusMsg}, hd::RecordHeader) - return read_status_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{InstrumentDefMsg}, hd::RecordHeader) - return read_instrument_def_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{ImbalanceMsg}, hd::RecordHeader) - return read_imbalance_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{StatMsg}, hd::RecordHeader) - return read_stat_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{CMBP1Msg}, hd::RecordHeader) - return read_cmbp1_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{CBBO1sMsg}, hd::RecordHeader) - return read_cbbo1s_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{CBBO1mMsg}, hd::RecordHeader) - return read_cbbo1m_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{TCBBOMsg}, hd::RecordHeader) - return read_tcbbo_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{BBO1sMsg}, hd::RecordHeader) - return read_bbo1s_msg(decoder, hd) -end - -@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{BBO1mMsg}, hd::RecordHeader) - return read_bbo1m_msg(decoder, hd) -end - -# ============================================================================ -# Near-Zero-Allocation Callback-Based Streaming -# ============================================================================ - -""" - foreach_record(f::Function, filename::String, ::Type{T}) where T - -Highly optimized callback-based streaming with minimal allocations. - -# Arguments -- `f::Function`: Callback function that processes each record -- `filename::String`: Path to DBN file -- `::Type{T}`: Record type (e.g., `TradeMsg`) - -# Performance -Achieves near-zero allocations (typically <50 total allocations for any size file) -when the callback uses `Ref` for mutable state. The Julia compiler optimizes away -most per-record allocations, making this significantly faster than iterator-based -streaming for pure processing workloads. - -# Performance tip: Use Ref for mutable state! -To achieve true near-zero allocations, use `Ref` for any mutable state in your -callback instead of plain variables: - -```julia -# ✅ GOOD: Only ~44 allocations total (not per record!) -total = Ref(0.0) -foreach_record("trades.dbn", TradeMsg) do trade - total[] += price_to_float(trade.price) -end -println(total[]) - -# ❌ SLOWER: May allocate per record due to closure overhead -total = 0.0 -foreach_record("trades.dbn", TradeMsg) do trade - total += price_to_float(trade.price) # Closure capture causes allocations -end -``` - -# More examples -```julia -# Count records by side (using Ref for mutable Dict) -counts = Ref(Dict('A' => 0, 'B' => 0)) -foreach_record("trades.dbn", TradeMsg) do trade - counts[][Char(trade.side)] += 1 -end - -# Collect records (push! automatically copies bitstypes) -trades = TradeMsg[] -sizehint!(trades, 100_000) -foreach_record("trades.dbn", TradeMsg) do trade - push!(trades, trade) # Safe: TradeMsg is copied -end - -# Filter while streaming - only store what you need -high_volume = TradeMsg[] -foreach_record("trades.dbn", TradeMsg) do trade - if trade.size > 1000 - push!(high_volume, trade) - end -end -``` - -# Note on bitstypes -All DBN message types are immutable bitstypes, so `push!(array, record)` -automatically makes a value copy. You can safely store records in arrays without -explicit copying. -""" -function foreach_record(f::Function, filename::String, ::Type{T}) where T - decoder = DBNDecoder(filename) - _foreach_record_impl(f, decoder, T) - return nothing -end - -# Internal implementation that works with an open decoder -# This allows eager read to avoid opening the file twice -function _foreach_record_impl(f::Function, decoder::DBNDecoder, ::Type{T}) where T - expected_rtype = _type_to_rtype_stream(T) - - # Pre-allocate a single buffer that we'll reuse - # Since T is a bitstype (all our message types are), we can safely mutate it - buffer = Ref{T}() - - try - while !eof(decoder.io) - # Read header - hd_result = read_record_header(decoder.io) - - # Handle unknown record types. `record_length` from - # read_record_header is in 4-byte units; the 2 bytes - # (length + rtype) of the header are already consumed. - if hd_result isa Tuple - _, rtype_raw, record_length = hd_result - skip(decoder.io, Int(record_length) * LENGTH_MULTIPLIER - 2) - continue - end - - hd = hd_result - - # Verify type matches expected - if hd.rtype != expected_rtype - # Special handling for OHLCV variants - if T === OHLCVMsg && hd.rtype in (RType.OHLCV_1S_MSG, RType.OHLCV_1M_MSG, RType.OHLCV_1H_MSG, RType.OHLCV_1D_MSG) - # OK, continue - else - error("Expected $(T) (rtype=$(expected_rtype)) but got rtype=$(hd.rtype)") - end - end - - # Read record directly into buffer. - # A reader can return nothing for a malformed record it skipped. - rec = _read_typed_record_stream(decoder, T, hd) - rec === nothing && continue - buffer[] = rec - - # Call user function with buffer contents - f(buffer[]) - end - finally - # Clean up - if decoder.io !== decoder.base_io - close(decoder.io) - end - if isa(decoder.base_io, IOStream) - close(decoder.base_io) - end - end -end - -# ============================================================================ -# Typed Streaming With Separate Control-Record Routing -# ============================================================================ - -""" - foreach_record_with_control(f_data, f_control, filename::String, ::Type{T}) where T - foreach_record_with_control(f_data, f_control, decoder::DBNDecoder, ::Type{T}) where T - -Like [`foreach_record`](@ref) but split into two callbacks: `f_data` receives -concrete `T`-typed records (the expected schema type) via a reused `Ref{T}()` -buffer with no per-record allocation, while `f_control` receives the -Union-typed value for *control* records (`ErrorMsg`, `SystemMsg`, -`SymbolMappingMsg`) that can interleave with data on a live stream. - -The typed-record path is the same hot loop as `_foreach_record_impl` and -achieves the same near-zero allocation profile. Control records do allocate -the Union container (one per control record), but they are rare relative -to data and don't contribute meaningfully to GC pressure. - -This is the primitive that backs the typed-channel live reader: -data records flow into a `Channel{T}` and control records flow into a -separate `Channel{DBNRecord}` for status/heartbeat/error handling. - -# Arguments -- `f_data`: called as `f_data(rec::T)` for every record whose `rtype` matches - `_type_to_rtype_stream(T)` (or any OHLCV variant when `T === OHLCVMsg`). -- `f_control`: called as `f_control(rec)` for every `ErrorMsg`, `SystemMsg`, - or `SymbolMappingMsg`. `rec` is one of those concrete types but typed - as the `DBNRecord` Union. -- `filename` / `decoder`: same as [`foreach_record`](@ref). -- `T`: the expected data record type (must have a `_type_to_rtype_stream` - overload — see [`record_type_for_dbn_schema`](@ref) for the supported set). - -Unknown rtypes (anything that's neither `T`-matching nor a control rtype) -are silently skipped, matching `read_record`'s permissive behaviour. This -mirrors what production live streams need: gateway-side schema additions -shouldn't crash existing consumers. -""" -function foreach_record_with_control(f_data::Function, f_control::Function, - filename::String, ::Type{T}) where T - decoder = DBNDecoder(filename) - _foreach_record_with_control_impl(f_data, f_control, decoder, T) - return nothing -end - -function foreach_record_with_control(f_data::Function, f_control::Function, - decoder::DBNDecoder, ::Type{T}) where T - _foreach_record_with_control_impl(f_data, f_control, decoder, T) - return nothing -end - -function _foreach_record_with_control_impl(f_data::Function, f_control::Function, - decoder::DBNDecoder, ::Type{T}) where T - expected_rtype = _type_to_rtype_stream(T) - # Reused buffer for the typed-record path — same trick as - # _foreach_record_impl; immutable bitstype, value-copied through `f_data`. - buffer = Ref{T}() - - try - while !eof(decoder.io) - hd_result = read_record_header(decoder.io) - - # Unknown-record-type case (header decoded as Tuple). Skip. - # `record_length` is in 4-byte units; we already consumed the - # 2 header bytes (length + rtype). - if hd_result isa Tuple - _, _, record_length = hd_result - skip(decoder.io, Int(record_length) * LENGTH_MULTIPLIER - 2) - continue - end - - hd = hd_result - - # Data record: matches expected rtype, or OHLCV-variant when T is OHLCVMsg. - is_match = hd.rtype == expected_rtype || - (T === OHLCVMsg && hd.rtype in (RType.OHLCV_1S_MSG, - RType.OHLCV_1M_MSG, - RType.OHLCV_1H_MSG, - RType.OHLCV_1D_MSG)) - if is_match - # A reader can return nothing for a malformed record it skipped. - rec = _read_typed_record_stream(decoder, T, hd) - rec === nothing && continue - buffer[] = rec - f_data(buffer[]) - continue - end - - # Control records: route via the generic Union dispatch. - if hd.rtype == RType.ERROR_MSG || - hd.rtype == RType.SYSTEM_MSG || - hd.rtype == RType.SYMBOL_MAPPING_MSG - rec = read_record_dispatch(decoder, hd, hd.rtype) - rec === nothing || f_control(rec) - continue - end - - # Unknown / unsupported rtype: skip. hd.length is in 4-byte units; - # 16 bytes of header already consumed. - skip(decoder.io, Int(hd.length) * LENGTH_MULTIPLIER - 16) - end - finally - if decoder.io !== decoder.base_io - close(decoder.io) - end - if isa(decoder.base_io, IOStream) - close(decoder.base_io) - end - end -end - -# Convenience functions for callback-based streaming - -""" - foreach_trade(f::Function, filename::String) - -Near-zero-allocation streaming of trade data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_trade(f::Function, filename::String) = foreach_record(f, filename, TradeMsg) - -""" - foreach_mbo(f::Function, filename::String) - -Near-zero-allocation streaming of MBO data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_mbo(f::Function, filename::String) = foreach_record(f, filename, MBOMsg) - -""" - foreach_mbp1(f::Function, filename::String) - -Near-zero-allocation streaming of MBP-1 (top-of-book) data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_mbp1(f::Function, filename::String) = foreach_record(f, filename, MBP1Msg) - -""" - foreach_mbp10(f::Function, filename::String) - -Near-zero-allocation streaming of MBP-10 (10-level depth) data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_mbp10(f::Function, filename::String) = foreach_record(f, filename, MBP10Msg) - -""" - foreach_tbbo(f::Function, filename::String) - -Near-zero-allocation streaming of TBBO (Trade BBO) data using callback pattern. -Uses MBP-1 records. See `foreach_record` for usage and performance tips. -""" -foreach_tbbo(f::Function, filename::String) = foreach_record(f, filename, MBP1Msg) - -""" - foreach_ohlcv(f::Function, filename::String) - -Near-zero-allocation streaming of OHLCV (candlestick) data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_ohlcv(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) - -""" - foreach_ohlcv_1s(f::Function, filename::String) - -Near-zero-allocation streaming of 1-second OHLCV data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_ohlcv_1s(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) - -""" - foreach_ohlcv_1m(f::Function, filename::String) - -Near-zero-allocation streaming of 1-minute OHLCV data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_ohlcv_1m(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) - -""" - foreach_ohlcv_1h(f::Function, filename::String) - -Near-zero-allocation streaming of 1-hour OHLCV data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_ohlcv_1h(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) - -""" - foreach_ohlcv_1d(f::Function, filename::String) - -Near-zero-allocation streaming of 1-day OHLCV data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_ohlcv_1d(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) - -# Consolidated/BBO message streaming -""" - foreach_cmbp1(f::Function, filename::String) - -Near-zero-allocation streaming of Consolidated MBP-1 data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_cmbp1(f::Function, filename::String) = foreach_record(f, filename, CMBP1Msg) - -""" - foreach_cbbo1s(f::Function, filename::String) - -Near-zero-allocation streaming of Consolidated BBO 1-second data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_cbbo1s(f::Function, filename::String) = foreach_record(f, filename, CBBO1sMsg) - -""" - foreach_cbbo1m(f::Function, filename::String) - -Near-zero-allocation streaming of Consolidated BBO 1-minute data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_cbbo1m(f::Function, filename::String) = foreach_record(f, filename, CBBO1mMsg) - -""" - foreach_tcbbo(f::Function, filename::String) - -Near-zero-allocation streaming of Top Consolidated BBO data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_tcbbo(f::Function, filename::String) = foreach_record(f, filename, TCBBOMsg) - -""" - foreach_bbo1s(f::Function, filename::String) - -Near-zero-allocation streaming of BBO 1-second data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_bbo1s(f::Function, filename::String) = foreach_record(f, filename, BBO1sMsg) - -""" - foreach_bbo1m(f::Function, filename::String) - -Near-zero-allocation streaming of BBO 1-minute data using callback pattern. -See `foreach_record` for usage and performance tips. -""" -foreach_bbo1m(f::Function, filename::String) = foreach_record(f, filename, BBO1mMsg) - -""" - DBNStreamWriter - -Streaming writer for real-time DBN data capture with automatic timestamp tracking. - -# Fields -- `encoder::DBNEncoder`: Underlying encoder for writing data -- `record_count::Int64`: Number of records written -- `first_ts::Int64`: First timestamp encountered -- `last_ts::Int64`: Last timestamp encountered -- `auto_flush::Bool`: Whether to automatically flush data -- `flush_interval::Int`: Number of records between automatic flushes -- `last_flush_count::Int64`: Record count at last flush - -# Usage -```julia -writer = DBNStreamWriter("live.dbn", "XNAS", Schema.TRADES) -write_record!(writer, trade_record) -close_writer!(writer) -``` -""" -mutable struct DBNStreamWriter - encoder::DBNEncoder - record_count::Int64 - first_ts::Int64 - last_ts::Int64 - auto_flush::Bool - flush_interval::Int - last_flush_count::Int64 -end - -""" - DBNStreamWriter(filename::String, dataset::String, schema::Schema.T; - symbols::Vector{String}=String[], - auto_flush::Bool=true, - flush_interval::Int=1000) - -Construct a streaming writer for real-time DBN data capture. - -# Arguments -- `filename::String`: Output file path -- `dataset::String`: Dataset identifier -- `schema::Schema.T`: Data schema type -- `symbols::Vector{String}`: List of symbols (optional) -- `auto_flush::Bool`: Enable automatic flushing (default: true) -- `flush_interval::Int`: Records between flushes (default: 1000) - -# Returns -- `DBNStreamWriter`: Writer instance ready for recording - -# Details -Creates a writer with placeholder timestamps that will be updated as records -are written. The header is written immediately but will be updated with -final timestamps when the writer is closed. -""" -function DBNStreamWriter(filename::String, dataset::String, schema::Schema.T; - symbols::Vector{String}=String[], - auto_flush::Bool=true, - flush_interval::Int=1000) - # Create metadata with placeholder timestamps (using 0 instead of typemin) - metadata = Metadata( - UInt8(DBN_VERSION), - dataset, - schema, - 0, # Will update with first record - 0, # Will update with last record - UInt64(0), - SType.RAW_SYMBOL, - SType.RAW_SYMBOL, - false, - symbols, - String[], - String[], - Tuple{String,String,Int64,Int64}[] - ) - - io = open(filename, "w") - encoder = DBNEncoder(io, metadata) - - # Write header (will update it later) - write_header(encoder) - - return DBNStreamWriter(encoder, 0, typemax(Int64), 0, - auto_flush, flush_interval, 0) -end - -""" - write_record!(writer::DBNStreamWriter, record) - -Write a record to the streaming writer and update timestamps. - -# Arguments -- `writer::DBNStreamWriter`: Writer instance -- `record`: Record to write (any DBN message type) - -# Details -Writes the record and automatically: -- Updates first/last timestamp tracking -- Increments record count -- Performs auto-flush if enabled and interval reached - -# Throws -- `IOError`: If the writer has been closed -""" -function write_record!(writer::DBNStreamWriter, record) - # Check if the stream is still open - if !isopen(writer.encoder.io) - throw(Base.IOError("Cannot write to closed DBNStreamWriter", 0)) - end - - # Update timestamps - if hasproperty(record, :hd) && hasproperty(record.hd, :ts_event) - ts = record.hd.ts_event - writer.first_ts = min(writer.first_ts, ts) - writer.last_ts = max(writer.last_ts, ts) - end - - # Write the record - write_record(writer.encoder, record) - writer.record_count += 1 - - # Auto-flush if enabled - if writer.auto_flush && (writer.record_count - writer.last_flush_count) >= writer.flush_interval - flush(writer.encoder.io) - writer.last_flush_count = writer.record_count - end -end - -""" - close_writer!(writer::DBNStreamWriter) - -Finalize and close the streaming writer, updating the header with final metadata. - -# Arguments -- `writer::DBNStreamWriter`: Writer to close - -# Details -Finalizes the file by: -- Flushing any remaining data -- Updating the header with final timestamps and record count -- Properly closing the file handle - -The header is rewritten with accurate metadata based on all records written. -""" -function close_writer!(writer::DBNStreamWriter) - # Flush any remaining data - flush(writer.encoder.io) - - # Save current position - current_pos = position(writer.encoder.io) - - # Update header with final timestamps and count - seekstart(writer.encoder.io) - - # Handle the case where no records were written - final_start_ts = writer.first_ts == typemax(Int64) ? 0 : writer.first_ts - final_end_ts = writer.last_ts == 0 ? 0 : writer.last_ts - - # Update metadata - writer.encoder.metadata = Metadata( - writer.encoder.metadata.version, - writer.encoder.metadata.dataset, - writer.encoder.metadata.schema, - final_start_ts, - final_end_ts, - UInt64(writer.record_count), - writer.encoder.metadata.stype_in, - writer.encoder.metadata.stype_out, - writer.encoder.metadata.ts_out, - writer.encoder.metadata.symbols, - writer.encoder.metadata.partial, - writer.encoder.metadata.not_found, - writer.encoder.metadata.mappings - ) - - # Rewrite header with updated metadata - write_header(writer.encoder) - - # Make sure we don't truncate the file - seek back to the end - if current_pos > position(writer.encoder.io) - seek(writer.encoder.io, current_pos) - end - - # Close the file - close(writer.encoder.io) -end - -""" - compress_dbn_file(input_file::String, output_file::String; - compression_level::Int=3, - delete_original::Bool=false) - -Compress a DBN file using Zstd compression. - -# Arguments -- `input_file::String`: Path to input DBN file -- `output_file::String`: Path for compressed output file -- `compression_level::Int`: Zstd compression level (default: 3) -- `delete_original::Bool`: Whether to delete input file after compression (default: false) - -# Returns -- `NamedTuple`: Compression statistics including: - - `original_size::Int`: Original file size in bytes - - `compressed_size::Int`: Compressed file size in bytes - - `compression_ratio::Float64`: Compression ratio (0.0-1.0) - - `space_saved::Int`: Bytes saved by compression - -# Details -Performs streaming compression to handle large files efficiently. -Preserves all metadata and record integrity. -""" -function compress_dbn_file(input_file::String, output_file::String; - compression_level::Int=3, - delete_original::Bool=false) - # Read header to get metadata - metadata = open(input_file, "r") do io - decoder = DBNDecoder(io) - read_header!(decoder) - decoder.metadata - end - - # Update metadata for compression - compressed_metadata = Metadata( - metadata.version, - metadata.dataset, - metadata.schema, - metadata.start_ts, - metadata.end_ts, - metadata.limit, - metadata.stype_in, - metadata.stype_out, - metadata.ts_out, - metadata.symbols, - metadata.partial, - metadata.not_found, - metadata.mappings - ) - - # Stream compress the file using Zstd compression - open(output_file, "w") do base_io - # Create a Zstd compression stream - compressed_io = TranscodingStream(ZstdCompressor(level=compression_level), base_io) - - try - encoder = DBNEncoder(compressed_io, compressed_metadata) - write_header(encoder) - - # Stream through input file - for record in DBNStream(input_file) - write_record(encoder, record) - end - - finalize_encoder(encoder) - finally - # Close the compression stream - close(compressed_io) - end - end - - # Force garbage collection to ensure file handles are released on Windows - GC.gc() - - # Get stats before potentially deleting original - original_size = filesize(input_file) - compressed_size = filesize(output_file) - compression_ratio = 1.0 - (compressed_size / original_size) - - # Optionally delete original - if delete_original - # Force GC again before deletion to ensure handles are released - GC.gc() - rm(input_file) - end - - return ( - original_size = original_size, - compressed_size = compressed_size, - compression_ratio = compression_ratio, - space_saved = original_size - compressed_size - ) -end - -""" - compress_daily_files(date::Date, base_dir::String; - pattern::Regex=r".*\\.dbn\$", - workers::Int=Threads.nthreads()) - -Compress multiple DBN files for a specific date in parallel. - -# Arguments -- `date::Date`: Date to process (looks for files containing "yyyy-mm-dd") -- `base_dir::String`: Directory containing DBN files -- `pattern::Regex`: File pattern to match (default: r".*\\.dbn\$") -- `workers::Int`: Number of parallel workers (default: thread count) - -# Returns -- `Vector`: Compression results for each file (or `nothing` for failures) - -# Details -Finds all uncompressed DBN files matching the date pattern and compresses -them in parallel. Original files are deleted after successful compression. -Provides detailed logging of compression results and any errors. - -# Example -```julia -results = compress_daily_files(Date("2024-01-01"), "data/") -``` -""" -function compress_daily_files(date::Date, base_dir::String; - pattern::Regex=r".*\.dbn$", - workers::Int=Threads.nthreads()) - - # Find all uncompressed DBN files for the date - date_str = Dates.format(date, "yyyy-mm-dd") - files = filter(readdir(base_dir, join=true)) do file - occursin(pattern, file) && occursin(date_str, file) - end - - # Compress in parallel - results = Vector{Any}(undef, length(files)) - - Threads.@threads for i in 1:length(files) - input_file = files[i] - output_file = replace(input_file, ".dbn" => ".dbn.zst") - - try - results[i] = compress_dbn_file(input_file, output_file, delete_original=true) - @info "Compressed $input_file" results[i]... - catch e - @error "Failed to compress $input_file" exception=e - results[i] = nothing - end - end - - return results +# DBN streaming functionality + +""" + DBNStream + +Iterator for streaming DBN file reading with automatic compression support. + +# Fields +- `decoder::DBNDecoder`: Decoder instance (stored to avoid allocation overhead) +- `cleanup::Ref{Bool}`: Flag to track if cleanup has been done + +# Usage +```julia +for record in DBNStream("data.dbn") + println(typeof(record)) +end +``` + +# Details +Provides memory-efficient streaming access to DBN files without loading +the entire file into memory. Automatically detects and handles Zstd compression. +Gracefully skips unknown record types. + +# Performance +`DBNStream` returns the abstract `Union` element type, so each iteration boxes +the record and allocates roughly ~120 bytes per record. This is acceptable for +exploration / debugging but bottlenecks high-throughput workloads. For +performance-critical loops over a known-schema file, prefer +[`foreach_record`](@ref): + +```julia +foreach_record("data.dbn", DBN.TradeMsg) do rec + # near-zero allocation per record +end +``` +""" +mutable struct DBNStream + decoder::DBNDecoder + cleanup::Ref{Bool} + + function DBNStream(filename::String) + decoder = DBNDecoder(filename) + new(decoder, Ref(false)) + end +end + +""" + Base.iterate(stream::DBNStream) + +Initialize iteration over a DBN stream. + +# Arguments +- `stream::DBNStream`: Stream to iterate over + +# Returns +- `Tuple`: (first_record, nothing) or `nothing` if empty +""" +Base.iterate(stream::DBNStream) = iterate(stream, nothing) + +""" + Base.iterate(stream::DBNStream, state) + +Continue iteration over a DBN stream. + +# Arguments +- `stream::DBNStream`: Stream being iterated +- `state`: Unused (kept for API compatibility) + +# Returns +- `Tuple`: (next_record, nothing) or `nothing` if end reached +""" +Base.iterate(stream::DBNStream, state) = begin + decoder = stream.decoder + + # Check if already cleaned up + if stream.cleanup[] + return nothing + end + + if eof(decoder.io) + # Clean up resources once + if !stream.cleanup[] + if decoder.io !== decoder.base_io + close(decoder.io) + end + if isa(decoder.base_io, IOStream) + close(decoder.base_io) + end + stream.cleanup[] = true + end + return nothing + end + + record = read_record(decoder) + if record === nothing + return iterate(stream, state) # Skip unknown records + end + return (record, nothing) +end + +""" + Base.IteratorSize(::Type{DBNStream}) + +Indicates that DBNStream has unknown size (cannot determine record count without reading). +""" +Base.IteratorSize(::Type{DBNStream}) = Base.SizeUnknown() +""" + Base.eltype(::Type{DBNStream}) + +Element type for DBNStream iterator (Any, since different record types are possible). +""" +Base.eltype(::Type{DBNStream}) = Any + +# ============================================================================ +# Internal Helpers (Shared by Eager Read and Callback Streaming) +# ============================================================================ + +# Map type to RType for validation +_type_to_rtype_stream(::Type{TradeMsg}) = RType.MBP_0_MSG +_type_to_rtype_stream(::Type{MBOMsg}) = RType.MBO_MSG +_type_to_rtype_stream(::Type{MBP1Msg}) = RType.MBP_1_MSG +_type_to_rtype_stream(::Type{MBP10Msg}) = RType.MBP_10_MSG +_type_to_rtype_stream(::Type{OHLCVMsg}) = RType.OHLCV_1S_MSG # Default to 1s +_type_to_rtype_stream(::Type{StatusMsg}) = RType.STATUS_MSG +_type_to_rtype_stream(::Type{InstrumentDefMsg}) = RType.INSTRUMENT_DEF_MSG +_type_to_rtype_stream(::Type{ImbalanceMsg}) = RType.IMBALANCE_MSG +_type_to_rtype_stream(::Type{StatMsg}) = RType.STAT_MSG +_type_to_rtype_stream(::Type{CMBP1Msg}) = RType.CMBP_1_MSG +_type_to_rtype_stream(::Type{CBBO1sMsg}) = RType.CBBO_1S_MSG +_type_to_rtype_stream(::Type{CBBO1mMsg}) = RType.CBBO_1M_MSG +_type_to_rtype_stream(::Type{TCBBOMsg}) = RType.TCBBO_MSG +_type_to_rtype_stream(::Type{BBO1sMsg}) = RType.BBO_1S_MSG +_type_to_rtype_stream(::Type{BBO1mMsg}) = RType.BBO_1M_MSG + +""" + record_type_for_dbn_schema(schema::Schema.T) -> Union{Type, Nothing} + +Map a DBN `Schema` to the concrete record struct it produces, when the schema +is type-pure AND DBN.jl has a `_read_typed_record_stream` overload for that +type. Returns `nothing` otherwise (callers should fall back to the generic +`read_record` Union path). + +Used by `read_dbn` and `read_dbn_with_metadata` to dispatch to the typed +zero-allocation reader when possible. +""" +function record_type_for_dbn_schema(schema::Schema.T) + schema == Schema.TRADES && return TradeMsg + schema == Schema.MBO && return MBOMsg + schema == Schema.MBP_1 && return MBP1Msg + schema == Schema.MBP_10 && return MBP10Msg + schema == Schema.TBBO && return MBP1Msg # TBBO records are MBP1Msg layout + schema == Schema.OHLCV_1S && return OHLCVMsg + schema == Schema.OHLCV_1M && return OHLCVMsg + schema == Schema.OHLCV_1H && return OHLCVMsg + schema == Schema.OHLCV_1D && return OHLCVMsg + schema == Schema.DEFINITION && return InstrumentDefMsg + schema == Schema.STATUS && return StatusMsg + schema == Schema.IMBALANCE && return ImbalanceMsg + schema == Schema.STATISTICS && return StatMsg + schema == Schema.CMBP_1 && return CMBP1Msg + schema == Schema.CBBO_1S && return CBBO1sMsg + schema == Schema.CBBO_1M && return CBBO1mMsg + schema == Schema.TCBBO && return TCBBOMsg + schema == Schema.BBO_1S && return BBO1sMsg + schema == Schema.BBO_1M && return BBO1mMsg + return nothing +end + +# Type-stable record reading helpers (shared by eager read and callback API) +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{TradeMsg}, hd::RecordHeader) + return read_trade_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{MBOMsg}, hd::RecordHeader) + return read_mbo_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{MBP1Msg}, hd::RecordHeader) + return read_mbp1_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{MBP10Msg}, hd::RecordHeader) + return read_mbp10_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{OHLCVMsg}, hd::RecordHeader) + return read_ohlcv_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{StatusMsg}, hd::RecordHeader) + return read_status_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{InstrumentDefMsg}, hd::RecordHeader) + return read_instrument_def_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{ImbalanceMsg}, hd::RecordHeader) + return read_imbalance_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{StatMsg}, hd::RecordHeader) + return read_stat_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{CMBP1Msg}, hd::RecordHeader) + return read_cmbp1_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{CBBO1sMsg}, hd::RecordHeader) + return read_cbbo1s_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{CBBO1mMsg}, hd::RecordHeader) + return read_cbbo1m_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{TCBBOMsg}, hd::RecordHeader) + return read_tcbbo_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{BBO1sMsg}, hd::RecordHeader) + return read_bbo1s_msg(decoder, hd) +end + +@inline function _read_typed_record_stream(decoder::DBNDecoder, ::Type{BBO1mMsg}, hd::RecordHeader) + return read_bbo1m_msg(decoder, hd) +end + +# ============================================================================ +# Near-Zero-Allocation Callback-Based Streaming +# ============================================================================ + +""" + foreach_record(f::Function, filename::String, ::Type{T}) where T + +Highly optimized callback-based streaming with minimal allocations. + +# Arguments +- `f::Function`: Callback function that processes each record +- `filename::String`: Path to DBN file +- `::Type{T}`: Record type (e.g., `TradeMsg`) + +# Performance +Achieves near-zero allocations (typically <50 total allocations for any size file) +when the callback uses `Ref` for mutable state. The Julia compiler optimizes away +most per-record allocations, making this significantly faster than iterator-based +streaming for pure processing workloads. + +# Performance tip: Use Ref for mutable state! +To achieve true near-zero allocations, use `Ref` for any mutable state in your +callback instead of plain variables: + +```julia +# ✅ GOOD: Only ~44 allocations total (not per record!) +total = Ref(0.0) +foreach_record("trades.dbn", TradeMsg) do trade + total[] += price_to_float(trade.price) +end +println(total[]) + +# ❌ SLOWER: May allocate per record due to closure overhead +total = 0.0 +foreach_record("trades.dbn", TradeMsg) do trade + total += price_to_float(trade.price) # Closure capture causes allocations +end +``` + +# More examples +```julia +# Count records by side (using Ref for mutable Dict) +counts = Ref(Dict('A' => 0, 'B' => 0)) +foreach_record("trades.dbn", TradeMsg) do trade + counts[][Char(trade.side)] += 1 +end + +# Collect records (push! automatically copies bitstypes) +trades = TradeMsg[] +sizehint!(trades, 100_000) +foreach_record("trades.dbn", TradeMsg) do trade + push!(trades, trade) # Safe: TradeMsg is copied +end + +# Filter while streaming - only store what you need +high_volume = TradeMsg[] +foreach_record("trades.dbn", TradeMsg) do trade + if trade.size > 1000 + push!(high_volume, trade) + end +end +``` + +# Note on bitstypes +All DBN message types are immutable bitstypes, so `push!(array, record)` +automatically makes a value copy. You can safely store records in arrays without +explicit copying. +""" +function foreach_record(f::Function, filename::String, ::Type{T}) where T + decoder = DBNDecoder(filename) + _foreach_record_impl(f, decoder, T) + return nothing +end + +# Internal implementation that works with an open decoder +# This allows eager read to avoid opening the file twice +function _foreach_record_impl(f::Function, decoder::DBNDecoder, ::Type{T}) where T + expected_rtype = _type_to_rtype_stream(T) + + # Pre-allocate a single buffer that we'll reuse + # Since T is a bitstype (all our message types are), we can safely mutate it + buffer = Ref{T}() + + try + while !eof(decoder.io) + # Read header + hd_result = read_record_header(decoder.io) + + # Handle unknown record types. `record_length` from + # read_record_header is in 4-byte units; the 2 bytes + # (length + rtype) of the header are already consumed. + if hd_result isa Tuple + _, rtype_raw, record_length = hd_result + skip(decoder.io, Int(record_length) * LENGTH_MULTIPLIER - 2) + continue + end + + hd = hd_result + + # Verify type matches expected + if hd.rtype != expected_rtype + # Special handling for OHLCV variants + if T === OHLCVMsg && hd.rtype in (RType.OHLCV_1S_MSG, RType.OHLCV_1M_MSG, RType.OHLCV_1H_MSG, RType.OHLCV_1D_MSG) + # OK, continue + else + error("Expected $(T) (rtype=$(expected_rtype)) but got rtype=$(hd.rtype)") + end + end + + # Read record directly into buffer. + # A reader can return nothing for a malformed record it skipped. + rec = _read_typed_record_stream(decoder, T, hd) + rec === nothing && continue + buffer[] = rec + + # Call user function with buffer contents + f(buffer[]) + end + finally + # Clean up + if decoder.io !== decoder.base_io + close(decoder.io) + end + if isa(decoder.base_io, IOStream) + close(decoder.base_io) + end + end +end + +# ============================================================================ +# Typed Streaming With Separate Control-Record Routing +# ============================================================================ + +""" + foreach_record_with_control(f_data, f_control, filename::String, ::Type{T}) where T + foreach_record_with_control(f_data, f_control, decoder::DBNDecoder, ::Type{T}) where T + +Like [`foreach_record`](@ref) but split into two callbacks: `f_data` receives +concrete `T`-typed records (the expected schema type) via a reused `Ref{T}()` +buffer with no per-record allocation, while `f_control` receives the +Union-typed value for *control* records (`ErrorMsg`, `SystemMsg`, +`SymbolMappingMsg`) that can interleave with data on a live stream. + +The typed-record path is the same hot loop as `_foreach_record_impl` and +achieves the same near-zero allocation profile. Control records do allocate +the Union container (one per control record), but they are rare relative +to data and don't contribute meaningfully to GC pressure. + +This is the primitive that backs the typed-channel live reader: +data records flow into a `Channel{T}` and control records flow into a +separate `Channel{DBNRecord}` for status/heartbeat/error handling. + +# Arguments +- `f_data`: called as `f_data(rec::T)` for every record whose `rtype` matches + `_type_to_rtype_stream(T)` (or any OHLCV variant when `T === OHLCVMsg`). +- `f_control`: called as `f_control(rec)` for every `ErrorMsg`, `SystemMsg`, + or `SymbolMappingMsg`. `rec` is one of those concrete types but typed + as the `DBNRecord` Union. +- `filename` / `decoder`: same as [`foreach_record`](@ref). +- `T`: the expected data record type (must have a `_type_to_rtype_stream` + overload — see [`record_type_for_dbn_schema`](@ref) for the supported set). + +Unknown rtypes (anything that's neither `T`-matching nor a control rtype) +are silently skipped, matching `read_record`'s permissive behaviour. This +mirrors what production live streams need: gateway-side schema additions +shouldn't crash existing consumers. +""" +function foreach_record_with_control(f_data::Function, f_control::Function, + filename::String, ::Type{T}) where T + decoder = DBNDecoder(filename) + _foreach_record_with_control_impl(f_data, f_control, decoder, T) + return nothing +end + +function foreach_record_with_control(f_data::Function, f_control::Function, + decoder::DBNDecoder, ::Type{T}) where T + _foreach_record_with_control_impl(f_data, f_control, decoder, T) + return nothing +end + +function _foreach_record_with_control_impl(f_data::Function, f_control::Function, + decoder::DBNDecoder, ::Type{T}) where T + expected_rtype = _type_to_rtype_stream(T) + # Reused buffer for the typed-record path — same trick as + # _foreach_record_impl; immutable bitstype, value-copied through `f_data`. + buffer = Ref{T}() + + try + while !eof(decoder.io) + hd_result = read_record_header(decoder.io) + + # Unknown-record-type case (header decoded as Tuple). Skip. + # `record_length` is in 4-byte units; we already consumed the + # 2 header bytes (length + rtype). + if hd_result isa Tuple + _, _, record_length = hd_result + skip(decoder.io, Int(record_length) * LENGTH_MULTIPLIER - 2) + continue + end + + hd = hd_result + + # Data record: matches expected rtype, or OHLCV-variant when T is OHLCVMsg. + is_match = hd.rtype == expected_rtype || + (T === OHLCVMsg && hd.rtype in (RType.OHLCV_1S_MSG, + RType.OHLCV_1M_MSG, + RType.OHLCV_1H_MSG, + RType.OHLCV_1D_MSG)) + if is_match + # A reader can return nothing for a malformed record it skipped. + rec = _read_typed_record_stream(decoder, T, hd) + rec === nothing && continue + buffer[] = rec + f_data(buffer[]) + continue + end + + # Control records: route via the generic Union dispatch. + if hd.rtype == RType.ERROR_MSG || + hd.rtype == RType.SYSTEM_MSG || + hd.rtype == RType.SYMBOL_MAPPING_MSG + rec = read_record_dispatch(decoder, hd, hd.rtype) + rec === nothing || f_control(rec) + continue + end + + # Unknown / unsupported rtype: skip. hd.length is in 4-byte units; + # 16 bytes of header already consumed. + skip(decoder.io, Int(hd.length) * LENGTH_MULTIPLIER - 16) + end + finally + if decoder.io !== decoder.base_io + close(decoder.io) + end + if isa(decoder.base_io, IOStream) + close(decoder.base_io) + end + end +end + +# Convenience functions for callback-based streaming + +""" + foreach_trade(f::Function, filename::String) + +Near-zero-allocation streaming of trade data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_trade(f::Function, filename::String) = foreach_record(f, filename, TradeMsg) + +""" + foreach_mbo(f::Function, filename::String) + +Near-zero-allocation streaming of MBO data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_mbo(f::Function, filename::String) = foreach_record(f, filename, MBOMsg) + +""" + foreach_mbp1(f::Function, filename::String) + +Near-zero-allocation streaming of MBP-1 (top-of-book) data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_mbp1(f::Function, filename::String) = foreach_record(f, filename, MBP1Msg) + +""" + foreach_mbp10(f::Function, filename::String) + +Near-zero-allocation streaming of MBP-10 (10-level depth) data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_mbp10(f::Function, filename::String) = foreach_record(f, filename, MBP10Msg) + +""" + foreach_tbbo(f::Function, filename::String) + +Near-zero-allocation streaming of TBBO (Trade BBO) data using callback pattern. +Uses MBP-1 records. See `foreach_record` for usage and performance tips. +""" +foreach_tbbo(f::Function, filename::String) = foreach_record(f, filename, MBP1Msg) + +""" + foreach_ohlcv(f::Function, filename::String) + +Near-zero-allocation streaming of OHLCV (candlestick) data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_ohlcv(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) + +""" + foreach_ohlcv_1s(f::Function, filename::String) + +Near-zero-allocation streaming of 1-second OHLCV data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_ohlcv_1s(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) + +""" + foreach_ohlcv_1m(f::Function, filename::String) + +Near-zero-allocation streaming of 1-minute OHLCV data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_ohlcv_1m(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) + +""" + foreach_ohlcv_1h(f::Function, filename::String) + +Near-zero-allocation streaming of 1-hour OHLCV data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_ohlcv_1h(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) + +""" + foreach_ohlcv_1d(f::Function, filename::String) + +Near-zero-allocation streaming of 1-day OHLCV data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_ohlcv_1d(f::Function, filename::String) = foreach_record(f, filename, OHLCVMsg) + +# Consolidated/BBO message streaming +""" + foreach_cmbp1(f::Function, filename::String) + +Near-zero-allocation streaming of Consolidated MBP-1 data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_cmbp1(f::Function, filename::String) = foreach_record(f, filename, CMBP1Msg) + +""" + foreach_cbbo1s(f::Function, filename::String) + +Near-zero-allocation streaming of Consolidated BBO 1-second data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_cbbo1s(f::Function, filename::String) = foreach_record(f, filename, CBBO1sMsg) + +""" + foreach_cbbo1m(f::Function, filename::String) + +Near-zero-allocation streaming of Consolidated BBO 1-minute data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_cbbo1m(f::Function, filename::String) = foreach_record(f, filename, CBBO1mMsg) + +""" + foreach_tcbbo(f::Function, filename::String) + +Near-zero-allocation streaming of Top Consolidated BBO data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_tcbbo(f::Function, filename::String) = foreach_record(f, filename, TCBBOMsg) + +""" + foreach_bbo1s(f::Function, filename::String) + +Near-zero-allocation streaming of BBO 1-second data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_bbo1s(f::Function, filename::String) = foreach_record(f, filename, BBO1sMsg) + +""" + foreach_bbo1m(f::Function, filename::String) + +Near-zero-allocation streaming of BBO 1-minute data using callback pattern. +See `foreach_record` for usage and performance tips. +""" +foreach_bbo1m(f::Function, filename::String) = foreach_record(f, filename, BBO1mMsg) + +""" + DBNStreamWriter + +Streaming writer for real-time DBN data capture with automatic timestamp tracking. + +# Fields +- `encoder::DBNEncoder`: Underlying encoder for writing data +- `record_count::Int64`: Number of records written +- `first_ts::Int64`: First timestamp encountered +- `last_ts::Int64`: Last timestamp encountered +- `auto_flush::Bool`: Whether to automatically flush data +- `flush_interval::Int`: Number of records between automatic flushes +- `last_flush_count::Int64`: Record count at last flush + +# Usage +```julia +writer = DBNStreamWriter("live.dbn", "XNAS", Schema.TRADES) +write_record!(writer, trade_record) +close_writer!(writer) +``` +""" +mutable struct DBNStreamWriter + encoder::DBNEncoder + record_count::Int64 + first_ts::Int64 + last_ts::Int64 + auto_flush::Bool + flush_interval::Int + last_flush_count::Int64 +end + +""" + DBNStreamWriter(filename::String, dataset::String, schema::Schema.T; + symbols::Vector{String}=String[], + auto_flush::Bool=true, + flush_interval::Int=1000) + +Construct a streaming writer for real-time DBN data capture. + +# Arguments +- `filename::String`: Output file path +- `dataset::String`: Dataset identifier +- `schema::Schema.T`: Data schema type +- `symbols::Vector{String}`: List of symbols (optional) +- `auto_flush::Bool`: Enable automatic flushing (default: true) +- `flush_interval::Int`: Records between flushes (default: 1000) + +# Returns +- `DBNStreamWriter`: Writer instance ready for recording + +# Details +Creates a writer with placeholder timestamps that will be updated as records +are written. The header is written immediately but will be updated with +final timestamps when the writer is closed. +""" +function DBNStreamWriter(filename::String, dataset::String, schema::Schema.T; + symbols::Vector{String}=String[], + auto_flush::Bool=true, + flush_interval::Int=1000) + # Create metadata with placeholder timestamps (using 0 instead of typemin) + metadata = Metadata( + UInt8(DBN_VERSION), + dataset, + schema, + 0, # Will update with first record + 0, # Will update with last record + UInt64(0), + SType.RAW_SYMBOL, + SType.RAW_SYMBOL, + false, + symbols, + String[], + String[], + Tuple{String,String,Int64,Int64}[] + ) + + io = open(filename, "w") + encoder = DBNEncoder(io, metadata) + + # Write header (will update it later) + write_header(encoder) + + return DBNStreamWriter(encoder, 0, typemax(Int64), 0, + auto_flush, flush_interval, 0) +end + +""" + write_record!(writer::DBNStreamWriter, record) + +Write a record to the streaming writer and update timestamps. + +# Arguments +- `writer::DBNStreamWriter`: Writer instance +- `record`: Record to write (any DBN message type) + +# Details +Writes the record and automatically: +- Updates first/last timestamp tracking +- Increments record count +- Performs auto-flush if enabled and interval reached + +# Throws +- `IOError`: If the writer has been closed +""" +function write_record!(writer::DBNStreamWriter, record) + # Check if the stream is still open + if !isopen(writer.encoder.io) + throw(Base.IOError("Cannot write to closed DBNStreamWriter", 0)) + end + + # Update timestamps + if hasproperty(record, :hd) && hasproperty(record.hd, :ts_event) + ts = record.hd.ts_event + writer.first_ts = min(writer.first_ts, ts) + writer.last_ts = max(writer.last_ts, ts) + end + + # Write the record + write_record(writer.encoder, record) + writer.record_count += 1 + + # Auto-flush if enabled + if writer.auto_flush && (writer.record_count - writer.last_flush_count) >= writer.flush_interval + flush(writer.encoder.io) + writer.last_flush_count = writer.record_count + end +end + +""" + close_writer!(writer::DBNStreamWriter) + +Finalize and close the streaming writer, updating the header with final metadata. + +# Arguments +- `writer::DBNStreamWriter`: Writer to close + +# Details +Finalizes the file by: +- Flushing any remaining data +- Updating the header with final timestamps and record count +- Properly closing the file handle + +The header is rewritten with accurate metadata based on all records written. +""" +function close_writer!(writer::DBNStreamWriter) + # Flush any remaining data + flush(writer.encoder.io) + + # Save current position + current_pos = position(writer.encoder.io) + + # Update header with final timestamps and count + seekstart(writer.encoder.io) + + # Handle the case where no records were written + final_start_ts = writer.first_ts == typemax(Int64) ? 0 : writer.first_ts + final_end_ts = writer.last_ts == 0 ? 0 : writer.last_ts + + # Update metadata + writer.encoder.metadata = Metadata( + writer.encoder.metadata.version, + writer.encoder.metadata.dataset, + writer.encoder.metadata.schema, + final_start_ts, + final_end_ts, + UInt64(writer.record_count), + writer.encoder.metadata.stype_in, + writer.encoder.metadata.stype_out, + writer.encoder.metadata.ts_out, + writer.encoder.metadata.symbols, + writer.encoder.metadata.partial, + writer.encoder.metadata.not_found, + writer.encoder.metadata.mappings + ) + + # Rewrite header with updated metadata + write_header(writer.encoder) + + # Make sure we don't truncate the file - seek back to the end + if current_pos > position(writer.encoder.io) + seek(writer.encoder.io, current_pos) + end + + # Close the file + close(writer.encoder.io) +end + +""" + compress_dbn_file(input_file::String, output_file::String; + compression_level::Int=3, + delete_original::Bool=false) + +Compress a DBN file using Zstd compression. + +# Arguments +- `input_file::String`: Path to input DBN file +- `output_file::String`: Path for compressed output file +- `compression_level::Int`: Zstd compression level (default: 3) +- `delete_original::Bool`: Whether to delete input file after compression (default: false) + +# Returns +- `NamedTuple`: Compression statistics including: + - `original_size::Int`: Original file size in bytes + - `compressed_size::Int`: Compressed file size in bytes + - `compression_ratio::Float64`: Compression ratio (0.0-1.0) + - `space_saved::Int`: Bytes saved by compression + +# Details +Performs streaming compression to handle large files efficiently. +Preserves all metadata and record integrity. +""" +function compress_dbn_file(input_file::String, output_file::String; + compression_level::Int=3, + delete_original::Bool=false) + # Read header to get metadata + metadata = open(input_file, "r") do io + decoder = DBNDecoder(io) + read_header!(decoder) + decoder.metadata + end + + # Update metadata for compression + compressed_metadata = Metadata( + metadata.version, + metadata.dataset, + metadata.schema, + metadata.start_ts, + metadata.end_ts, + metadata.limit, + metadata.stype_in, + metadata.stype_out, + metadata.ts_out, + metadata.symbols, + metadata.partial, + metadata.not_found, + metadata.mappings + ) + + # Stream compress the file using Zstd compression + open(output_file, "w") do base_io + # Create a Zstd compression stream + compressed_io = TranscodingStream(ZstdCompressor(level=compression_level), base_io) + + try + encoder = DBNEncoder(compressed_io, compressed_metadata) + write_header(encoder) + + # Stream through input file + for record in DBNStream(input_file) + write_record(encoder, record) + end + + finalize_encoder(encoder) + finally + # Close the compression stream + close(compressed_io) + end + end + + # Force garbage collection to ensure file handles are released on Windows + GC.gc() + + # Get stats before potentially deleting original + original_size = filesize(input_file) + compressed_size = filesize(output_file) + compression_ratio = 1.0 - (compressed_size / original_size) + + # Optionally delete original + if delete_original + # Force GC again before deletion to ensure handles are released + GC.gc() + rm(input_file) + end + + return ( + original_size = original_size, + compressed_size = compressed_size, + compression_ratio = compression_ratio, + space_saved = original_size - compressed_size + ) +end + +""" + compress_daily_files(date::Date, base_dir::String; + pattern::Regex=r".*\\.dbn\$", + workers::Int=Threads.nthreads()) + +Compress multiple DBN files for a specific date in parallel. + +# Arguments +- `date::Date`: Date to process (looks for files containing "yyyy-mm-dd") +- `base_dir::String`: Directory containing DBN files +- `pattern::Regex`: File pattern to match (default: r".*\\.dbn\$") +- `workers::Int`: Number of parallel workers (default: thread count) + +# Returns +- `Vector`: Compression results for each file (or `nothing` for failures) + +# Details +Finds all uncompressed DBN files matching the date pattern and compresses +them in parallel. Original files are deleted after successful compression. +Provides detailed logging of compression results and any errors. + +# Example +```julia +results = compress_daily_files(Date("2024-01-01"), "data/") +``` +""" +function compress_daily_files(date::Date, base_dir::String; + pattern::Regex=r".*\.dbn$", + workers::Int=Threads.nthreads()) + + # Find all uncompressed DBN files for the date + date_str = Dates.format(date, "yyyy-mm-dd") + files = filter(readdir(base_dir, join=true)) do file + occursin(pattern, file) && occursin(date_str, file) + end + + # Compress in parallel + results = Vector{Any}(undef, length(files)) + + Threads.@threads for i in 1:length(files) + input_file = files[i] + output_file = replace(input_file, ".dbn" => ".dbn.zst") + + try + results[i] = compress_dbn_file(input_file, output_file, delete_original=true) + @info "Compressed $input_file" results[i]... + catch e + @error "Failed to compress $input_file" exception=e + results[i] = nothing + end + end + + return results end \ No newline at end of file diff --git a/src/types.jl b/src/types.jl index c7507e54..418e4e6a 100644 --- a/src/types.jl +++ b/src/types.jl @@ -1,824 +1,824 @@ -# DBN types, enums, and data structures - -# Constants -"""The current DBN format version supported by this implementation.""" -const DBN_VERSION = 3 - -"""Fixed-length symbol string size for DBN v2/v3 metadata and records.""" -const SYMBOL_CSTR_LEN = 71 - -"""Fixed-point price scaling factor for converting between integer and float prices.""" -const FIXED_PRICE_SCALE = Int32(1_000_000_000) - -"""Sentinel value indicating an undefined or missing price.""" -const UNDEF_PRICE = typemax(Int64) - -"""Sentinel value indicating an undefined or missing order size.""" -const UNDEF_ORDER_SIZE = typemax(UInt32) - -"""Sentinel value indicating an undefined or missing timestamp.""" -const UNDEF_TIMESTAMP = typemax(Int64) - -"""Multiplier for converting the length field to bytes in record headers.""" -const LENGTH_MULTIPLIER = 4 - -# Enums using EnumX for better namespace management - -""" - Schema - -DBN data schemas representing different types of market data. - -# Values -- `MBO`: Market-by-order data -- `MBP_1`: Market-by-price with book depth 1 (TBBO) -- `MBP_10`: Market-by-price with book depth 10 -- `TBBO`: Top-of-book bid/offer -- `TRADES`: Trade messages only -- `OHLCV_1S`: OHLCV data at 1-second intervals -- `OHLCV_1M`: OHLCV data at 1-minute intervals -- `OHLCV_1H`: OHLCV data at 1-hour intervals -- `OHLCV_1D`: OHLCV data at 1-day intervals -- `DEFINITION`: Instrument definition data -- `STATISTICS`: Market statistics -- `STATUS`: Exchange status messages -- `IMBALANCE`: Order imbalance data -- `CBBO`: Consolidated best bid/offer -- `CBBO_1S`: Consolidated BBO at 1-second intervals -- `CBBO_1M`: Consolidated BBO at 1-minute intervals -- `CMBP_1`: Consolidated market-by-price depth 1 -- `TCBBO`: Trade-consolidated BBO -- `BBO_1S`: BBO at 1-second intervals -- `BBO_1M`: BBO at 1-minute intervals -""" -@enumx Schema::UInt16 begin - MBO = 0 - MBP_1 = 1 - MBP_10 = 2 - TBBO = 3 - TRADES = 4 - OHLCV_1S = 5 - OHLCV_1M = 6 - OHLCV_1H = 7 - OHLCV_1D = 8 - DEFINITION = 9 - STATISTICS = 10 - STATUS = 11 - IMBALANCE = 12 - CBBO = 13 - CBBO_1S = 14 - CBBO_1M = 15 - CMBP_1 = 16 - TCBBO = 17 - BBO_1S = 18 - BBO_1M = 19 - MIX = 0xFFFF -end - -""" - Compression - -Compression algorithms supported for DBN files. - -# Values -- `NONE`: No compression -- `ZSTD`: Zstandard compression -""" -@enumx Compression::UInt8 begin - NONE = 0 - ZSTD = 1 -end - -""" - Encoding - -Output encoding formats for market data. - -# Values -- `DBN`: Databento Binary Encoding -- `CSV`: Comma-separated values -- `JSON`: JavaScript Object Notation -""" -@enumx Encoding::UInt8 begin - DBN = 0 - CSV = 1 - JSON = 2 -end - -""" - SType - -Symbol types for identifying instruments in DBN data. Numeric values match the -official Databento DBN spec — wire-encoded as a `UInt8` in metadata and in -`SymbolMappingMsg` (v2+). - -# Values -- `INSTRUMENT_ID = 0`: Numeric instrument identifier -- `RAW_SYMBOL = 1`: Raw symbol string from exchange -- `SMART = 2`: Deprecated alias (was split into `CONTINUOUS` and `PARENT`) -- `CONTINUOUS = 3`: Continuous contract symbol -- `PARENT = 4`: Parent symbol for derived instruments (e.g. `SPXW.OPT`) -- `NASDAQ_SYMBOL = 5`: Nasdaq-specific symbol -- `CMS_SYMBOL = 6`: CMS symbol -- `ISIN = 7`: ISO 6166 International Securities Identification Number -- `US_CODE = 8`: US Code (CUSIP-style) -- `BBG_COMP_ID = 9`: Bloomberg composite ID -- `BBG_COMP_TICKER = 10`: Bloomberg composite ticker -- `FIGI = 11`: OpenFIGI identifier -- `FIGI_TICKER = 12`: OpenFIGI ticker -- `UNDEF = 255`: Unset/undefined sentinel (`0xFF`). Databento wire-encodes an - unset `stype` as `0xFF`; this member lets non-nullable `stype` fields (e.g. - in `SymbolMappingMsg`) represent and round-trip that sentinel. Metadata's - nullable `stype_in` decodes `0xFF` to `nothing` instead. -""" -@enumx SType::UInt8 begin - INSTRUMENT_ID = 0 - RAW_SYMBOL = 1 - SMART = 2 # deprecated (kept for round-trip with v1 wire data) - CONTINUOUS = 3 - PARENT = 4 - NASDAQ_SYMBOL = 5 - CMS_SYMBOL = 6 - ISIN = 7 - US_CODE = 8 - BBG_COMP_ID = 9 - BBG_COMP_TICKER = 10 - FIGI = 11 - FIGI_TICKER = 12 - UNDEF = 255 # 0xFF "unset" sentinel; see docstring -end - -""" - RType - -Record types for different kinds of market data messages in DBN format. - -# Values -- `MBP_0_MSG`: Trades (book depth 0) -- `MBP_1_MSG`: TBBO/MBP-1 (book depth 1) -- `MBP_10_MSG`: MBP-10 (book depth 10) -- `STATUS_MSG`: Exchange status record -- `INSTRUMENT_DEF_MSG`: Instrument definition record -- `IMBALANCE_MSG`: Order imbalance record -- `ERROR_MSG`: Error record from live gateway -- `SYMBOL_MAPPING_MSG`: Symbol mapping record from live gateway -- `SYSTEM_MSG`: Non-error record from live gateway -- `STAT_MSG`: Statistics record from publisher -- `OHLCV_1S_MSG`: OHLCV at 1-second cadence -- `OHLCV_1M_MSG`: OHLCV at 1-minute cadence -- `OHLCV_1H_MSG`: OHLCV at hourly cadence -- `OHLCV_1D_MSG`: OHLCV at daily cadence -- `MBO_MSG`: Market-by-order record -- `CMBP_1_MSG`: Consolidated market-by-price with book depth 1 -- `CBBO_1S_MSG`: Consolidated market-by-price with book depth 1 at 1-second cadence -- `CBBO_1M_MSG`: Consolidated market-by-price with book depth 1 at 1-minute cadence -- `TCBBO_MSG`: Consolidated market-by-price with book depth 1 (trades only) -- `BBO_1S_MSG`: Market-by-price with book depth 1 at 1-second cadence -- `BBO_1M_MSG`: Market-by-price with book depth 1 at 1-minute cadence -""" -@enumx RType::UInt8 begin - MBP_0_MSG = 0x00 # Trades (book depth 0) - MBP_1_MSG = 0x01 # TBBO/MBP-1 (book depth 1) - MBP_10_MSG = 0x0A # MBP-10 (book depth 10) - STATUS_MSG = 0x12 # Exchange status record - INSTRUMENT_DEF_MSG = 0x13 # Instrument definition record - IMBALANCE_MSG = 0x14 # Order imbalance record - ERROR_MSG = 0x15 # Error record from live gateway - SYMBOL_MAPPING_MSG = 0x16 # Symbol mapping record from live gateway - SYSTEM_MSG = 0x17 # Non-error record from live gateway - STAT_MSG = 0x18 # Statistics record from publisher - OHLCV_1S_MSG = 0x20 # OHLCV at 1-second cadence - OHLCV_1M_MSG = 0x21 # OHLCV at 1-minute cadence - OHLCV_1H_MSG = 0x22 # OHLCV at hourly cadence - OHLCV_1D_MSG = 0x23 # OHLCV at daily cadence - MBO_MSG = 0xA0 # Market-by-order record - CMBP_1_MSG = 0xB1 # Consolidated market-by-price with book depth 1 - CBBO_1S_MSG = 0xC0 # Consolidated market-by-price with book depth 1 at 1-second cadence - CBBO_1M_MSG = 0xC1 # Consolidated market-by-price with book depth 1 at 1-minute cadence - TCBBO_MSG = 0xC2 # Consolidated market-by-price with book depth 1 (trades only) - BBO_1S_MSG = 0xC3 # Market-by-price with book depth 1 at 1-second cadence - BBO_1M_MSG = 0xC4 # Market-by-price with book depth 1 at 1-minute cadence -end - -""" - Action - -Market actions that can be applied to orders or trades. - -# Values -- `ADD`: Insert a new order into the book -- `MODIFY`: Change an order's price and/or size -- `CANCEL`: Fully or partially cancel an order from the book -- `CLEAR`: Remove all resting orders for the instrument -- `TRADE`: An aggressing order traded. Does not affect the book -- `FILL`: A resting order was filled. Does not affect the book -- `NONE`: No action: does not affect the book, but may carry flags or other information -""" -@enumx Action::UInt8 begin - ADD = UInt8('A') # Insert a new order into the book - MODIFY = UInt8('M') # Change an order's price and/or size - CANCEL = UInt8('C') # Fully or partially cancel an order from the book - CLEAR = UInt8('R') # Remove all resting orders for the instrument - TRADE = UInt8('T') # An aggressing order traded. Does not affect the book - FILL = UInt8('F') # A resting order was filled. Does not affect the book - NONE = UInt8('N') # No action: does not affect the book, but may carry flags or other information -end - -""" - Side - -Market sides for orders and trades. - -# Values -- `ASK`: Ask/offer side (sell orders) -- `BID`: Bid side (buy orders) -- `NONE`: No specific side or not applicable -""" -@enumx Side::UInt8 begin - ASK = UInt8('A') - BID = UInt8('B') - NONE = UInt8('N') -end - -""" - InstrumentClass - -Classification of financial instruments. Values match Databento's DBN -`InstrumentClass` enum. - -# Values -- `STOCK`: Equity instruments -- `CALL`: Call option contracts -- `PUT`: Put option contracts -- `FUTURE`: Futures contracts -- `INDEX`: Index instruments -- `BOND`: Fixed income securities -- `MIXED_SPREAD`: Mixed spread instruments -- `FUTURE_SPREAD`: Futures spread instruments -- `OPTION_SPREAD`: Option spread instruments -- `FX_SPOT`: Foreign exchange spot -- `COMMODITY_SPOT`: Commodity spot -- `UNKNOWN_0`, `UNKNOWN_45`: Numeric fallback values for unknown classes -""" -@enumx InstrumentClass::UInt8 begin - BOND = UInt8('B') - CALL = UInt8('C') - FUTURE = UInt8('F') - INDEX = UInt8('I') - STOCK = UInt8('K') - MIXED_SPREAD = UInt8('M') - PUT = UInt8('P') - FUTURE_SPREAD = UInt8('S') - OPTION_SPREAD = UInt8('T') - FX_SPOT = UInt8('X') - COMMODITY_SPOT = UInt8('Y') - # Also support numeric values - UNKNOWN_0 = 0 - UNKNOWN_45 = 45 - OTHER = UInt8('?') -end - -# Basic structures - -""" - VersionUpgradePolicy - -Encapsulates the version upgrade policy for DBN files. - -# Fields -- `upgrade_policy::UInt8`: Policy for handling version upgrades -""" -struct VersionUpgradePolicy - upgrade_policy::UInt8 -end - -""" - DatasetCondition - -Conditions and constraints for a dataset. - -# Fields -- `last_ts_out::Int64`: Last timestamp output -- `start_ts::Int64`: Dataset start timestamp -- `end_ts::Int64`: Dataset end timestamp -- `limit::UInt64`: Record limit for the dataset -""" -struct DatasetCondition - last_ts_out::Int64 - start_ts::Int64 - end_ts::Int64 - limit::UInt64 -end - -""" - Metadata - -Metadata information for a DBN dataset. - -# Fields -- `version::UInt8`: DBN format version -- `dataset::String`: Dataset identifier -- `schema::Schema.T`: Data schema type -- `start_ts::Int64`: Start timestamp for the data -- `end_ts::Union{Int64,Nothing}`: End timestamp (can be null) -- `limit::Union{UInt64,Nothing}`: Record count limit (can be null) -- `stype_in::Union{SType.T,Nothing}`: Input symbol type (can be null) -- `stype_out::SType.T`: Output symbol type -- `ts_out::Bool`: Whether timestamps are included in output -- `symbols::Vector{String}`: List of symbols in the dataset -- `partial::Vector{String}`: Partially available symbols -- `not_found::Vector{String}`: Symbols that were not found -- `mappings::Vector{Tuple{String,String,Int64,Int64}}`: Symbol mappings as - `(raw_symbol, mapped_symbol, start_date, end_date)`, one tuple per mapping - interval. A symbol with multiple intervals (e.g. a continuous contract's roll - history) contributes consecutive tuples sharing the same raw symbol. Dates - are raw `YYYYMMDD` integers. -""" -struct Metadata - version::UInt8 - dataset::String - schema::Schema.T - start_ts::Int64 - end_ts::Union{Int64,Nothing} # Can be null - limit::Union{UInt64,Nothing} # Can be null - stype_in::Union{SType.T,Nothing} # Can be null - stype_out::SType.T - ts_out::Bool - symbols::Vector{String} - partial::Vector{String} - not_found::Vector{String} - mappings::Vector{Tuple{String,String,Int64,Int64}} -end - -""" - DBNHeader - -Complete header information for a DBN file. - -# Fields -- `version_upgrade_policy::VersionUpgradePolicy`: Version handling policy -- `dataset_condition::DatasetCondition`: Dataset conditions and constraints -- `metadata::Metadata`: Dataset metadata -""" -struct DBNHeader - version_upgrade_policy::VersionUpgradePolicy - dataset_condition::DatasetCondition - metadata::Metadata -end - -""" - RecordHeader - -Standard header present in all DBN record types. - -# Fields -- `length::UInt8`: Length of the record in 4-byte units (multiply by LENGTH_MULTIPLIER for bytes) -- `rtype::RType.T`: Record type identifier -- `publisher_id::UInt16`: Publisher/venue identifier -- `instrument_id::UInt32`: Instrument identifier -- `ts_event::Int64`: Event timestamp in nanoseconds since Unix epoch -""" -struct RecordHeader - length::UInt8 - rtype::RType.T - publisher_id::UInt16 - instrument_id::UInt32 - ts_event::Int64 -end - -""" - record_length_bytes(hd::RecordHeader) - -Get the actual record length in bytes from a RecordHeader. -The length field stores 4-byte units, so multiply by LENGTH_MULTIPLIER. -""" -record_length_bytes(hd::RecordHeader) = hd.length * LENGTH_MULTIPLIER - -""" - BidAskPair - -Bid and ask price/size information for market data. - -# Fields -- `bid_px::Int64`: Bid price (scaled by FIXED_PRICE_SCALE) -- `ask_px::Int64`: Ask price (scaled by FIXED_PRICE_SCALE) -- `bid_sz::UInt32`: Bid size/quantity -- `ask_sz::UInt32`: Ask size/quantity -- `bid_ct::UInt32`: Number of bid orders -- `ask_ct::UInt32`: Number of ask orders -""" -struct BidAskPair - bid_px::Int64 - ask_px::Int64 - bid_sz::UInt32 - ask_sz::UInt32 - bid_ct::UInt32 - ask_ct::UInt32 -end - -# Timestamp utilities - -""" - DBNTimestamp - -High-precision timestamp representation with nanosecond accuracy. - -# Fields -- `seconds::Int64`: Unix epoch seconds -- `nanoseconds::Int32`: Nanoseconds within the second (0-999_999_999) -""" -struct DBNTimestamp - seconds::Int64 # Unix epoch seconds - nanoseconds::Int32 # Nanoseconds within the second (0-999_999_999) -end - -""" - DBNTimestamp(ns::Int64) - -Construct a DBNTimestamp from nanoseconds since Unix epoch. - -# Arguments -- `ns::Int64`: Nanoseconds since Unix epoch - -# Returns -- `DBNTimestamp`: Timestamp split into seconds and nanoseconds components -""" -function DBNTimestamp(ns::Int64) - if ns == UNDEF_TIMESTAMP - return DBNTimestamp(UNDEF_TIMESTAMP, 0) - end - seconds = ns ÷ 1_000_000_000 - nanoseconds = Int32(ns % 1_000_000_000) - return DBNTimestamp(seconds, nanoseconds) -end - -""" - to_nanoseconds(ts::DBNTimestamp) - -Convert a DBNTimestamp back to nanoseconds since Unix epoch. - -# Arguments -- `ts::DBNTimestamp`: Timestamp to convert - -# Returns -- `Int64`: Nanoseconds since Unix epoch, or UNDEF_TIMESTAMP if undefined -""" -function to_nanoseconds(ts::DBNTimestamp) - if ts.seconds == UNDEF_TIMESTAMP - return UNDEF_TIMESTAMP - end - return ts.seconds * 1_000_000_000 + ts.nanoseconds -end - -""" - ts_to_datetime(ts::Int64) - -Convert a nanosecond timestamp to DateTime with nanosecond precision information. - -# Arguments -- `ts::Int64`: Nanoseconds since Unix epoch - -# Returns -- `NamedTuple`: Contains `datetime` (DateTime) and `nanoseconds` (Int32), or `nothing` if undefined -""" -function ts_to_datetime(ts::Int64) - if ts == UNDEF_TIMESTAMP - return nothing - end - # Returns DateTime with millisecond precision and separate nanosecond component - dbn_ts = DBNTimestamp(ts) - dt = unix2datetime(Float64(dbn_ts.seconds) + dbn_ts.nanoseconds / 1_000_000_000) - return (datetime=dt, nanoseconds=dbn_ts.nanoseconds) -end - -""" - datetime_to_ts(dt::DateTime, nanoseconds::Int32=0) - -Convert a DateTime with optional nanosecond precision to nanosecond timestamp. - -# Arguments -- `dt::DateTime`: DateTime to convert -- `nanoseconds::Int32`: Additional nanoseconds within the second (default: 0) - -# Returns -- `Int64`: Nanoseconds since Unix epoch -""" -function datetime_to_ts(dt::DateTime, nanoseconds::Union{Int32,Int64}=0) - # Convert DateTime to nanoseconds, preserving additional precision - seconds = Int64(round(datetime2unix(dt))) - return seconds * 1_000_000_000 + nanoseconds -end - -""" - ts_to_date_time(ts::Int64) - -Convert a nanosecond timestamp to separate Date and Time components with full nanosecond precision. - -# Arguments -- `ts::Int64`: Nanoseconds since Unix epoch - -# Returns -- `NamedTuple`: Contains `date` (Date), `time` (Time), and `timestamp` (DBNTimestamp), or `nothing` if undefined -""" -function ts_to_date_time(ts::Int64) - if ts == UNDEF_TIMESTAMP - return nothing - end - dbn_ts = DBNTimestamp(ts) - - # Get the date part - dt_seconds = unix2datetime(Float64(dbn_ts.seconds)) - date_part = Date(dt_seconds) - - # Get time within the day with nanosecond precision - seconds_in_day = dbn_ts.seconds % 86400 - time_ns = seconds_in_day * 1_000_000_000 + dbn_ts.nanoseconds - time_part = Dates.Time(Dates.Nanosecond(time_ns)) - - return (date=date_part, time=time_part, timestamp=dbn_ts) -end - -""" - date_time_to_ts(date::Date, time::Dates.Time) - -Convert separate Date and Time components to nanosecond timestamp. - -# Arguments -- `date::Date`: Date component -- `time::Dates.Time`: Time component with nanosecond precision - -# Returns -- `Int64`: Nanoseconds since Unix epoch -""" -function date_time_to_ts(date::Date, time::Dates.Time) - # Convert date to seconds since epoch - dt = DateTime(date) - date_seconds = Int64(round(datetime2unix(dt))) - - # Extract nanoseconds from time - time_ns = Dates.value(time) # Total nanoseconds since midnight - - return date_seconds * 1_000_000_000 + time_ns -end - -# Price conversion utilities - -""" - price_to_float(price::Int64, scale::Int32=FIXED_PRICE_SCALE) - -Convert a fixed-point price to floating-point representation. - -# Arguments -- `price::Int64`: Fixed-point price value -- `scale::Int32`: Scaling factor (default: FIXED_PRICE_SCALE) - -# Returns -- `Float64`: Floating-point price, or NaN if price is UNDEF_PRICE -""" -function price_to_float(price::Int64, scale::Int32=FIXED_PRICE_SCALE) - if price == UNDEF_PRICE - return NaN - end - return Float64(price) / Float64(scale) -end - -""" - float_to_price(value::Float64, scale::Int32=FIXED_PRICE_SCALE) - -Convert a floating-point price to fixed-point representation. - -# Arguments -- `value::Float64`: Floating-point price -- `scale::Int32`: Scaling factor (default: FIXED_PRICE_SCALE) - -# Returns -- `Int64`: Fixed-point price, or UNDEF_PRICE if value is NaN or infinite -""" -function float_to_price(value::Float64, scale::Int32=FIXED_PRICE_SCALE) - if isnan(value) || isinf(value) - return UNDEF_PRICE - end - return Int64(round(value * Float64(scale))) -end - -# Helper functions for safe enum conversion. -# All enums share one policy: an invalid byte warns (once per enum) and maps to -# a sentinel value instead of throwing, so one bad byte can't kill a decode -# stream. Lookup tables keep the hot path free of try/catch. - -function _build_enum_lookup(::Type{E}, default::E) where {E} - table = fill(default, 256) - valid = falses(256) - for inst in instances(E) - idx = Int(UInt8(inst)) + 1 - table[idx] = inst - valid[idx] = true - end - return table, valid -end - -const _ACTION_LOOKUP, _ACTION_VALID = _build_enum_lookup(Action.T, Action.NONE) -const _SIDE_LOOKUP, _SIDE_VALID = _build_enum_lookup(Side.T, Side.NONE) -const _INSTRUMENT_CLASS_LOOKUP, _INSTRUMENT_CLASS_VALID = - _build_enum_lookup(InstrumentClass.T, InstrumentClass.OTHER) - -""" - safe_action(raw_val::UInt8) - -Convert a raw byte value to an Action enum. `0x00` and invalid values map to -`Action.NONE`; invalid values additionally log a warning (once). -""" -@inline function safe_action(raw_val::UInt8) - # 0 indicates no action for certain record types - raw_val == 0x00 && return Action.NONE - idx = Int(raw_val) + 1 - @inbounds if !_ACTION_VALID[idx] - @warn "Unknown Action value: $raw_val (0x$(string(raw_val, base=16))), using NONE as default" maxlog = 1 - end - return @inbounds _ACTION_LOOKUP[idx] -end - -""" - safe_side(raw_val::UInt8) - -Convert a raw byte value to a Side enum. `0x00` and invalid values map to -`Side.NONE`; invalid values additionally log a warning (once). -""" -@inline function safe_side(raw_val::UInt8) - # 0 indicates no side for certain record types - raw_val == 0x00 && return Side.NONE - idx = Int(raw_val) + 1 - @inbounds if !_SIDE_VALID[idx] - @warn "Unknown Side value: $raw_val (0x$(string(raw_val, base=16))), using NONE as default" maxlog = 1 - end - return @inbounds _SIDE_LOOKUP[idx] -end - -""" - safe_instrument_class(raw_val::UInt8) - -Convert a raw byte value to an InstrumentClass enum. Invalid values log a -warning (once) and map to `InstrumentClass.OTHER`. -""" -@inline function safe_instrument_class(raw_val::UInt8) - idx = Int(raw_val) + 1 - @inbounds if !_INSTRUMENT_CLASS_VALID[idx] - @warn "Unknown InstrumentClass value: $raw_val (0x$(string(raw_val, base=16))), using OTHER as default" maxlog = 1 - end - return @inbounds _INSTRUMENT_CLASS_LOOKUP[idx] -end - -# --------------------------------------------------------------------------- -# Venue-code enums and record flags -# -# `StatMsg.stat_type` and `StatusMsg.trading_event` stay raw `UInt16` on the -# record structs: the wire fields are plain integers and publishers may emit -# codes outside the published enum. The enums below are for *interpreting* -# those codes. Compare with `r.stat_type == UInt16(StatType.SETTLEMENT_PRICE)` -# or convert with `safe_stat_type(r.stat_type)` / `safe_trading_event(...)`. -# --------------------------------------------------------------------------- - -""" - StatType - -Type of statistic carried by a [`StatMsg`](@ref) (`stat_type` field). Values -match the official DBN `StatType` enum (`UInt16`). `UNKNOWN = 0` is a local -sentinel returned by [`safe_stat_type`](@ref) for codes this package does not -recognize (publisher-specific, or newer than this release). - -# Values -- `OPENING_PRICE = 1`: Price of the first trade of an instrument -- `INDICATIVE_OPENING_PRICE = 2`: Probable opening price, published pre-open -- `SETTLEMENT_PRICE = 3`: Settlement price -- `TRADING_SESSION_LOW_PRICE = 4`: Lowest trade price of the session -- `TRADING_SESSION_HIGH_PRICE = 5`: Highest trade price of the session -- `CLEARED_VOLUME = 6`: Contracts cleared on the previous trading date -- `LOWEST_OFFER = 7`: Lowest offer price of the session -- `HIGHEST_BID = 8`: Highest bid price of the session -- `OPEN_INTEREST = 9`: Number of outstanding contracts -- `FIXING_PRICE = 10`: VWAP over a fixing period -- `CLOSE_PRICE = 11`: Last trade price of the session -- `NET_CHANGE = 12`: Change from the previous session's close -- `VWAP = 13`: Session volume-weighted average price -- `VOLATILITY = 14`: Implied volatility associated with the settlement price -- `DELTA = 15`: Option delta associated with the settlement price -- `UNCROSSING_PRICE = 16`: Auction uncrossing price -- `UPPER_PRICE_LIMIT = 17`: Exchange-defined upper price limit (published for - CME GLBX.MDP3 since the 2026-07 normalization change) -- `LOWER_PRICE_LIMIT = 18`: Exchange-defined lower price limit (as above) -- `BLOCK_VOLUME = 19`: Block contracts cleared on the previous trading date -- `INDICATIVE_CLOSE_PRICE = 20`: Probable closing price -- `MWCB_LEVEL_1 = 21`, `MWCB_LEVEL_2 = 22`, `MWCB_LEVEL_3 = 23`: Market-wide - circuit-breaker thresholds (7% / 13% / 20%) -- `AUCTION_COLLAR_REFERENCE_PRICE = 24`, `AUCTION_COLLAR_UPPER_PRICE = 25`, - `AUCTION_COLLAR_LOWER_PRICE = 26`: Auction collar prices -- `VENUE_SPECIFIC_VOLUME_1 = 10001`, `VENUE_SPECIFIC_PRICE_1 = 10002`: - Venue-specific statistics -- `UNKNOWN = 0`: Unrecognized code (local sentinel, not part of the DBN spec) -""" -@enumx StatType::UInt16 begin - UNKNOWN = 0 - OPENING_PRICE = 1 - INDICATIVE_OPENING_PRICE = 2 - SETTLEMENT_PRICE = 3 - TRADING_SESSION_LOW_PRICE = 4 - TRADING_SESSION_HIGH_PRICE = 5 - CLEARED_VOLUME = 6 - LOWEST_OFFER = 7 - HIGHEST_BID = 8 - OPEN_INTEREST = 9 - FIXING_PRICE = 10 - CLOSE_PRICE = 11 - NET_CHANGE = 12 - VWAP = 13 - VOLATILITY = 14 - DELTA = 15 - UNCROSSING_PRICE = 16 - UPPER_PRICE_LIMIT = 17 - LOWER_PRICE_LIMIT = 18 - BLOCK_VOLUME = 19 - INDICATIVE_CLOSE_PRICE = 20 - MWCB_LEVEL_1 = 21 - MWCB_LEVEL_2 = 22 - MWCB_LEVEL_3 = 23 - AUCTION_COLLAR_REFERENCE_PRICE = 24 - AUCTION_COLLAR_UPPER_PRICE = 25 - AUCTION_COLLAR_LOWER_PRICE = 26 - VENUE_SPECIFIC_VOLUME_1 = 10001 - VENUE_SPECIFIC_PRICE_1 = 10002 -end - -""" - TradingEvent - -Additional context for a [`StatusMsg`](@ref) (`trading_event` field). Values -match the official DBN `TradingEvent` enum (`UInt16`). - -# Values -- `NONE = 0`: No additional information given -- `NO_CANCEL = 1`: Order entry is allowed; modification and cancellation are not -- `CHANGE_TRADING_SESSION = 2`: A change of trading session occurred; daily - statistics are reset -- `IMPLIED_MATCHING_ON = 3`: Implied matching is available (CME's matching - engine is constructing implied depth) -- `IMPLIED_MATCHING_OFF = 4`: Implied matching is not available - -CME GLBX.MDP3 publishes `IMPLIED_MATCHING_ON`/`OFF` status records since the -2026-07 normalization change. -""" -@enumx TradingEvent::UInt16 begin - NONE = 0 - NO_CANCEL = 1 - CHANGE_TRADING_SESSION = 2 - IMPLIED_MATCHING_ON = 3 - IMPLIED_MATCHING_OFF = 4 -end - -const _STAT_TYPE_LOOKUP = Dict{UInt16,StatType.T}(UInt16(v) => v for v in instances(StatType.T)) -const _TRADING_EVENT_LOOKUP = Dict{UInt16,TradingEvent.T}(UInt16(v) => v for v in instances(TradingEvent.T)) - -""" - safe_stat_type(raw::Integer) -> StatType.T - -Interpret a raw `StatMsg.stat_type` code. Unrecognized codes map to -`StatType.UNKNOWN` without a warning (publisher-specific codes are legitimate). -""" -safe_stat_type(raw::Integer) = get(_STAT_TYPE_LOOKUP, UInt16(raw), StatType.UNKNOWN) - -""" - safe_trading_event(raw::Integer) -> TradingEvent.T - -Interpret a raw `StatusMsg.trading_event` code. Unrecognized codes map to -`TradingEvent.NONE` ("no additional information") without a warning. -""" -safe_trading_event(raw::Integer) = get(_TRADING_EVENT_LOOKUP, UInt16(raw), TradingEvent.NONE) - -# Record `flags` bit field (MBO / MBP / trade / BBO records). Values match the -# official DBN `flags` module. - -""" -Flag bit: last record in the event for a given `instrument_id`. Since the -2026-07 CME normalization change, GLBX.MDP3 MBO emits this on a standalone -record (`action = Action.NONE`, `price = UNDEF_PRICE`, `size = 0`) that -follows the book updates, rather than on the final book update itself. -""" -const F_LAST = 0x80 -"""Flag bit: top-of-book record, not an individual order.""" -const F_TOB = 0x40 -"""Flag bit: record sourced from a replay, such as a snapshot server.""" -const F_SNAPSHOT = 0x20 -"""Flag bit: aggregated price-level record, not an individual order.""" -const F_MBP = 0x10 -"""Flag bit: `ts_recv` is inaccurate due to clock issues or packet reordering.""" -const F_BAD_TS_RECV = 0x08 -"""Flag bit: an unrecoverable gap was detected in the channel.""" -const F_MAYBE_BAD_BOOK = 0x04 -"""Flag bit: publisher-specific event.""" -const F_PUBLISHER_SPECIFIC = 0x02 - -""" - has_flag(flags, flag) -> Bool - -`true` if the `flag` bit (e.g. [`F_LAST`](@ref)) is set in a record's `flags`. -""" -has_flag(flags::Integer, flag::Integer) = (flags & flag) != 0 +# DBN types, enums, and data structures + +# Constants +"""The current DBN format version supported by this implementation.""" +const DBN_VERSION = 3 + +"""Fixed-length symbol string size for DBN v2/v3 metadata and records.""" +const SYMBOL_CSTR_LEN = 71 + +"""Fixed-point price scaling factor for converting between integer and float prices.""" +const FIXED_PRICE_SCALE = Int32(1_000_000_000) + +"""Sentinel value indicating an undefined or missing price.""" +const UNDEF_PRICE = typemax(Int64) + +"""Sentinel value indicating an undefined or missing order size.""" +const UNDEF_ORDER_SIZE = typemax(UInt32) + +"""Sentinel value indicating an undefined or missing timestamp.""" +const UNDEF_TIMESTAMP = typemax(Int64) + +"""Multiplier for converting the length field to bytes in record headers.""" +const LENGTH_MULTIPLIER = 4 + +# Enums using EnumX for better namespace management + +""" + Schema + +DBN data schemas representing different types of market data. + +# Values +- `MBO`: Market-by-order data +- `MBP_1`: Market-by-price with book depth 1 (TBBO) +- `MBP_10`: Market-by-price with book depth 10 +- `TBBO`: Top-of-book bid/offer +- `TRADES`: Trade messages only +- `OHLCV_1S`: OHLCV data at 1-second intervals +- `OHLCV_1M`: OHLCV data at 1-minute intervals +- `OHLCV_1H`: OHLCV data at 1-hour intervals +- `OHLCV_1D`: OHLCV data at 1-day intervals +- `DEFINITION`: Instrument definition data +- `STATISTICS`: Market statistics +- `STATUS`: Exchange status messages +- `IMBALANCE`: Order imbalance data +- `CBBO`: Consolidated best bid/offer +- `CBBO_1S`: Consolidated BBO at 1-second intervals +- `CBBO_1M`: Consolidated BBO at 1-minute intervals +- `CMBP_1`: Consolidated market-by-price depth 1 +- `TCBBO`: Trade-consolidated BBO +- `BBO_1S`: BBO at 1-second intervals +- `BBO_1M`: BBO at 1-minute intervals +""" +@enumx Schema::UInt16 begin + MBO = 0 + MBP_1 = 1 + MBP_10 = 2 + TBBO = 3 + TRADES = 4 + OHLCV_1S = 5 + OHLCV_1M = 6 + OHLCV_1H = 7 + OHLCV_1D = 8 + DEFINITION = 9 + STATISTICS = 10 + STATUS = 11 + IMBALANCE = 12 + CBBO = 13 + CBBO_1S = 14 + CBBO_1M = 15 + CMBP_1 = 16 + TCBBO = 17 + BBO_1S = 18 + BBO_1M = 19 + MIX = 0xFFFF +end + +""" + Compression + +Compression algorithms supported for DBN files. + +# Values +- `NONE`: No compression +- `ZSTD`: Zstandard compression +""" +@enumx Compression::UInt8 begin + NONE = 0 + ZSTD = 1 +end + +""" + Encoding + +Output encoding formats for market data. + +# Values +- `DBN`: Databento Binary Encoding +- `CSV`: Comma-separated values +- `JSON`: JavaScript Object Notation +""" +@enumx Encoding::UInt8 begin + DBN = 0 + CSV = 1 + JSON = 2 +end + +""" + SType + +Symbol types for identifying instruments in DBN data. Numeric values match the +official Databento DBN spec — wire-encoded as a `UInt8` in metadata and in +`SymbolMappingMsg` (v2+). + +# Values +- `INSTRUMENT_ID = 0`: Numeric instrument identifier +- `RAW_SYMBOL = 1`: Raw symbol string from exchange +- `SMART = 2`: Deprecated alias (was split into `CONTINUOUS` and `PARENT`) +- `CONTINUOUS = 3`: Continuous contract symbol +- `PARENT = 4`: Parent symbol for derived instruments (e.g. `SPXW.OPT`) +- `NASDAQ_SYMBOL = 5`: Nasdaq-specific symbol +- `CMS_SYMBOL = 6`: CMS symbol +- `ISIN = 7`: ISO 6166 International Securities Identification Number +- `US_CODE = 8`: US Code (CUSIP-style) +- `BBG_COMP_ID = 9`: Bloomberg composite ID +- `BBG_COMP_TICKER = 10`: Bloomberg composite ticker +- `FIGI = 11`: OpenFIGI identifier +- `FIGI_TICKER = 12`: OpenFIGI ticker +- `UNDEF = 255`: Unset/undefined sentinel (`0xFF`). Databento wire-encodes an + unset `stype` as `0xFF`; this member lets non-nullable `stype` fields (e.g. + in `SymbolMappingMsg`) represent and round-trip that sentinel. Metadata's + nullable `stype_in` decodes `0xFF` to `nothing` instead. +""" +@enumx SType::UInt8 begin + INSTRUMENT_ID = 0 + RAW_SYMBOL = 1 + SMART = 2 # deprecated (kept for round-trip with v1 wire data) + CONTINUOUS = 3 + PARENT = 4 + NASDAQ_SYMBOL = 5 + CMS_SYMBOL = 6 + ISIN = 7 + US_CODE = 8 + BBG_COMP_ID = 9 + BBG_COMP_TICKER = 10 + FIGI = 11 + FIGI_TICKER = 12 + UNDEF = 255 # 0xFF "unset" sentinel; see docstring +end + +""" + RType + +Record types for different kinds of market data messages in DBN format. + +# Values +- `MBP_0_MSG`: Trades (book depth 0) +- `MBP_1_MSG`: TBBO/MBP-1 (book depth 1) +- `MBP_10_MSG`: MBP-10 (book depth 10) +- `STATUS_MSG`: Exchange status record +- `INSTRUMENT_DEF_MSG`: Instrument definition record +- `IMBALANCE_MSG`: Order imbalance record +- `ERROR_MSG`: Error record from live gateway +- `SYMBOL_MAPPING_MSG`: Symbol mapping record from live gateway +- `SYSTEM_MSG`: Non-error record from live gateway +- `STAT_MSG`: Statistics record from publisher +- `OHLCV_1S_MSG`: OHLCV at 1-second cadence +- `OHLCV_1M_MSG`: OHLCV at 1-minute cadence +- `OHLCV_1H_MSG`: OHLCV at hourly cadence +- `OHLCV_1D_MSG`: OHLCV at daily cadence +- `MBO_MSG`: Market-by-order record +- `CMBP_1_MSG`: Consolidated market-by-price with book depth 1 +- `CBBO_1S_MSG`: Consolidated market-by-price with book depth 1 at 1-second cadence +- `CBBO_1M_MSG`: Consolidated market-by-price with book depth 1 at 1-minute cadence +- `TCBBO_MSG`: Consolidated market-by-price with book depth 1 (trades only) +- `BBO_1S_MSG`: Market-by-price with book depth 1 at 1-second cadence +- `BBO_1M_MSG`: Market-by-price with book depth 1 at 1-minute cadence +""" +@enumx RType::UInt8 begin + MBP_0_MSG = 0x00 # Trades (book depth 0) + MBP_1_MSG = 0x01 # TBBO/MBP-1 (book depth 1) + MBP_10_MSG = 0x0A # MBP-10 (book depth 10) + STATUS_MSG = 0x12 # Exchange status record + INSTRUMENT_DEF_MSG = 0x13 # Instrument definition record + IMBALANCE_MSG = 0x14 # Order imbalance record + ERROR_MSG = 0x15 # Error record from live gateway + SYMBOL_MAPPING_MSG = 0x16 # Symbol mapping record from live gateway + SYSTEM_MSG = 0x17 # Non-error record from live gateway + STAT_MSG = 0x18 # Statistics record from publisher + OHLCV_1S_MSG = 0x20 # OHLCV at 1-second cadence + OHLCV_1M_MSG = 0x21 # OHLCV at 1-minute cadence + OHLCV_1H_MSG = 0x22 # OHLCV at hourly cadence + OHLCV_1D_MSG = 0x23 # OHLCV at daily cadence + MBO_MSG = 0xA0 # Market-by-order record + CMBP_1_MSG = 0xB1 # Consolidated market-by-price with book depth 1 + CBBO_1S_MSG = 0xC0 # Consolidated market-by-price with book depth 1 at 1-second cadence + CBBO_1M_MSG = 0xC1 # Consolidated market-by-price with book depth 1 at 1-minute cadence + TCBBO_MSG = 0xC2 # Consolidated market-by-price with book depth 1 (trades only) + BBO_1S_MSG = 0xC3 # Market-by-price with book depth 1 at 1-second cadence + BBO_1M_MSG = 0xC4 # Market-by-price with book depth 1 at 1-minute cadence +end + +""" + Action + +Market actions that can be applied to orders or trades. + +# Values +- `ADD`: Insert a new order into the book +- `MODIFY`: Change an order's price and/or size +- `CANCEL`: Fully or partially cancel an order from the book +- `CLEAR`: Remove all resting orders for the instrument +- `TRADE`: An aggressing order traded. Does not affect the book +- `FILL`: A resting order was filled. Does not affect the book +- `NONE`: No action: does not affect the book, but may carry flags or other information +""" +@enumx Action::UInt8 begin + ADD = UInt8('A') # Insert a new order into the book + MODIFY = UInt8('M') # Change an order's price and/or size + CANCEL = UInt8('C') # Fully or partially cancel an order from the book + CLEAR = UInt8('R') # Remove all resting orders for the instrument + TRADE = UInt8('T') # An aggressing order traded. Does not affect the book + FILL = UInt8('F') # A resting order was filled. Does not affect the book + NONE = UInt8('N') # No action: does not affect the book, but may carry flags or other information +end + +""" + Side + +Market sides for orders and trades. + +# Values +- `ASK`: Ask/offer side (sell orders) +- `BID`: Bid side (buy orders) +- `NONE`: No specific side or not applicable +""" +@enumx Side::UInt8 begin + ASK = UInt8('A') + BID = UInt8('B') + NONE = UInt8('N') +end + +""" + InstrumentClass + +Classification of financial instruments. Values match Databento's DBN +`InstrumentClass` enum. + +# Values +- `STOCK`: Equity instruments +- `CALL`: Call option contracts +- `PUT`: Put option contracts +- `FUTURE`: Futures contracts +- `INDEX`: Index instruments +- `BOND`: Fixed income securities +- `MIXED_SPREAD`: Mixed spread instruments +- `FUTURE_SPREAD`: Futures spread instruments +- `OPTION_SPREAD`: Option spread instruments +- `FX_SPOT`: Foreign exchange spot +- `COMMODITY_SPOT`: Commodity spot +- `UNKNOWN_0`, `UNKNOWN_45`: Numeric fallback values for unknown classes +""" +@enumx InstrumentClass::UInt8 begin + BOND = UInt8('B') + CALL = UInt8('C') + FUTURE = UInt8('F') + INDEX = UInt8('I') + STOCK = UInt8('K') + MIXED_SPREAD = UInt8('M') + PUT = UInt8('P') + FUTURE_SPREAD = UInt8('S') + OPTION_SPREAD = UInt8('T') + FX_SPOT = UInt8('X') + COMMODITY_SPOT = UInt8('Y') + # Also support numeric values + UNKNOWN_0 = 0 + UNKNOWN_45 = 45 + OTHER = UInt8('?') +end + +# Basic structures + +""" + VersionUpgradePolicy + +Encapsulates the version upgrade policy for DBN files. + +# Fields +- `upgrade_policy::UInt8`: Policy for handling version upgrades +""" +struct VersionUpgradePolicy + upgrade_policy::UInt8 +end + +""" + DatasetCondition + +Conditions and constraints for a dataset. + +# Fields +- `last_ts_out::Int64`: Last timestamp output +- `start_ts::Int64`: Dataset start timestamp +- `end_ts::Int64`: Dataset end timestamp +- `limit::UInt64`: Record limit for the dataset +""" +struct DatasetCondition + last_ts_out::Int64 + start_ts::Int64 + end_ts::Int64 + limit::UInt64 +end + +""" + Metadata + +Metadata information for a DBN dataset. + +# Fields +- `version::UInt8`: DBN format version +- `dataset::String`: Dataset identifier +- `schema::Schema.T`: Data schema type +- `start_ts::Int64`: Start timestamp for the data +- `end_ts::Union{Int64,Nothing}`: End timestamp (can be null) +- `limit::Union{UInt64,Nothing}`: Record count limit (can be null) +- `stype_in::Union{SType.T,Nothing}`: Input symbol type (can be null) +- `stype_out::SType.T`: Output symbol type +- `ts_out::Bool`: Whether timestamps are included in output +- `symbols::Vector{String}`: List of symbols in the dataset +- `partial::Vector{String}`: Partially available symbols +- `not_found::Vector{String}`: Symbols that were not found +- `mappings::Vector{Tuple{String,String,Int64,Int64}}`: Symbol mappings as + `(raw_symbol, mapped_symbol, start_date, end_date)`, one tuple per mapping + interval. A symbol with multiple intervals (e.g. a continuous contract's roll + history) contributes consecutive tuples sharing the same raw symbol. Dates + are raw `YYYYMMDD` integers. +""" +struct Metadata + version::UInt8 + dataset::String + schema::Schema.T + start_ts::Int64 + end_ts::Union{Int64,Nothing} # Can be null + limit::Union{UInt64,Nothing} # Can be null + stype_in::Union{SType.T,Nothing} # Can be null + stype_out::SType.T + ts_out::Bool + symbols::Vector{String} + partial::Vector{String} + not_found::Vector{String} + mappings::Vector{Tuple{String,String,Int64,Int64}} +end + +""" + DBNHeader + +Complete header information for a DBN file. + +# Fields +- `version_upgrade_policy::VersionUpgradePolicy`: Version handling policy +- `dataset_condition::DatasetCondition`: Dataset conditions and constraints +- `metadata::Metadata`: Dataset metadata +""" +struct DBNHeader + version_upgrade_policy::VersionUpgradePolicy + dataset_condition::DatasetCondition + metadata::Metadata +end + +""" + RecordHeader + +Standard header present in all DBN record types. + +# Fields +- `length::UInt8`: Length of the record in 4-byte units (multiply by LENGTH_MULTIPLIER for bytes) +- `rtype::RType.T`: Record type identifier +- `publisher_id::UInt16`: Publisher/venue identifier +- `instrument_id::UInt32`: Instrument identifier +- `ts_event::Int64`: Event timestamp in nanoseconds since Unix epoch +""" +struct RecordHeader + length::UInt8 + rtype::RType.T + publisher_id::UInt16 + instrument_id::UInt32 + ts_event::Int64 +end + +""" + record_length_bytes(hd::RecordHeader) + +Get the actual record length in bytes from a RecordHeader. +The length field stores 4-byte units, so multiply by LENGTH_MULTIPLIER. +""" +record_length_bytes(hd::RecordHeader) = hd.length * LENGTH_MULTIPLIER + +""" + BidAskPair + +Bid and ask price/size information for market data. + +# Fields +- `bid_px::Int64`: Bid price (scaled by FIXED_PRICE_SCALE) +- `ask_px::Int64`: Ask price (scaled by FIXED_PRICE_SCALE) +- `bid_sz::UInt32`: Bid size/quantity +- `ask_sz::UInt32`: Ask size/quantity +- `bid_ct::UInt32`: Number of bid orders +- `ask_ct::UInt32`: Number of ask orders +""" +struct BidAskPair + bid_px::Int64 + ask_px::Int64 + bid_sz::UInt32 + ask_sz::UInt32 + bid_ct::UInt32 + ask_ct::UInt32 +end + +# Timestamp utilities + +""" + DBNTimestamp + +High-precision timestamp representation with nanosecond accuracy. + +# Fields +- `seconds::Int64`: Unix epoch seconds +- `nanoseconds::Int32`: Nanoseconds within the second (0-999_999_999) +""" +struct DBNTimestamp + seconds::Int64 # Unix epoch seconds + nanoseconds::Int32 # Nanoseconds within the second (0-999_999_999) +end + +""" + DBNTimestamp(ns::Int64) + +Construct a DBNTimestamp from nanoseconds since Unix epoch. + +# Arguments +- `ns::Int64`: Nanoseconds since Unix epoch + +# Returns +- `DBNTimestamp`: Timestamp split into seconds and nanoseconds components +""" +function DBNTimestamp(ns::Int64) + if ns == UNDEF_TIMESTAMP + return DBNTimestamp(UNDEF_TIMESTAMP, 0) + end + seconds = ns ÷ 1_000_000_000 + nanoseconds = Int32(ns % 1_000_000_000) + return DBNTimestamp(seconds, nanoseconds) +end + +""" + to_nanoseconds(ts::DBNTimestamp) + +Convert a DBNTimestamp back to nanoseconds since Unix epoch. + +# Arguments +- `ts::DBNTimestamp`: Timestamp to convert + +# Returns +- `Int64`: Nanoseconds since Unix epoch, or UNDEF_TIMESTAMP if undefined +""" +function to_nanoseconds(ts::DBNTimestamp) + if ts.seconds == UNDEF_TIMESTAMP + return UNDEF_TIMESTAMP + end + return ts.seconds * 1_000_000_000 + ts.nanoseconds +end + +""" + ts_to_datetime(ts::Int64) + +Convert a nanosecond timestamp to DateTime with nanosecond precision information. + +# Arguments +- `ts::Int64`: Nanoseconds since Unix epoch + +# Returns +- `NamedTuple`: Contains `datetime` (DateTime) and `nanoseconds` (Int32), or `nothing` if undefined +""" +function ts_to_datetime(ts::Int64) + if ts == UNDEF_TIMESTAMP + return nothing + end + # Returns DateTime with millisecond precision and separate nanosecond component + dbn_ts = DBNTimestamp(ts) + dt = unix2datetime(Float64(dbn_ts.seconds) + dbn_ts.nanoseconds / 1_000_000_000) + return (datetime=dt, nanoseconds=dbn_ts.nanoseconds) +end + +""" + datetime_to_ts(dt::DateTime, nanoseconds::Int32=0) + +Convert a DateTime with optional nanosecond precision to nanosecond timestamp. + +# Arguments +- `dt::DateTime`: DateTime to convert +- `nanoseconds::Int32`: Additional nanoseconds within the second (default: 0) + +# Returns +- `Int64`: Nanoseconds since Unix epoch +""" +function datetime_to_ts(dt::DateTime, nanoseconds::Union{Int32,Int64}=0) + # Convert DateTime to nanoseconds, preserving additional precision + seconds = Int64(round(datetime2unix(dt))) + return seconds * 1_000_000_000 + nanoseconds +end + +""" + ts_to_date_time(ts::Int64) + +Convert a nanosecond timestamp to separate Date and Time components with full nanosecond precision. + +# Arguments +- `ts::Int64`: Nanoseconds since Unix epoch + +# Returns +- `NamedTuple`: Contains `date` (Date), `time` (Time), and `timestamp` (DBNTimestamp), or `nothing` if undefined +""" +function ts_to_date_time(ts::Int64) + if ts == UNDEF_TIMESTAMP + return nothing + end + dbn_ts = DBNTimestamp(ts) + + # Get the date part + dt_seconds = unix2datetime(Float64(dbn_ts.seconds)) + date_part = Date(dt_seconds) + + # Get time within the day with nanosecond precision + seconds_in_day = dbn_ts.seconds % 86400 + time_ns = seconds_in_day * 1_000_000_000 + dbn_ts.nanoseconds + time_part = Dates.Time(Dates.Nanosecond(time_ns)) + + return (date=date_part, time=time_part, timestamp=dbn_ts) +end + +""" + date_time_to_ts(date::Date, time::Dates.Time) + +Convert separate Date and Time components to nanosecond timestamp. + +# Arguments +- `date::Date`: Date component +- `time::Dates.Time`: Time component with nanosecond precision + +# Returns +- `Int64`: Nanoseconds since Unix epoch +""" +function date_time_to_ts(date::Date, time::Dates.Time) + # Convert date to seconds since epoch + dt = DateTime(date) + date_seconds = Int64(round(datetime2unix(dt))) + + # Extract nanoseconds from time + time_ns = Dates.value(time) # Total nanoseconds since midnight + + return date_seconds * 1_000_000_000 + time_ns +end + +# Price conversion utilities + +""" + price_to_float(price::Int64, scale::Int32=FIXED_PRICE_SCALE) + +Convert a fixed-point price to floating-point representation. + +# Arguments +- `price::Int64`: Fixed-point price value +- `scale::Int32`: Scaling factor (default: FIXED_PRICE_SCALE) + +# Returns +- `Float64`: Floating-point price, or NaN if price is UNDEF_PRICE +""" +function price_to_float(price::Int64, scale::Int32=FIXED_PRICE_SCALE) + if price == UNDEF_PRICE + return NaN + end + return Float64(price) / Float64(scale) +end + +""" + float_to_price(value::Float64, scale::Int32=FIXED_PRICE_SCALE) + +Convert a floating-point price to fixed-point representation. + +# Arguments +- `value::Float64`: Floating-point price +- `scale::Int32`: Scaling factor (default: FIXED_PRICE_SCALE) + +# Returns +- `Int64`: Fixed-point price, or UNDEF_PRICE if value is NaN or infinite +""" +function float_to_price(value::Float64, scale::Int32=FIXED_PRICE_SCALE) + if isnan(value) || isinf(value) + return UNDEF_PRICE + end + return Int64(round(value * Float64(scale))) +end + +# Helper functions for safe enum conversion. +# All enums share one policy: an invalid byte warns (once per enum) and maps to +# a sentinel value instead of throwing, so one bad byte can't kill a decode +# stream. Lookup tables keep the hot path free of try/catch. + +function _build_enum_lookup(::Type{E}, default::E) where {E} + table = fill(default, 256) + valid = falses(256) + for inst in instances(E) + idx = Int(UInt8(inst)) + 1 + table[idx] = inst + valid[idx] = true + end + return table, valid +end + +const _ACTION_LOOKUP, _ACTION_VALID = _build_enum_lookup(Action.T, Action.NONE) +const _SIDE_LOOKUP, _SIDE_VALID = _build_enum_lookup(Side.T, Side.NONE) +const _INSTRUMENT_CLASS_LOOKUP, _INSTRUMENT_CLASS_VALID = + _build_enum_lookup(InstrumentClass.T, InstrumentClass.OTHER) + +""" + safe_action(raw_val::UInt8) + +Convert a raw byte value to an Action enum. `0x00` and invalid values map to +`Action.NONE`; invalid values additionally log a warning (once). +""" +@inline function safe_action(raw_val::UInt8) + # 0 indicates no action for certain record types + raw_val == 0x00 && return Action.NONE + idx = Int(raw_val) + 1 + @inbounds if !_ACTION_VALID[idx] + @warn "Unknown Action value: $raw_val (0x$(string(raw_val, base=16))), using NONE as default" maxlog = 1 + end + return @inbounds _ACTION_LOOKUP[idx] +end + +""" + safe_side(raw_val::UInt8) + +Convert a raw byte value to a Side enum. `0x00` and invalid values map to +`Side.NONE`; invalid values additionally log a warning (once). +""" +@inline function safe_side(raw_val::UInt8) + # 0 indicates no side for certain record types + raw_val == 0x00 && return Side.NONE + idx = Int(raw_val) + 1 + @inbounds if !_SIDE_VALID[idx] + @warn "Unknown Side value: $raw_val (0x$(string(raw_val, base=16))), using NONE as default" maxlog = 1 + end + return @inbounds _SIDE_LOOKUP[idx] +end + +""" + safe_instrument_class(raw_val::UInt8) + +Convert a raw byte value to an InstrumentClass enum. Invalid values log a +warning (once) and map to `InstrumentClass.OTHER`. +""" +@inline function safe_instrument_class(raw_val::UInt8) + idx = Int(raw_val) + 1 + @inbounds if !_INSTRUMENT_CLASS_VALID[idx] + @warn "Unknown InstrumentClass value: $raw_val (0x$(string(raw_val, base=16))), using OTHER as default" maxlog = 1 + end + return @inbounds _INSTRUMENT_CLASS_LOOKUP[idx] +end + +# --------------------------------------------------------------------------- +# Venue-code enums and record flags +# +# `StatMsg.stat_type` and `StatusMsg.trading_event` stay raw `UInt16` on the +# record structs: the wire fields are plain integers and publishers may emit +# codes outside the published enum. The enums below are for *interpreting* +# those codes. Compare with `r.stat_type == UInt16(StatType.SETTLEMENT_PRICE)` +# or convert with `safe_stat_type(r.stat_type)` / `safe_trading_event(...)`. +# --------------------------------------------------------------------------- + +""" + StatType + +Type of statistic carried by a [`StatMsg`](@ref) (`stat_type` field). Values +match the official DBN `StatType` enum (`UInt16`). `UNKNOWN = 0` is a local +sentinel returned by [`safe_stat_type`](@ref) for codes this package does not +recognize (publisher-specific, or newer than this release). + +# Values +- `OPENING_PRICE = 1`: Price of the first trade of an instrument +- `INDICATIVE_OPENING_PRICE = 2`: Probable opening price, published pre-open +- `SETTLEMENT_PRICE = 3`: Settlement price +- `TRADING_SESSION_LOW_PRICE = 4`: Lowest trade price of the session +- `TRADING_SESSION_HIGH_PRICE = 5`: Highest trade price of the session +- `CLEARED_VOLUME = 6`: Contracts cleared on the previous trading date +- `LOWEST_OFFER = 7`: Lowest offer price of the session +- `HIGHEST_BID = 8`: Highest bid price of the session +- `OPEN_INTEREST = 9`: Number of outstanding contracts +- `FIXING_PRICE = 10`: VWAP over a fixing period +- `CLOSE_PRICE = 11`: Last trade price of the session +- `NET_CHANGE = 12`: Change from the previous session's close +- `VWAP = 13`: Session volume-weighted average price +- `VOLATILITY = 14`: Implied volatility associated with the settlement price +- `DELTA = 15`: Option delta associated with the settlement price +- `UNCROSSING_PRICE = 16`: Auction uncrossing price +- `UPPER_PRICE_LIMIT = 17`: Exchange-defined upper price limit (published for + CME GLBX.MDP3 since the 2026-07 normalization change) +- `LOWER_PRICE_LIMIT = 18`: Exchange-defined lower price limit (as above) +- `BLOCK_VOLUME = 19`: Block contracts cleared on the previous trading date +- `INDICATIVE_CLOSE_PRICE = 20`: Probable closing price +- `MWCB_LEVEL_1 = 21`, `MWCB_LEVEL_2 = 22`, `MWCB_LEVEL_3 = 23`: Market-wide + circuit-breaker thresholds (7% / 13% / 20%) +- `AUCTION_COLLAR_REFERENCE_PRICE = 24`, `AUCTION_COLLAR_UPPER_PRICE = 25`, + `AUCTION_COLLAR_LOWER_PRICE = 26`: Auction collar prices +- `VENUE_SPECIFIC_VOLUME_1 = 10001`, `VENUE_SPECIFIC_PRICE_1 = 10002`: + Venue-specific statistics +- `UNKNOWN = 0`: Unrecognized code (local sentinel, not part of the DBN spec) +""" +@enumx StatType::UInt16 begin + UNKNOWN = 0 + OPENING_PRICE = 1 + INDICATIVE_OPENING_PRICE = 2 + SETTLEMENT_PRICE = 3 + TRADING_SESSION_LOW_PRICE = 4 + TRADING_SESSION_HIGH_PRICE = 5 + CLEARED_VOLUME = 6 + LOWEST_OFFER = 7 + HIGHEST_BID = 8 + OPEN_INTEREST = 9 + FIXING_PRICE = 10 + CLOSE_PRICE = 11 + NET_CHANGE = 12 + VWAP = 13 + VOLATILITY = 14 + DELTA = 15 + UNCROSSING_PRICE = 16 + UPPER_PRICE_LIMIT = 17 + LOWER_PRICE_LIMIT = 18 + BLOCK_VOLUME = 19 + INDICATIVE_CLOSE_PRICE = 20 + MWCB_LEVEL_1 = 21 + MWCB_LEVEL_2 = 22 + MWCB_LEVEL_3 = 23 + AUCTION_COLLAR_REFERENCE_PRICE = 24 + AUCTION_COLLAR_UPPER_PRICE = 25 + AUCTION_COLLAR_LOWER_PRICE = 26 + VENUE_SPECIFIC_VOLUME_1 = 10001 + VENUE_SPECIFIC_PRICE_1 = 10002 +end + +""" + TradingEvent + +Additional context for a [`StatusMsg`](@ref) (`trading_event` field). Values +match the official DBN `TradingEvent` enum (`UInt16`). + +# Values +- `NONE = 0`: No additional information given +- `NO_CANCEL = 1`: Order entry is allowed; modification and cancellation are not +- `CHANGE_TRADING_SESSION = 2`: A change of trading session occurred; daily + statistics are reset +- `IMPLIED_MATCHING_ON = 3`: Implied matching is available (CME's matching + engine is constructing implied depth) +- `IMPLIED_MATCHING_OFF = 4`: Implied matching is not available + +CME GLBX.MDP3 publishes `IMPLIED_MATCHING_ON`/`OFF` status records since the +2026-07 normalization change. +""" +@enumx TradingEvent::UInt16 begin + NONE = 0 + NO_CANCEL = 1 + CHANGE_TRADING_SESSION = 2 + IMPLIED_MATCHING_ON = 3 + IMPLIED_MATCHING_OFF = 4 +end + +const _STAT_TYPE_LOOKUP = Dict{UInt16,StatType.T}(UInt16(v) => v for v in instances(StatType.T)) +const _TRADING_EVENT_LOOKUP = Dict{UInt16,TradingEvent.T}(UInt16(v) => v for v in instances(TradingEvent.T)) + +""" + safe_stat_type(raw::Integer) -> StatType.T + +Interpret a raw `StatMsg.stat_type` code. Unrecognized codes map to +`StatType.UNKNOWN` without a warning (publisher-specific codes are legitimate). +""" +safe_stat_type(raw::Integer) = get(_STAT_TYPE_LOOKUP, UInt16(raw), StatType.UNKNOWN) + +""" + safe_trading_event(raw::Integer) -> TradingEvent.T + +Interpret a raw `StatusMsg.trading_event` code. Unrecognized codes map to +`TradingEvent.NONE` ("no additional information") without a warning. +""" +safe_trading_event(raw::Integer) = get(_TRADING_EVENT_LOOKUP, UInt16(raw), TradingEvent.NONE) + +# Record `flags` bit field (MBO / MBP / trade / BBO records). Values match the +# official DBN `flags` module. + +""" +Flag bit: last record in the event for a given `instrument_id`. Since the +2026-07 CME normalization change, GLBX.MDP3 MBO emits this on a standalone +record (`action = Action.NONE`, `price = UNDEF_PRICE`, `size = 0`) that +follows the book updates, rather than on the final book update itself. +""" +const F_LAST = 0x80 +"""Flag bit: top-of-book record, not an individual order.""" +const F_TOB = 0x40 +"""Flag bit: record sourced from a replay, such as a snapshot server.""" +const F_SNAPSHOT = 0x20 +"""Flag bit: aggregated price-level record, not an individual order.""" +const F_MBP = 0x10 +"""Flag bit: `ts_recv` is inaccurate due to clock issues or packet reordering.""" +const F_BAD_TS_RECV = 0x08 +"""Flag bit: an unrecoverable gap was detected in the channel.""" +const F_MAYBE_BAD_BOOK = 0x04 +"""Flag bit: publisher-specific event.""" +const F_PUBLISHER_SPECIFIC = 0x02 + +""" + has_flag(flags, flag) -> Bool + +`true` if the `flag` bit (e.g. [`F_LAST`](@ref)) is set in a record's `flags`. +""" +has_flag(flags::Integer, flag::Integer) = (flags & flag) != 0 diff --git a/test/runtests.jl b/test/runtests.jl index 195c66af..9105a071 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,47 +1,47 @@ -using Test -using DatabentoBinaryEncoding -import DatabentoBinaryEncoding as DBN -using Dates - -# Load test utilities (safe_rm, etc.) -include("test_utils.jl") - -@testset "DBN.jl Tests" begin - include("test_phase1.jl") - include("test_phase2.jl") - include("test_phase3.jl") - include("test_phase4.jl") - include("test_phase5.jl") - include("test_phase6.jl") - include("test_phase7.jl") - include("test_phase8.jl") - include("test_phase9_working.jl") # Edge cases and error handling - include("test_phase10_complete.jl") # Integration and performance testing - include("test_convenience_functions.jl") # Test all convenience read_*/foreach_* functions - include("test_phase11_typed_with_control.jl") # foreach_record_with_control: typed data + Union control split - include("test_replay.jl") # replay_dbn / replay_records: timestamp-paced re-emission - include("test_issue23_unset_stype.jl") # regression: 0xFF unset stype in v3 SymbolMappingMsg (issue #23) - include("test_issues_32_35.jl") # regressions: pre-v3 StatMsg layout, stat_to_dataframe, mapping intervals, invalid enum bytes (issues #32-#35) - include("test_show.jl") # compact one-line Base.show for record types - include("test_symbols.jl") # symbol_map / symbol_for / add_symbol_column! / records_to_dataframe(records, metadata) - include("test_issue40_mbp_dataframe.jl") # regression: MBP-1/MBP-10/BBO records_to_dataframe via nested levels (issue #40) - include("test_cme_2026_07.jl") # CME 2026-07 normalization: per-leg defs, standalone F_LAST, StatType/TradingEvent enums, flags - include("test_mbo_wire_layout.jl") # regression: MBOMsg wire layout (order_id/price/ts_recv offsets) vs Databento's own fixtures - include("test_stat_quantity_sentinel.jl") # regression: v3 StatMsg UNDEF quantity written as typemax(Int64), not -1 - - # Run compatibility tests if the Rust CLI is available - dbn_cli_path = if Sys.iswindows() - joinpath(homedir(), "dbn-workspace", "dbn", "target", "release", "dbn.exe") - else - joinpath(homedir(), "dbn-workspace", "dbn", "target", "release", "dbn") - end - - if isfile(dbn_cli_path) - include("test_compatibility_updated.jl") # Updated cross-implementation compatibility testing - else - @warn "Skipping compatibility tests - Rust dbn-cli not found at $dbn_cli_path" - end - - # Import/export tests (optional - uncomment if needed) - # include("test_import_simple.jl") +using Test +using DatabentoBinaryEncoding +import DatabentoBinaryEncoding as DBN +using Dates + +# Load test utilities (safe_rm, etc.) +include("test_utils.jl") + +@testset "DBN.jl Tests" begin + include("test_phase1.jl") + include("test_phase2.jl") + include("test_phase3.jl") + include("test_phase4.jl") + include("test_phase5.jl") + include("test_phase6.jl") + include("test_phase7.jl") + include("test_phase8.jl") + include("test_phase9_working.jl") # Edge cases and error handling + include("test_phase10_complete.jl") # Integration and performance testing + include("test_convenience_functions.jl") # Test all convenience read_*/foreach_* functions + include("test_phase11_typed_with_control.jl") # foreach_record_with_control: typed data + Union control split + include("test_replay.jl") # replay_dbn / replay_records: timestamp-paced re-emission + include("test_issue23_unset_stype.jl") # regression: 0xFF unset stype in v3 SymbolMappingMsg (issue #23) + include("test_issues_32_35.jl") # regressions: pre-v3 StatMsg layout, stat_to_dataframe, mapping intervals, invalid enum bytes (issues #32-#35) + include("test_show.jl") # compact one-line Base.show for record types + include("test_symbols.jl") # symbol_map / symbol_for / add_symbol_column! / records_to_dataframe(records, metadata) + include("test_issue40_mbp_dataframe.jl") # regression: MBP-1/MBP-10/BBO records_to_dataframe via nested levels (issue #40) + include("test_cme_2026_07.jl") # CME 2026-07 normalization: per-leg defs, standalone F_LAST, StatType/TradingEvent enums, flags + include("test_mbo_wire_layout.jl") # regression: MBOMsg wire layout (order_id/price/ts_recv offsets) vs Databento's own fixtures + include("test_stat_quantity_sentinel.jl") # regression: v3 StatMsg UNDEF quantity written as typemax(Int64), not -1 + + # Run compatibility tests if the Rust CLI is available + dbn_cli_path = if Sys.iswindows() + joinpath(homedir(), "dbn-workspace", "dbn", "target", "release", "dbn.exe") + else + joinpath(homedir(), "dbn-workspace", "dbn", "target", "release", "dbn") + end + + if isfile(dbn_cli_path) + include("test_compatibility_updated.jl") # Updated cross-implementation compatibility testing + else + @warn "Skipping compatibility tests - Rust dbn-cli not found at $dbn_cli_path" + end + + # Import/export tests (optional - uncomment if needed) + # include("test_import_simple.jl") end \ No newline at end of file diff --git a/test/test_import_simple.jl b/test/test_import_simple.jl index e3c78ea3..80cc099e 100644 --- a/test/test_import_simple.jl +++ b/test/test_import_simple.jl @@ -1,98 +1,98 @@ -using Test +using Test using DatabentoBinaryEncoding import DatabentoBinaryEncoding as DBN -using Dates - -@testset "Basic Import Functionality" begin - - @testset "JSON Import Basic Test" begin - # Create a simple JSON structure that we know should work - json_content = """{ - "metadata": { - "version": 3, - "dataset": "TEST", - "schema": "TRADES", - "start_ts": "1704099000000000000", - "end_ts": "1704099030000000000", - "limit": 1, - "stype_in": "RAW_SYMBOL", - "stype_out": "RAW_SYMBOL", - "ts_out": false, - "symbols": [], - "partial": [], - "not_found": [], - "mappings": [] - }, - "records": [ - { - "hd": { - "ts_event": "1704099000000000000", - "rtype": 0, - "publisher_id": 1, - "instrument_id": 12345 - }, - "price": "100500000000", - "size": 100, - "action": "T", - "side": "B", - "flags": 0, - "depth": 0, - "ts_recv": "1704099000000000000", - "ts_in_delta": 0, - "sequence": 1 - } - ] - }""" - - json_file = "test_simple.json" - dbn_file = "test_simple.dbn" - - try - # Write test JSON - open(json_file, "w") do f - write(f, json_content) - end - - # Test conversion - this should work with structured JSON - record_count = json_to_dbn(json_file, dbn_file) - @test record_count == 1 - - # Verify output file exists and has content - @test isfile(dbn_file) - @test filesize(dbn_file) > 0 - - finally - safe_rm(json_file) - safe_rm(dbn_file) - end - end - - @testset "Error Handling" begin - # Test with malformed JSON - json_file = "malformed.json" - dbn_file = "output.dbn" - - try - open(json_file, "w") do f - write(f, "{invalid json") - end - - @test_throws Exception json_to_dbn(json_file, dbn_file) - - finally - safe_rm(json_file) - safe_rm(dbn_file) - end - - # Test missing file - @test_throws SystemError json_to_dbn("nonexistent.json", "output.dbn") - end - - @testset "Parameter Validation" begin - # Test CSV/Parquet missing parameters - @test_throws ArgumentError csv_to_dbn("test.csv", "test.dbn", dataset="TEST") - @test_throws ArgumentError csv_to_dbn("test.csv", "test.dbn", schema=Schema.TRADES) - @test_throws ArgumentError parquet_to_dbn("test.parquet", "test.dbn", dataset="TEST") - @test_throws ArgumentError parquet_to_dbn("test.parquet", "test.dbn", schema=Schema.TRADES) - end +using Dates + +@testset "Basic Import Functionality" begin + + @testset "JSON Import Basic Test" begin + # Create a simple JSON structure that we know should work + json_content = """{ + "metadata": { + "version": 3, + "dataset": "TEST", + "schema": "TRADES", + "start_ts": "1704099000000000000", + "end_ts": "1704099030000000000", + "limit": 1, + "stype_in": "RAW_SYMBOL", + "stype_out": "RAW_SYMBOL", + "ts_out": false, + "symbols": [], + "partial": [], + "not_found": [], + "mappings": [] + }, + "records": [ + { + "hd": { + "ts_event": "1704099000000000000", + "rtype": 0, + "publisher_id": 1, + "instrument_id": 12345 + }, + "price": "100500000000", + "size": 100, + "action": "T", + "side": "B", + "flags": 0, + "depth": 0, + "ts_recv": "1704099000000000000", + "ts_in_delta": 0, + "sequence": 1 + } + ] + }""" + + json_file = "test_simple.json" + dbn_file = "test_simple.dbn" + + try + # Write test JSON + open(json_file, "w") do f + write(f, json_content) + end + + # Test conversion - this should work with structured JSON + record_count = json_to_dbn(json_file, dbn_file) + @test record_count == 1 + + # Verify output file exists and has content + @test isfile(dbn_file) + @test filesize(dbn_file) > 0 + + finally + safe_rm(json_file) + safe_rm(dbn_file) + end + end + + @testset "Error Handling" begin + # Test with malformed JSON + json_file = "malformed.json" + dbn_file = "output.dbn" + + try + open(json_file, "w") do f + write(f, "{invalid json") + end + + @test_throws Exception json_to_dbn(json_file, dbn_file) + + finally + safe_rm(json_file) + safe_rm(dbn_file) + end + + # Test missing file + @test_throws SystemError json_to_dbn("nonexistent.json", "output.dbn") + end + + @testset "Parameter Validation" begin + # Test CSV/Parquet missing parameters + @test_throws ArgumentError csv_to_dbn("test.csv", "test.dbn", dataset="TEST") + @test_throws ArgumentError csv_to_dbn("test.csv", "test.dbn", schema=Schema.TRADES) + @test_throws ArgumentError parquet_to_dbn("test.parquet", "test.dbn", dataset="TEST") + @test_throws ArgumentError parquet_to_dbn("test.parquet", "test.dbn", schema=Schema.TRADES) + end end \ No newline at end of file diff --git a/test/test_phase1.jl b/test/test_phase1.jl index 4c928878..4168e0fc 100644 --- a/test/test_phase1.jl +++ b/test/test_phase1.jl @@ -1,116 +1,116 @@ -# Phase 1: Basic Module Setup and Loading Tests - -@testset "Phase 1: Basic Module Setup and Loading" begin - - @testset "Module loads without syntax errors" begin - # The fact that we got here means the module loaded - @test isdefined(Main, :DBN) - @test DBN isa Module - end - - @testset "All exports are properly defined" begin - # Check each exported symbol - exports = [ - # Core types - :DBNDecoder, :DBNEncoder, :read_dbn, :write_dbn, - :Metadata, :DBNHeader, :RecordHeader, :DBNTimestamp, - # Message types - :MBOMsg, :TradeMsg, :MBP1Msg, :MBP10Msg, :OHLCVMsg, - :StatusMsg, :ImbalanceMsg, :StatMsg, - :ErrorMsg, :SymbolMappingMsg, :SystemMsg, :InstrumentDefMsg, - # Streaming support - :DBNStream, :DBNStreamWriter, :write_record!, :close_writer!, - # Compression utilities - :compress_dbn_file, :compress_daily_files, - # Enums - :Schema, :Compression, :Encoding, :SType, :RType, :Action, :Side, :InstrumentClass, - # Utility functions - :price_to_float, :float_to_price, :ts_to_datetime, :datetime_to_ts, :ts_to_date_time, :date_time_to_ts, :to_nanoseconds, - # Constants - :DBN_VERSION, :FIXED_PRICE_SCALE, :UNDEF_PRICE, :UNDEF_ORDER_SIZE, :UNDEF_TIMESTAMP, - # Helper structs - :BidAskPair, :VersionUpgradePolicy, :DatasetCondition, - # Low-level functions - :write_header, :read_header!, :write_record, :read_record, :finalize_encoder - ] - - for sym in exports - @test isdefined(DBN, sym) - end - end - - @testset "All enums can be instantiated" begin - # Test Schema enum - @test Schema.MBO == Schema.T(0) - @test Schema.MBP_1 == Schema.T(1) - @test Schema.MBP_10 == Schema.T(2) - @test Schema.TRADES == Schema.T(4) - @test Schema.OHLCV_1S == Schema.T(5) - @test Schema.DEFINITION == Schema.T(9) - @test Schema.STATISTICS == Schema.T(10) - @test Schema.STATUS == Schema.T(11) - @test Schema.IMBALANCE == Schema.T(12) - - # Test Compression enum - @test Compression.NONE == Compression.T(0) - @test Compression.ZSTD == Compression.T(1) - - # Test Encoding enum - @test Encoding.DBN == Encoding.T(0) - @test Encoding.CSV == Encoding.T(1) - @test Encoding.JSON == Encoding.T(2) - - # Test SType enum (numeric values match the official DBN spec) - @test SType.INSTRUMENT_ID == SType.T(0) - @test SType.RAW_SYMBOL == SType.T(1) - @test SType.SMART == SType.T(2) - @test SType.CONTINUOUS == SType.T(3) - @test SType.PARENT == SType.T(4) - @test SType.NASDAQ_SYMBOL == SType.T(5) - @test SType.CMS_SYMBOL == SType.T(6) - @test SType.ISIN == SType.T(7) - @test SType.US_CODE == SType.T(8) - @test SType.BBG_COMP_ID == SType.T(9) - @test SType.BBG_COMP_TICKER == SType.T(10) - @test SType.FIGI == SType.T(11) - @test SType.FIGI_TICKER == SType.T(12) - - # Test RType enum (DBN v3 values) - @test RType.MBP_0_MSG == RType.T(0x00) # Trades (book depth 0) - @test RType.MBP_1_MSG == RType.T(0x01) # TBBO/MBP-1 (book depth 1) - @test RType.MBP_10_MSG == RType.T(0x0A) # MBP-10 (book depth 10) - @test RType.STATUS_MSG == RType.T(0x12) # Exchange status record - @test RType.INSTRUMENT_DEF_MSG == RType.T(0x13) # Instrument definition record - @test RType.IMBALANCE_MSG == RType.T(0x14) # Order imbalance record - @test RType.ERROR_MSG == RType.T(0x15) # Error record from live gateway - @test RType.SYMBOL_MAPPING_MSG == RType.T(0x16) # Symbol mapping record from live gateway - @test RType.SYSTEM_MSG == RType.T(0x17) # Non-error record from live gateway - @test RType.STAT_MSG == RType.T(0x18) # Statistics record from publisher - @test RType.OHLCV_1S_MSG == RType.T(0x20) # OHLCV at 1-second cadence - @test RType.OHLCV_1M_MSG == RType.T(0x21) # OHLCV at 1-minute cadence - @test RType.OHLCV_1H_MSG == RType.T(0x22) # OHLCV at hourly cadence - @test RType.OHLCV_1D_MSG == RType.T(0x23) # OHLCV at daily cadence - @test RType.MBO_MSG == RType.T(0xA0) # Market-by-order record - @test RType.CMBP_1_MSG == RType.T(0xB1) # Consolidated market-by-price with book depth 1 - @test RType.CBBO_1S_MSG == RType.T(0xC0) # Consolidated market-by-price with book depth 1 at 1-second cadence - @test RType.CBBO_1M_MSG == RType.T(0xC1) # Consolidated market-by-price with book depth 1 at 1-minute cadence - @test RType.TCBBO_MSG == RType.T(0xC2) # Consolidated market-by-price with book depth 1 (trades only) - @test RType.BBO_1S_MSG == RType.T(0xC3) # Market-by-price with book depth 1 at 1-second cadence - @test RType.BBO_1M_MSG == RType.T(0xC4) # Market-by-price with book depth 1 at 1-minute cadence - - # Test Action enum - @test Action.ADD == Action.T(UInt8('A')) - @test Action.CANCEL == Action.T(UInt8('C')) - @test Action.MODIFY == Action.T(UInt8('M')) - @test Action.TRADE == Action.T(UInt8('T')) - @test Action.FILL == Action.T(UInt8('F')) - @test Action.CLEAR == Action.T(UInt8('R')) - - # Test Side enum - @test Side.ASK == Side.T(UInt8('A')) - @test Side.BID == Side.T(UInt8('B')) - @test Side.NONE == Side.T(UInt8('N')) - +# Phase 1: Basic Module Setup and Loading Tests + +@testset "Phase 1: Basic Module Setup and Loading" begin + + @testset "Module loads without syntax errors" begin + # The fact that we got here means the module loaded + @test isdefined(Main, :DBN) + @test DBN isa Module + end + + @testset "All exports are properly defined" begin + # Check each exported symbol + exports = [ + # Core types + :DBNDecoder, :DBNEncoder, :read_dbn, :write_dbn, + :Metadata, :DBNHeader, :RecordHeader, :DBNTimestamp, + # Message types + :MBOMsg, :TradeMsg, :MBP1Msg, :MBP10Msg, :OHLCVMsg, + :StatusMsg, :ImbalanceMsg, :StatMsg, + :ErrorMsg, :SymbolMappingMsg, :SystemMsg, :InstrumentDefMsg, + # Streaming support + :DBNStream, :DBNStreamWriter, :write_record!, :close_writer!, + # Compression utilities + :compress_dbn_file, :compress_daily_files, + # Enums + :Schema, :Compression, :Encoding, :SType, :RType, :Action, :Side, :InstrumentClass, + # Utility functions + :price_to_float, :float_to_price, :ts_to_datetime, :datetime_to_ts, :ts_to_date_time, :date_time_to_ts, :to_nanoseconds, + # Constants + :DBN_VERSION, :FIXED_PRICE_SCALE, :UNDEF_PRICE, :UNDEF_ORDER_SIZE, :UNDEF_TIMESTAMP, + # Helper structs + :BidAskPair, :VersionUpgradePolicy, :DatasetCondition, + # Low-level functions + :write_header, :read_header!, :write_record, :read_record, :finalize_encoder + ] + + for sym in exports + @test isdefined(DBN, sym) + end + end + + @testset "All enums can be instantiated" begin + # Test Schema enum + @test Schema.MBO == Schema.T(0) + @test Schema.MBP_1 == Schema.T(1) + @test Schema.MBP_10 == Schema.T(2) + @test Schema.TRADES == Schema.T(4) + @test Schema.OHLCV_1S == Schema.T(5) + @test Schema.DEFINITION == Schema.T(9) + @test Schema.STATISTICS == Schema.T(10) + @test Schema.STATUS == Schema.T(11) + @test Schema.IMBALANCE == Schema.T(12) + + # Test Compression enum + @test Compression.NONE == Compression.T(0) + @test Compression.ZSTD == Compression.T(1) + + # Test Encoding enum + @test Encoding.DBN == Encoding.T(0) + @test Encoding.CSV == Encoding.T(1) + @test Encoding.JSON == Encoding.T(2) + + # Test SType enum (numeric values match the official DBN spec) + @test SType.INSTRUMENT_ID == SType.T(0) + @test SType.RAW_SYMBOL == SType.T(1) + @test SType.SMART == SType.T(2) + @test SType.CONTINUOUS == SType.T(3) + @test SType.PARENT == SType.T(4) + @test SType.NASDAQ_SYMBOL == SType.T(5) + @test SType.CMS_SYMBOL == SType.T(6) + @test SType.ISIN == SType.T(7) + @test SType.US_CODE == SType.T(8) + @test SType.BBG_COMP_ID == SType.T(9) + @test SType.BBG_COMP_TICKER == SType.T(10) + @test SType.FIGI == SType.T(11) + @test SType.FIGI_TICKER == SType.T(12) + + # Test RType enum (DBN v3 values) + @test RType.MBP_0_MSG == RType.T(0x00) # Trades (book depth 0) + @test RType.MBP_1_MSG == RType.T(0x01) # TBBO/MBP-1 (book depth 1) + @test RType.MBP_10_MSG == RType.T(0x0A) # MBP-10 (book depth 10) + @test RType.STATUS_MSG == RType.T(0x12) # Exchange status record + @test RType.INSTRUMENT_DEF_MSG == RType.T(0x13) # Instrument definition record + @test RType.IMBALANCE_MSG == RType.T(0x14) # Order imbalance record + @test RType.ERROR_MSG == RType.T(0x15) # Error record from live gateway + @test RType.SYMBOL_MAPPING_MSG == RType.T(0x16) # Symbol mapping record from live gateway + @test RType.SYSTEM_MSG == RType.T(0x17) # Non-error record from live gateway + @test RType.STAT_MSG == RType.T(0x18) # Statistics record from publisher + @test RType.OHLCV_1S_MSG == RType.T(0x20) # OHLCV at 1-second cadence + @test RType.OHLCV_1M_MSG == RType.T(0x21) # OHLCV at 1-minute cadence + @test RType.OHLCV_1H_MSG == RType.T(0x22) # OHLCV at hourly cadence + @test RType.OHLCV_1D_MSG == RType.T(0x23) # OHLCV at daily cadence + @test RType.MBO_MSG == RType.T(0xA0) # Market-by-order record + @test RType.CMBP_1_MSG == RType.T(0xB1) # Consolidated market-by-price with book depth 1 + @test RType.CBBO_1S_MSG == RType.T(0xC0) # Consolidated market-by-price with book depth 1 at 1-second cadence + @test RType.CBBO_1M_MSG == RType.T(0xC1) # Consolidated market-by-price with book depth 1 at 1-minute cadence + @test RType.TCBBO_MSG == RType.T(0xC2) # Consolidated market-by-price with book depth 1 (trades only) + @test RType.BBO_1S_MSG == RType.T(0xC3) # Market-by-price with book depth 1 at 1-second cadence + @test RType.BBO_1M_MSG == RType.T(0xC4) # Market-by-price with book depth 1 at 1-minute cadence + + # Test Action enum + @test Action.ADD == Action.T(UInt8('A')) + @test Action.CANCEL == Action.T(UInt8('C')) + @test Action.MODIFY == Action.T(UInt8('M')) + @test Action.TRADE == Action.T(UInt8('T')) + @test Action.FILL == Action.T(UInt8('F')) + @test Action.CLEAR == Action.T(UInt8('R')) + + # Test Side enum + @test Side.ASK == Side.T(UInt8('A')) + @test Side.BID == Side.T(UInt8('B')) + @test Side.NONE == Side.T(UInt8('N')) + # Test InstrumentClass enum @test InstrumentClass.STOCK == InstrumentClass.T(UInt8('K')) @test InstrumentClass.CALL == InstrumentClass.T(UInt8('C')) @@ -119,13 +119,13 @@ @test InstrumentClass.FX_SPOT == InstrumentClass.T(UInt8('X')) @test InstrumentClass.BOND == InstrumentClass.T(UInt8('B')) end - - @testset "Constants are defined" begin - @test DBN_VERSION == 3 - @test FIXED_PRICE_SCALE == Int32(1_000_000_000) - @test UNDEF_PRICE == typemax(Int64) - @test UNDEF_ORDER_SIZE == typemax(UInt32) - @test UNDEF_TIMESTAMP == typemax(Int64) - end - + + @testset "Constants are defined" begin + @test DBN_VERSION == 3 + @test FIXED_PRICE_SCALE == Int32(1_000_000_000) + @test UNDEF_PRICE == typemax(Int64) + @test UNDEF_ORDER_SIZE == typemax(UInt32) + @test UNDEF_TIMESTAMP == typemax(Int64) + end + end diff --git a/test/test_phase10_complete.jl b/test/test_phase10_complete.jl index e942b28e..663f90f9 100644 --- a/test/test_phase10_complete.jl +++ b/test/test_phase10_complete.jl @@ -1,254 +1,254 @@ -using Test +using Test using DatabentoBinaryEncoding import DatabentoBinaryEncoding as DBN -using Dates -using Statistics -using BenchmarkTools -using DataFrames # for nrow / ncol on dbn_to_csv / dbn_to_parquet / records_to_dataframe output - -# Phase 10: Complete Integration and Performance Testing -@testset "Phase 10: Integration and Performance Testing" begin - - @testset "Sample DBN File Compatibility" begin - # Test reading various sample files from the reference implementation - sample_files = [ - "test_data.trades.dbn", - "test_data.mbo.dbn", - "test_data.mbp-1.dbn", - "test_data.mbp-10.dbn", - "test_data.ohlcv-1s.dbn", - "test_data.definition.dbn", - "test_data.status.dbn", - "test_data.imbalance.dbn" - ] - - for filename in sample_files - filepath = joinpath("test", "data", filename) - if isfile(filepath) - @testset "Reading $filename" begin - @test_nowarn begin - metadata, records = read_dbn_with_metadata(filepath) - @test !isnothing(metadata) - @test length(records) > 0 - @test metadata.dataset != "" - @test metadata.schema != Schema.MIX - println("✓ $filename - $(length(records)) records, schema: $(metadata.schema)") - end - end - end - end - - # Test compressed files - compressed_files = [ - "test_data.trades.v3.dbn.zst", - "test_data.mbo.v3.dbn.zst", - "test_data.mbp-1.v3.dbn.zst" - ] - - for filename in compressed_files - filepath = joinpath("test", "data", filename) - if isfile(filepath) - @testset "Reading compressed $filename" begin - @test_nowarn begin - metadata, records = read_dbn_with_metadata(filepath) - @test !isnothing(metadata) - @test length(records) > 0 - println("✓ $filename - $(length(records)) records (compressed)") - end - end - end - end - end - - @testset "Performance Benchmarking" begin - test_file = joinpath("test", "data", "test_data.trades.dbn") - if !isfile(test_file) - test_file = joinpath("test", "data", "test_data.mbo.dbn") - end - - if isfile(test_file) - @testset "Read Performance" begin - read_benchmark = @benchmark read_dbn($test_file) - read_time = median(read_benchmark.times) / 1e9 - file_size = filesize(test_file) - throughput_mb_per_sec = (file_size / 1024 / 1024) / read_time - - println("Read Performance:") - println(" File size: $(round(file_size/1024/1024, digits=2)) MB") - println(" Read time: $(round(read_time*1000, digits=2)) ms") - println(" Throughput: $(round(throughput_mb_per_sec, digits=2)) MB/s") - - @test read_time < 1.0 - @test throughput_mb_per_sec > 1.0 - end - - @testset "Write Performance" begin - metadata, records = read_dbn_with_metadata(test_file) - temp_file = tempname() * ".dbn" - - write_benchmark = @benchmark write_dbn($temp_file, $metadata, $records) - write_time = median(write_benchmark.times) / 1e9 - written_size = filesize(temp_file) - write_throughput = (written_size / 1024 / 1024) / write_time - - println("Write Performance:") - println(" Records: $(length(records))") - println(" Output size: $(round(written_size/1024/1024, digits=2)) MB") - println(" Write time: $(round(write_time*1000, digits=2)) ms") - println(" Throughput: $(round(write_throughput, digits=2)) MB/s") - - @test write_time < 2.0 - @test write_throughput > 0.5 - - safe_rm(temp_file) - end - end - end - - @testset "Memory Usage Profiling" begin - test_file = joinpath("test", "data", "test_data.trades.dbn") - if !isfile(test_file) - test_file = joinpath("test", "data", "test_data.mbo.dbn") - end - - if isfile(test_file) - @testset "Memory Efficiency" begin - GC.gc() - mem_before = Base.gc_live_bytes() - - metadata, records = read_dbn_with_metadata(test_file) - record_count = length(records) - - GC.gc() - mem_after = Base.gc_live_bytes() - mem_used = mem_after - mem_before - mem_per_record = mem_used / record_count - - println("Memory Usage:") - println(" Records: $record_count") - println(" Memory used: $(round(mem_used/1024/1024, digits=2)) MB") - println(" Memory per record: $(round(mem_per_record, digits=2)) bytes") - - @test mem_per_record < 1000 - @test mem_used < 100_000_000 - end - - @testset "Streaming Memory Usage" begin - record_count = 0 - max_memory = 0 - - GC.gc() - initial_memory = Base.gc_live_bytes() - - for record in DBNStream(test_file) - record_count += 1 - if record_count % 100 == 0 - current_memory = Base.gc_live_bytes() - initial_memory - max_memory = max(max_memory, current_memory) - end - if record_count > 1000 - break - end - end - - println("Streaming Memory:") - println(" Records processed: $record_count") - println(" Max memory delta: $(round(max_memory/1024/1024, digits=2)) MB") - - @test max_memory < 50_000_000 - end - end - end - - @testset "Thread Safety" begin - @testset "compress_daily_files Thread Safety" begin - temp_dir = mktempdir() - test_date = Date("2024-01-01") - - try - # Create test files - for i in 1:3 - filename = joinpath(temp_dir, "$(Dates.format(test_date, "yyyymmdd"))_file$i.dbn") - - metadata = Metadata( - UInt8(3), "TEST.PHASE10", Schema.TRADES, - 1640995200000000000, 1640995260000000000, UInt64(1), - SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - - hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) - trade = TradeMsg(hd, 100000000, 100, Action.TRADE, Side.NONE, 0x00, 0, 1640995200000000000, 0, 1) - - write_dbn(filename, metadata, [trade]) - end - - # Test concurrent compression simulation - success_count = 0 - for i in 1:2 - try - stats = compress_daily_files(test_date, temp_dir) - success_count += 1 - @test isa(stats, Vector) - catch e - println("Compression run $i failed: $e") - end - end - - @test success_count >= 1 - println("Thread safety test: $success_count/2 runs succeeded") - - finally - rm(temp_dir, recursive=true, force=true) - end - end - end - - @testset "Export Functionality" begin - test_file = joinpath("test", "data", "test_data.trades.dbn") - if !isfile(test_file) - test_file = joinpath("test", "data", "test_data.mbo.dbn") - end - - if isfile(test_file) - @testset "CSV Export" begin - temp_csv = tempname() * ".csv" - try - df = dbn_to_csv(test_file, temp_csv) - @test isfile(temp_csv) - @test nrow(df) > 0 - @test ncol(df) > 0 - println(" Exported $(nrow(df)) records to CSV") - finally - safe_rm(temp_csv) # GC.gc() + retry handles Windows EBUSY - end - end - - @testset "JSON Export" begin - temp_json = tempname() * ".json" - try - output = dbn_to_json(test_file, temp_json, pretty=true) - @test isfile(temp_json) - @test haskey(output, "metadata") - @test haskey(output, "records") - @test length(output["records"]) > 0 - println(" Exported $(length(output["records"])) records to JSON") - finally - safe_rm(temp_json) - end - end - +using Dates +using Statistics +using BenchmarkTools +using DataFrames # for nrow / ncol on dbn_to_csv / dbn_to_parquet / records_to_dataframe output + +# Phase 10: Complete Integration and Performance Testing +@testset "Phase 10: Integration and Performance Testing" begin + + @testset "Sample DBN File Compatibility" begin + # Test reading various sample files from the reference implementation + sample_files = [ + "test_data.trades.dbn", + "test_data.mbo.dbn", + "test_data.mbp-1.dbn", + "test_data.mbp-10.dbn", + "test_data.ohlcv-1s.dbn", + "test_data.definition.dbn", + "test_data.status.dbn", + "test_data.imbalance.dbn" + ] + + for filename in sample_files + filepath = joinpath("test", "data", filename) + if isfile(filepath) + @testset "Reading $filename" begin + @test_nowarn begin + metadata, records = read_dbn_with_metadata(filepath) + @test !isnothing(metadata) + @test length(records) > 0 + @test metadata.dataset != "" + @test metadata.schema != Schema.MIX + println("✓ $filename - $(length(records)) records, schema: $(metadata.schema)") + end + end + end + end + + # Test compressed files + compressed_files = [ + "test_data.trades.v3.dbn.zst", + "test_data.mbo.v3.dbn.zst", + "test_data.mbp-1.v3.dbn.zst" + ] + + for filename in compressed_files + filepath = joinpath("test", "data", filename) + if isfile(filepath) + @testset "Reading compressed $filename" begin + @test_nowarn begin + metadata, records = read_dbn_with_metadata(filepath) + @test !isnothing(metadata) + @test length(records) > 0 + println("✓ $filename - $(length(records)) records (compressed)") + end + end + end + end + end + + @testset "Performance Benchmarking" begin + test_file = joinpath("test", "data", "test_data.trades.dbn") + if !isfile(test_file) + test_file = joinpath("test", "data", "test_data.mbo.dbn") + end + + if isfile(test_file) + @testset "Read Performance" begin + read_benchmark = @benchmark read_dbn($test_file) + read_time = median(read_benchmark.times) / 1e9 + file_size = filesize(test_file) + throughput_mb_per_sec = (file_size / 1024 / 1024) / read_time + + println("Read Performance:") + println(" File size: $(round(file_size/1024/1024, digits=2)) MB") + println(" Read time: $(round(read_time*1000, digits=2)) ms") + println(" Throughput: $(round(throughput_mb_per_sec, digits=2)) MB/s") + + @test read_time < 1.0 + @test throughput_mb_per_sec > 1.0 + end + + @testset "Write Performance" begin + metadata, records = read_dbn_with_metadata(test_file) + temp_file = tempname() * ".dbn" + + write_benchmark = @benchmark write_dbn($temp_file, $metadata, $records) + write_time = median(write_benchmark.times) / 1e9 + written_size = filesize(temp_file) + write_throughput = (written_size / 1024 / 1024) / write_time + + println("Write Performance:") + println(" Records: $(length(records))") + println(" Output size: $(round(written_size/1024/1024, digits=2)) MB") + println(" Write time: $(round(write_time*1000, digits=2)) ms") + println(" Throughput: $(round(write_throughput, digits=2)) MB/s") + + @test write_time < 2.0 + @test write_throughput > 0.5 + + safe_rm(temp_file) + end + end + end + + @testset "Memory Usage Profiling" begin + test_file = joinpath("test", "data", "test_data.trades.dbn") + if !isfile(test_file) + test_file = joinpath("test", "data", "test_data.mbo.dbn") + end + + if isfile(test_file) + @testset "Memory Efficiency" begin + GC.gc() + mem_before = Base.gc_live_bytes() + + metadata, records = read_dbn_with_metadata(test_file) + record_count = length(records) + + GC.gc() + mem_after = Base.gc_live_bytes() + mem_used = mem_after - mem_before + mem_per_record = mem_used / record_count + + println("Memory Usage:") + println(" Records: $record_count") + println(" Memory used: $(round(mem_used/1024/1024, digits=2)) MB") + println(" Memory per record: $(round(mem_per_record, digits=2)) bytes") + + @test mem_per_record < 1000 + @test mem_used < 100_000_000 + end + + @testset "Streaming Memory Usage" begin + record_count = 0 + max_memory = 0 + + GC.gc() + initial_memory = Base.gc_live_bytes() + + for record in DBNStream(test_file) + record_count += 1 + if record_count % 100 == 0 + current_memory = Base.gc_live_bytes() - initial_memory + max_memory = max(max_memory, current_memory) + end + if record_count > 1000 + break + end + end + + println("Streaming Memory:") + println(" Records processed: $record_count") + println(" Max memory delta: $(round(max_memory/1024/1024, digits=2)) MB") + + @test max_memory < 50_000_000 + end + end + end + + @testset "Thread Safety" begin + @testset "compress_daily_files Thread Safety" begin + temp_dir = mktempdir() + test_date = Date("2024-01-01") + + try + # Create test files + for i in 1:3 + filename = joinpath(temp_dir, "$(Dates.format(test_date, "yyyymmdd"))_file$i.dbn") + + metadata = Metadata( + UInt8(3), "TEST.PHASE10", Schema.TRADES, + 1640995200000000000, 1640995260000000000, UInt64(1), + SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + + hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) + trade = TradeMsg(hd, 100000000, 100, Action.TRADE, Side.NONE, 0x00, 0, 1640995200000000000, 0, 1) + + write_dbn(filename, metadata, [trade]) + end + + # Test concurrent compression simulation + success_count = 0 + for i in 1:2 + try + stats = compress_daily_files(test_date, temp_dir) + success_count += 1 + @test isa(stats, Vector) + catch e + println("Compression run $i failed: $e") + end + end + + @test success_count >= 1 + println("Thread safety test: $success_count/2 runs succeeded") + + finally + rm(temp_dir, recursive=true, force=true) + end + end + end + + @testset "Export Functionality" begin + test_file = joinpath("test", "data", "test_data.trades.dbn") + if !isfile(test_file) + test_file = joinpath("test", "data", "test_data.mbo.dbn") + end + + if isfile(test_file) + @testset "CSV Export" begin + temp_csv = tempname() * ".csv" + try + df = dbn_to_csv(test_file, temp_csv) + @test isfile(temp_csv) + @test nrow(df) > 0 + @test ncol(df) > 0 + println(" Exported $(nrow(df)) records to CSV") + finally + safe_rm(temp_csv) # GC.gc() + retry handles Windows EBUSY + end + end + + @testset "JSON Export" begin + temp_json = tempname() * ".json" + try + output = dbn_to_json(test_file, temp_json, pretty=true) + @test isfile(temp_json) + @test haskey(output, "metadata") + @test haskey(output, "records") + @test length(output["records"]) > 0 + println(" Exported $(length(output["records"])) records to JSON") + finally + safe_rm(temp_json) + end + end + @testset "Parquet Export" begin - temp_parquet = tempname() * ".parquet" - try - df = dbn_to_parquet(test_file, temp_parquet) - @test isfile(temp_parquet) - @test nrow(df) > 0 - @test ncol(df) > 0 - println(" Exported $(nrow(df)) records to Parquet") - finally - safe_rm(temp_parquet) - end + temp_parquet = tempname() * ".parquet" + try + df = dbn_to_parquet(test_file, temp_parquet) + @test isfile(temp_parquet) + @test nrow(df) > 0 + @test ncol(df) > 0 + println(" Exported $(nrow(df)) records to Parquet") + finally + safe_rm(temp_parquet) + end end @testset "Parquet compression validation" begin @@ -261,14 +261,14 @@ using DataFrames # for nrow / ncol on dbn_to_csv / dbn_to_parquet / records_to safe_rm(temp_parquet) end end - - @testset "DataFrame Conversion" begin - metadata, records = read_dbn_with_metadata(test_file) - df = records_to_dataframe(records) - @test nrow(df) == length(records) - @test ncol(df) > 0 - println(" Converted $(nrow(df)) records to DataFrame") - end - end - end + + @testset "DataFrame Conversion" begin + metadata, records = read_dbn_with_metadata(test_file) + df = records_to_dataframe(records) + @test nrow(df) == length(records) + @test ncol(df) > 0 + println(" Converted $(nrow(df)) records to DataFrame") + end + end + end end diff --git a/test/test_phase2.jl b/test/test_phase2.jl index fc13689a..42a6a7f5 100644 --- a/test/test_phase2.jl +++ b/test/test_phase2.jl @@ -1,405 +1,405 @@ -# Phase 2: Struct and Type Testing - -@testset "Phase 2: Struct and Type Testing" begin - - @testset "Simple struct creation" begin - # Test RecordHeader creation - @testset "RecordHeader" begin - hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) - @test hd.length == 40 - @test hd.rtype == RType.MBP_0_MSG - @test hd.publisher_id == 1 - @test hd.instrument_id == 12345 - @test hd.ts_event == 1640995200000000000 - end - - # Test BidAskPair creation - @testset "BidAskPair" begin - pair = BidAskPair(10050000000, 10060000000, 100, 200, 5, 3) - @test pair.bid_px == 10050000000 - @test pair.ask_px == 10060000000 - @test pair.bid_sz == 100 - @test pair.ask_sz == 200 - @test pair.bid_ct == 5 - @test pair.ask_ct == 3 - end - - # Test VersionUpgradePolicy creation - @testset "VersionUpgradePolicy" begin - policy = VersionUpgradePolicy(0) - @test policy.upgrade_policy == 0 - end - - # Test DatasetCondition creation - @testset "DatasetCondition" begin - condition = DatasetCondition(0, 1640995200000000000, 1640995260000000000, 1000) - @test condition.last_ts_out == 0 - @test condition.start_ts == 1640995200000000000 - @test condition.end_ts == 1640995260000000000 - @test condition.limit == 1000 - end - end - - @testset "Metadata struct creation" begin - symbols = ["AAPL", "MSFT", "GOOGL"] - partial = String[] - not_found = String[] - mappings = Tuple{String,String,Int64,Int64}[] - - metadata = Metadata( - UInt8(DBN_VERSION), # version - "XNAS.ITCH", # dataset - Schema.TRADES, # schema - 1640995200000000000, # start - 1640995260000000000, # end_ts - UInt64(1000), # limit - SType.RAW_SYMBOL, # stype_in - SType.RAW_SYMBOL, # stype_out - false, # ts_out - symbols, # symbols - partial, # partial - not_found, # not_found - mappings # mappings - ) - - @test metadata.version == DBN_VERSION - @test metadata.dataset == "XNAS.ITCH" - @test metadata.schema == Schema.TRADES - @test metadata.start_ts == 1640995200000000000 - @test metadata.end_ts == 1640995260000000000 - @test metadata.limit == 1000 - @test metadata.stype_in == SType.RAW_SYMBOL - @test metadata.stype_out == SType.RAW_SYMBOL - @test metadata.ts_out == false - @test metadata.symbols == symbols - @test length(metadata.partial) == 0 - @test length(metadata.not_found) == 0 - @test length(metadata.mappings) == 0 - end - - @testset "Message type struct creation" begin - # Common record header for all messages - hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) - - @testset "MBOMsg" begin - msg = MBOMsg( - hd, # hd - 9876543210, # order_id - 10050000000, # price - 100, # size - 0x01, # flags - 1, # channel_id - Action.ADD, # action - Side.BID, # side - 1640995200000000001, # ts_recv - 1000, # ts_in_delta - 12345 # sequence - ) - - @test msg.hd == hd - @test msg.order_id == 9876543210 - @test msg.price == 10050000000 - @test msg.size == 100 - @test msg.flags == 0x01 - @test msg.channel_id == 1 - @test msg.action == Action.ADD - @test msg.side == Side.BID - @test msg.ts_recv == 1640995200000000001 - @test msg.ts_in_delta == 1000 - @test msg.sequence == 12345 - end - - @testset "TradeMsg" begin - msg = TradeMsg( - hd, # hd - 10055000000, # price - 250, # size - Action.TRADE, # action - Side.NONE, # side - 0x02, # flags - 0, # depth - 1640995200000000002, # ts_recv - 2000, # ts_in_delta - 12346 # sequence - ) - - @test msg.hd == hd - @test msg.price == 10055000000 - @test msg.size == 250 - @test msg.action == Action.TRADE - @test msg.side == Side.NONE - @test msg.flags == 0x02 - @test msg.depth == 0 - @test msg.ts_recv == 1640995200000000002 - @test msg.ts_in_delta == 2000 - @test msg.sequence == 12346 - end - - @testset "MBP1Msg" begin - levels = BidAskPair(10050000000, 10060000000, 100, 200, 5, 3) - - msg = MBP1Msg( - hd, # hd - 10055000000, # price - 150, # size - Action.MODIFY, # action - Side.ASK, # side - 0x04, # flags - 1, # depth - 1640995200000000003, # ts_recv - 3000, # ts_in_delta - 12347, # sequence - levels # levels - ) - - @test msg.hd == hd - @test msg.price == 10055000000 - @test msg.size == 150 - @test msg.action == Action.MODIFY - @test msg.side == Side.ASK - @test msg.flags == 0x04 - @test msg.depth == 1 - @test msg.ts_recv == 1640995200000000003 - @test msg.ts_in_delta == 3000 - @test msg.sequence == 12347 - @test msg.levels == levels - end - - @testset "MBP10Msg" begin - # Create 10 bid-ask pairs - levels = ntuple(10) do i - BidAskPair( - 10050000000 - (i-1)*1000000, # bid decreasing - 10060000000 + (i-1)*1000000, # ask increasing - 100 + i*10, # bid size - 200 + i*10, # ask size - 5 + i, # bid count - 3 + i # ask count - ) - end - - msg = MBP10Msg( - hd, # hd - 10055000000, # price - 175, # size - Action.CLEAR, # action - Side.BID, # side - 0x08, # flags - 2, # depth - 1640995200000000004, # ts_recv - 4000, # ts_in_delta - 12348, # sequence - levels # levels - ) - - @test msg.hd == hd - @test msg.price == 10055000000 - @test msg.size == 175 - @test msg.action == Action.CLEAR - @test msg.side == Side.BID - @test msg.flags == 0x08 - @test msg.depth == 2 - @test msg.ts_recv == 1640995200000000004 - @test msg.ts_in_delta == 4000 - @test msg.sequence == 12348 - @test length(msg.levels) == 10 - @test msg.levels[1].bid_px == 10050000000 - @test msg.levels[10].ask_px == 10069000000 - end - - @testset "OHLCVMsg" begin - msg = OHLCVMsg( - hd, # hd - 10050000000, # open - 10070000000, # high - 10040000000, # low - 10065000000, # close - 125000 # volume - ) - - @test msg.hd == hd - @test msg.open == 10050000000 - @test msg.high == 10070000000 - @test msg.low == 10040000000 - @test msg.close == 10065000000 - @test msg.volume == 125000 - end - - @testset "StatusMsg" begin - msg = StatusMsg( - hd, # hd - 1640995200000000005, # ts_recv - 1, # action - 0, # reason - 2, # trading_event - true, # is_trading - true, # is_quoting - false # is_short_sell_restricted - ) - - @test msg.hd == hd - @test msg.ts_recv == 1640995200000000005 - @test msg.action == 1 - @test msg.reason == 0 - @test msg.trading_event == 2 - @test msg.is_trading == true - @test msg.is_quoting == true - @test msg.is_short_sell_restricted == false - end - - @testset "ImbalanceMsg" begin - msg = ImbalanceMsg( - hd, # hd - 1640995200000000006, # ts_recv - 10055000000, # ref_price - UInt64(1640995230000000000), # auction_time - 10066000000, # cont_book_clr_price - 10067000000, # auct_interest_clr_price - 10068000000, # ssr_filling_price - 10069000000, # ind_match_price - 10070000000, # upper_collar - 10060000000, # lower_collar - UInt32(5000), # paired_qty - UInt32(15000), # total_imbalance_qty - UInt32(8000), # market_imbalance_qty - UInt32(2000), # unpaired_qty - UInt8('O'), # auction_type - Side.BID, # side - UInt8(1), # auction_status - UInt8(0), # freeze_status - UInt8(0), # num_extensions - UInt8('A'), # unpaired_side - UInt8('N') # significant_imbalance - ) - - @test msg.hd == hd - @test msg.ts_recv == 1640995200000000006 - @test msg.ref_price == 10055000000 - @test msg.auction_time == 1640995230000000000 - @test msg.cont_book_clr_price == 10066000000 - @test msg.auct_interest_clr_price == 10067000000 - @test msg.total_imbalance_qty == 15000 - @test msg.side == Side.BID - @test msg.auction_type == UInt8('O') - end - - @testset "StatMsg (DBN v3)" begin - msg = StatMsg( - hd, # hd - 1640995200000000007, # ts_recv - 1640995200000000000, # ts_ref - 10055000000, # price - 9876543210123456, # quantity (now Int64 in v3) - 12349, # sequence - 5000, # ts_in_delta - 1, # stat_type - 1, # channel_id - 2, # update_action - 0x01 # stat_flags - ) - - @test msg.hd == hd - @test msg.ts_recv == 1640995200000000007 - @test msg.ts_ref == 1640995200000000000 - @test msg.price == 10055000000 - @test msg.quantity == 9876543210123456 # Test large 64-bit value - @test msg.sequence == 12349 - @test msg.ts_in_delta == 5000 - @test msg.stat_type == 1 - @test msg.channel_id == 1 - @test msg.update_action == 2 - @test msg.stat_flags == 0x01 - end - - @testset "InstrumentDefMsg (DBN v3)" begin - msg = InstrumentDefMsg( - hd, # hd - 1640995200000000008, # ts_recv - 1000000, # min_price_increment - 1000000000, # display_factor - 1672531200000000000, # expiration - 1640995200000000000, # activation - 15000000000, # high_limit_price - 5000000000, # low_limit_price - 1000000000, # max_price_variation - 0, # trading_reference_price (v2 only) - 100, # unit_of_measure_qty - 1000000, # min_price_increment_amount - 1000000000, # price_ratio - 0, # inst_attrib_value - 0, # underlying_id - 9876543210123456789, # raw_instrument_id (now UInt64 in v3) - 0, # market_depth_implied - 10, # market_depth - 1, # market_segment_id - 1000000, # max_trade_vol - 1, # min_lot_size - 100, # min_lot_size_block - 1, # min_lot_size_round_lot - 1, # min_trade_vol - 1, # contract_multiplier - 0, # decay_quantity - 100, # original_contract_size - 0, # trading_reference_date (v2 only) - 1, # appl_id - 2024, # maturity_year - 0, # decay_start_date - 1, # channel_id - "USD", # currency - "USD", # settl_currency - "CS", # secsubtype - "AAPL", # raw_symbol - "TECH", # group - "XNAS", # exchange - "AAPL.NASDAQ", # asset (expanded to 11 bytes in v3) - "ESTVPS", # cfi - "CS", # security_type - "Shares", # unit_of_measure - "", # underlying - "", # strike_price_currency - InstrumentClass.STOCK, # instrument_class - 0, # strike_price - 'P', # match_algorithm - 0, # md_security_trading_status (v2 only) - 2, # main_fraction - 0, # price_display_format - 0, # settl_price_type (v2 only) - 0, # sub_fraction - 0, # underlying_product - 'A', # security_update_action - 0, # maturity_month - 0, # maturity_day - 0, # maturity_week - false, # user_defined_instrument - 0, # contract_multiplier_unit - 0, # flow_schedule_type - 0, # tick_rule - # New strategy leg fields in DBN v3 - 0, # leg_count - 0, # leg_index - 0, # leg_instrument_id - "", # leg_raw_symbol - Side.NONE, # leg_side - 0, # leg_underlying_id - InstrumentClass.STOCK, # leg_instrument_class - 0, # leg_ratio_qty_numerator - 0, # leg_ratio_qty_denominator - 0, # leg_ratio_price_numerator - 0, # leg_ratio_price_denominator - 0, # leg_price - 0 # leg_delta - ) - - @test msg.hd == hd - @test msg.ts_recv == 1640995200000000008 - @test msg.raw_instrument_id == 9876543210123456789 # Test UInt64 value - @test msg.currency == "USD" - @test msg.raw_symbol == "AAPL" - @test msg.asset == "AAPL.NASDAQ" # Test expanded asset field - @test msg.instrument_class == InstrumentClass.STOCK - @test msg.leg_count == 0 # Test new strategy leg fields - @test msg.leg_side == Side.NONE - @test msg.leg_instrument_class == InstrumentClass.STOCK - end - end +# Phase 2: Struct and Type Testing + +@testset "Phase 2: Struct and Type Testing" begin + + @testset "Simple struct creation" begin + # Test RecordHeader creation + @testset "RecordHeader" begin + hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) + @test hd.length == 40 + @test hd.rtype == RType.MBP_0_MSG + @test hd.publisher_id == 1 + @test hd.instrument_id == 12345 + @test hd.ts_event == 1640995200000000000 + end + + # Test BidAskPair creation + @testset "BidAskPair" begin + pair = BidAskPair(10050000000, 10060000000, 100, 200, 5, 3) + @test pair.bid_px == 10050000000 + @test pair.ask_px == 10060000000 + @test pair.bid_sz == 100 + @test pair.ask_sz == 200 + @test pair.bid_ct == 5 + @test pair.ask_ct == 3 + end + + # Test VersionUpgradePolicy creation + @testset "VersionUpgradePolicy" begin + policy = VersionUpgradePolicy(0) + @test policy.upgrade_policy == 0 + end + + # Test DatasetCondition creation + @testset "DatasetCondition" begin + condition = DatasetCondition(0, 1640995200000000000, 1640995260000000000, 1000) + @test condition.last_ts_out == 0 + @test condition.start_ts == 1640995200000000000 + @test condition.end_ts == 1640995260000000000 + @test condition.limit == 1000 + end + end + + @testset "Metadata struct creation" begin + symbols = ["AAPL", "MSFT", "GOOGL"] + partial = String[] + not_found = String[] + mappings = Tuple{String,String,Int64,Int64}[] + + metadata = Metadata( + UInt8(DBN_VERSION), # version + "XNAS.ITCH", # dataset + Schema.TRADES, # schema + 1640995200000000000, # start + 1640995260000000000, # end_ts + UInt64(1000), # limit + SType.RAW_SYMBOL, # stype_in + SType.RAW_SYMBOL, # stype_out + false, # ts_out + symbols, # symbols + partial, # partial + not_found, # not_found + mappings # mappings + ) + + @test metadata.version == DBN_VERSION + @test metadata.dataset == "XNAS.ITCH" + @test metadata.schema == Schema.TRADES + @test metadata.start_ts == 1640995200000000000 + @test metadata.end_ts == 1640995260000000000 + @test metadata.limit == 1000 + @test metadata.stype_in == SType.RAW_SYMBOL + @test metadata.stype_out == SType.RAW_SYMBOL + @test metadata.ts_out == false + @test metadata.symbols == symbols + @test length(metadata.partial) == 0 + @test length(metadata.not_found) == 0 + @test length(metadata.mappings) == 0 + end + + @testset "Message type struct creation" begin + # Common record header for all messages + hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) + + @testset "MBOMsg" begin + msg = MBOMsg( + hd, # hd + 9876543210, # order_id + 10050000000, # price + 100, # size + 0x01, # flags + 1, # channel_id + Action.ADD, # action + Side.BID, # side + 1640995200000000001, # ts_recv + 1000, # ts_in_delta + 12345 # sequence + ) + + @test msg.hd == hd + @test msg.order_id == 9876543210 + @test msg.price == 10050000000 + @test msg.size == 100 + @test msg.flags == 0x01 + @test msg.channel_id == 1 + @test msg.action == Action.ADD + @test msg.side == Side.BID + @test msg.ts_recv == 1640995200000000001 + @test msg.ts_in_delta == 1000 + @test msg.sequence == 12345 + end + + @testset "TradeMsg" begin + msg = TradeMsg( + hd, # hd + 10055000000, # price + 250, # size + Action.TRADE, # action + Side.NONE, # side + 0x02, # flags + 0, # depth + 1640995200000000002, # ts_recv + 2000, # ts_in_delta + 12346 # sequence + ) + + @test msg.hd == hd + @test msg.price == 10055000000 + @test msg.size == 250 + @test msg.action == Action.TRADE + @test msg.side == Side.NONE + @test msg.flags == 0x02 + @test msg.depth == 0 + @test msg.ts_recv == 1640995200000000002 + @test msg.ts_in_delta == 2000 + @test msg.sequence == 12346 + end + + @testset "MBP1Msg" begin + levels = BidAskPair(10050000000, 10060000000, 100, 200, 5, 3) + + msg = MBP1Msg( + hd, # hd + 10055000000, # price + 150, # size + Action.MODIFY, # action + Side.ASK, # side + 0x04, # flags + 1, # depth + 1640995200000000003, # ts_recv + 3000, # ts_in_delta + 12347, # sequence + levels # levels + ) + + @test msg.hd == hd + @test msg.price == 10055000000 + @test msg.size == 150 + @test msg.action == Action.MODIFY + @test msg.side == Side.ASK + @test msg.flags == 0x04 + @test msg.depth == 1 + @test msg.ts_recv == 1640995200000000003 + @test msg.ts_in_delta == 3000 + @test msg.sequence == 12347 + @test msg.levels == levels + end + + @testset "MBP10Msg" begin + # Create 10 bid-ask pairs + levels = ntuple(10) do i + BidAskPair( + 10050000000 - (i-1)*1000000, # bid decreasing + 10060000000 + (i-1)*1000000, # ask increasing + 100 + i*10, # bid size + 200 + i*10, # ask size + 5 + i, # bid count + 3 + i # ask count + ) + end + + msg = MBP10Msg( + hd, # hd + 10055000000, # price + 175, # size + Action.CLEAR, # action + Side.BID, # side + 0x08, # flags + 2, # depth + 1640995200000000004, # ts_recv + 4000, # ts_in_delta + 12348, # sequence + levels # levels + ) + + @test msg.hd == hd + @test msg.price == 10055000000 + @test msg.size == 175 + @test msg.action == Action.CLEAR + @test msg.side == Side.BID + @test msg.flags == 0x08 + @test msg.depth == 2 + @test msg.ts_recv == 1640995200000000004 + @test msg.ts_in_delta == 4000 + @test msg.sequence == 12348 + @test length(msg.levels) == 10 + @test msg.levels[1].bid_px == 10050000000 + @test msg.levels[10].ask_px == 10069000000 + end + + @testset "OHLCVMsg" begin + msg = OHLCVMsg( + hd, # hd + 10050000000, # open + 10070000000, # high + 10040000000, # low + 10065000000, # close + 125000 # volume + ) + + @test msg.hd == hd + @test msg.open == 10050000000 + @test msg.high == 10070000000 + @test msg.low == 10040000000 + @test msg.close == 10065000000 + @test msg.volume == 125000 + end + + @testset "StatusMsg" begin + msg = StatusMsg( + hd, # hd + 1640995200000000005, # ts_recv + 1, # action + 0, # reason + 2, # trading_event + true, # is_trading + true, # is_quoting + false # is_short_sell_restricted + ) + + @test msg.hd == hd + @test msg.ts_recv == 1640995200000000005 + @test msg.action == 1 + @test msg.reason == 0 + @test msg.trading_event == 2 + @test msg.is_trading == true + @test msg.is_quoting == true + @test msg.is_short_sell_restricted == false + end + + @testset "ImbalanceMsg" begin + msg = ImbalanceMsg( + hd, # hd + 1640995200000000006, # ts_recv + 10055000000, # ref_price + UInt64(1640995230000000000), # auction_time + 10066000000, # cont_book_clr_price + 10067000000, # auct_interest_clr_price + 10068000000, # ssr_filling_price + 10069000000, # ind_match_price + 10070000000, # upper_collar + 10060000000, # lower_collar + UInt32(5000), # paired_qty + UInt32(15000), # total_imbalance_qty + UInt32(8000), # market_imbalance_qty + UInt32(2000), # unpaired_qty + UInt8('O'), # auction_type + Side.BID, # side + UInt8(1), # auction_status + UInt8(0), # freeze_status + UInt8(0), # num_extensions + UInt8('A'), # unpaired_side + UInt8('N') # significant_imbalance + ) + + @test msg.hd == hd + @test msg.ts_recv == 1640995200000000006 + @test msg.ref_price == 10055000000 + @test msg.auction_time == 1640995230000000000 + @test msg.cont_book_clr_price == 10066000000 + @test msg.auct_interest_clr_price == 10067000000 + @test msg.total_imbalance_qty == 15000 + @test msg.side == Side.BID + @test msg.auction_type == UInt8('O') + end + + @testset "StatMsg (DBN v3)" begin + msg = StatMsg( + hd, # hd + 1640995200000000007, # ts_recv + 1640995200000000000, # ts_ref + 10055000000, # price + 9876543210123456, # quantity (now Int64 in v3) + 12349, # sequence + 5000, # ts_in_delta + 1, # stat_type + 1, # channel_id + 2, # update_action + 0x01 # stat_flags + ) + + @test msg.hd == hd + @test msg.ts_recv == 1640995200000000007 + @test msg.ts_ref == 1640995200000000000 + @test msg.price == 10055000000 + @test msg.quantity == 9876543210123456 # Test large 64-bit value + @test msg.sequence == 12349 + @test msg.ts_in_delta == 5000 + @test msg.stat_type == 1 + @test msg.channel_id == 1 + @test msg.update_action == 2 + @test msg.stat_flags == 0x01 + end + + @testset "InstrumentDefMsg (DBN v3)" begin + msg = InstrumentDefMsg( + hd, # hd + 1640995200000000008, # ts_recv + 1000000, # min_price_increment + 1000000000, # display_factor + 1672531200000000000, # expiration + 1640995200000000000, # activation + 15000000000, # high_limit_price + 5000000000, # low_limit_price + 1000000000, # max_price_variation + 0, # trading_reference_price (v2 only) + 100, # unit_of_measure_qty + 1000000, # min_price_increment_amount + 1000000000, # price_ratio + 0, # inst_attrib_value + 0, # underlying_id + 9876543210123456789, # raw_instrument_id (now UInt64 in v3) + 0, # market_depth_implied + 10, # market_depth + 1, # market_segment_id + 1000000, # max_trade_vol + 1, # min_lot_size + 100, # min_lot_size_block + 1, # min_lot_size_round_lot + 1, # min_trade_vol + 1, # contract_multiplier + 0, # decay_quantity + 100, # original_contract_size + 0, # trading_reference_date (v2 only) + 1, # appl_id + 2024, # maturity_year + 0, # decay_start_date + 1, # channel_id + "USD", # currency + "USD", # settl_currency + "CS", # secsubtype + "AAPL", # raw_symbol + "TECH", # group + "XNAS", # exchange + "AAPL.NASDAQ", # asset (expanded to 11 bytes in v3) + "ESTVPS", # cfi + "CS", # security_type + "Shares", # unit_of_measure + "", # underlying + "", # strike_price_currency + InstrumentClass.STOCK, # instrument_class + 0, # strike_price + 'P', # match_algorithm + 0, # md_security_trading_status (v2 only) + 2, # main_fraction + 0, # price_display_format + 0, # settl_price_type (v2 only) + 0, # sub_fraction + 0, # underlying_product + 'A', # security_update_action + 0, # maturity_month + 0, # maturity_day + 0, # maturity_week + false, # user_defined_instrument + 0, # contract_multiplier_unit + 0, # flow_schedule_type + 0, # tick_rule + # New strategy leg fields in DBN v3 + 0, # leg_count + 0, # leg_index + 0, # leg_instrument_id + "", # leg_raw_symbol + Side.NONE, # leg_side + 0, # leg_underlying_id + InstrumentClass.STOCK, # leg_instrument_class + 0, # leg_ratio_qty_numerator + 0, # leg_ratio_qty_denominator + 0, # leg_ratio_price_numerator + 0, # leg_ratio_price_denominator + 0, # leg_price + 0 # leg_delta + ) + + @test msg.hd == hd + @test msg.ts_recv == 1640995200000000008 + @test msg.raw_instrument_id == 9876543210123456789 # Test UInt64 value + @test msg.currency == "USD" + @test msg.raw_symbol == "AAPL" + @test msg.asset == "AAPL.NASDAQ" # Test expanded asset field + @test msg.instrument_class == InstrumentClass.STOCK + @test msg.leg_count == 0 # Test new strategy leg fields + @test msg.leg_side == Side.NONE + @test msg.leg_instrument_class == InstrumentClass.STOCK + end + end end \ No newline at end of file diff --git a/test/test_phase3.jl b/test/test_phase3.jl index 21854c0b..2d0bd511 100644 --- a/test/test_phase3.jl +++ b/test/test_phase3.jl @@ -1,258 +1,258 @@ -# Phase 3: Utility Function Testing -using Dates - - -@testset "Phase 3: Utility Function Testing" begin - - @testset "Price conversion functions" begin - @testset "price_to_float" begin - # Test normal prices - @test price_to_float(10050000000) ≈ 10.05 - @test price_to_float(1000000000) ≈ 1.0 - @test price_to_float(500000000) ≈ 0.5 - @test price_to_float(0) ≈ 0.0 - @test price_to_float(-1000000000) ≈ -1.0 - - # Test with custom scale - @test price_to_float(10050, Int32(1000)) ≈ 10.05 - @test price_to_float(1000, Int32(100)) ≈ 10.0 - - # Test with UNDEF_PRICE - @test isnan(price_to_float(UNDEF_PRICE)) - - # Test edge cases (avoid typemax since it's UNDEF_PRICE) - large_price = typemax(Int64) ÷ 2 - @test price_to_float(large_price, FIXED_PRICE_SCALE) ≈ Float64(large_price) / Float64(FIXED_PRICE_SCALE) - @test price_to_float(typemin(Int64), FIXED_PRICE_SCALE) ≈ Float64(typemin(Int64)) / Float64(FIXED_PRICE_SCALE) - end - - @testset "float_to_price" begin - # Test normal values - @test float_to_price(10.05) == 10050000000 - @test float_to_price(1.0) == 1000000000 - @test float_to_price(0.5) == 500000000 - @test float_to_price(0.0) == 0 - @test float_to_price(-1.0) == -1000000000 - - # Test with custom scale - @test float_to_price(10.05, Int32(1000)) == 10050 - @test float_to_price(10.0, Int32(100)) == 1000 - - # Test with NaN and Inf - @test float_to_price(NaN) == UNDEF_PRICE - @test float_to_price(Inf) == UNDEF_PRICE - @test float_to_price(-Inf) == UNDEF_PRICE - - # Test rounding - @test float_to_price(10.0546) == 10054600000 - @test float_to_price(10.0545) == 10054500000 - end - - @testset "price_to_float and float_to_price round-trip" begin - test_prices = [0, 1000000000, 10050000000, 999999999, -1000000000] - for price in test_prices - @test float_to_price(price_to_float(price)) == price - end - - test_floats = [0.0, 1.0, 10.05, 0.999999999, -1.0] - for value in test_floats - @test price_to_float(float_to_price(value)) ≈ value - end - end - end - - @testset "Timestamp conversion functions" begin - @testset "DBNTimestamp constructor" begin - # Test normal timestamp - ts = DBNTimestamp(1640995200123456789) - @test ts.seconds == 1640995200 - @test ts.nanoseconds == 123456789 - - # Test with UNDEF_TIMESTAMP - ts_undef = DBNTimestamp(UNDEF_TIMESTAMP) - @test ts_undef.seconds == UNDEF_TIMESTAMP - @test ts_undef.nanoseconds == 0 - - # Test zero timestamp - ts_zero = DBNTimestamp(0) - @test ts_zero.seconds == 0 - @test ts_zero.nanoseconds == 0 - - # Test edge cases - ts_max_ns = DBNTimestamp(999999999) - @test ts_max_ns.seconds == 0 - @test ts_max_ns.nanoseconds == 999999999 - - ts_one_sec = DBNTimestamp(1000000000) - @test ts_one_sec.seconds == 1 - @test ts_one_sec.nanoseconds == 0 - end - - @testset "to_nanoseconds" begin - # Test normal timestamp - ts = DBNTimestamp(1640995200, 123456789) - @test to_nanoseconds(ts) == 1640995200123456789 - - # Test with UNDEF_TIMESTAMP - ts_undef = DBNTimestamp(UNDEF_TIMESTAMP, 0) - @test to_nanoseconds(ts_undef) == UNDEF_TIMESTAMP - - # Test zero values - ts_zero = DBNTimestamp(0, 0) - @test to_nanoseconds(ts_zero) == 0 - - # Test maximum nanoseconds - ts_max = DBNTimestamp(1640995200, 999999999) - @test to_nanoseconds(ts_max) == 1640995200999999999 - end - - @testset "DBNTimestamp and to_nanoseconds round-trip" begin - test_timestamps = [ - 0, - 1000000000, # 1 second - 1640995200123456789, # Typical timestamp - 999999999, # Max nanoseconds in first second - typemax(Int64) ÷ 2 # Large but safe value - ] - - for ts_ns in test_timestamps - if ts_ns != UNDEF_TIMESTAMP - @test to_nanoseconds(DBNTimestamp(ts_ns)) == ts_ns - end - end - end - - @testset "ts_to_datetime" begin - # Test normal timestamp - result = ts_to_datetime(1640995200123456789) - @test result !== nothing - @test result.datetime isa DateTime - @test result.nanoseconds == 123456789 - - # Test with UNDEF_TIMESTAMP - @test ts_to_datetime(UNDEF_TIMESTAMP) === nothing - - # Test zero timestamp (Unix epoch) - result_epoch = ts_to_datetime(0) - @test result_epoch !== nothing - @test result_epoch.datetime == DateTime(1970, 1, 1, 0, 0, 0) - @test result_epoch.nanoseconds == 0 - - # Test timestamp with only seconds - result_sec = ts_to_datetime(1640995200000000000) - @test result_sec !== nothing - @test result_sec.nanoseconds == 0 - end - - @testset "datetime_to_ts" begin - # Test normal datetime - dt = DateTime(2022, 1, 1, 12, 0, 0) - ts = datetime_to_ts(dt, Int32(0)) - @test ts > 0 - - # Test with nanoseconds - ts_with_ns = datetime_to_ts(dt, Int32(123456789)) - @test ts_with_ns == ts + 123456789 - - # Test epoch - epoch_dt = DateTime(1970, 1, 1, 0, 0, 0) - epoch_ts = datetime_to_ts(epoch_dt, Int32(0)) - @test epoch_ts == 0 - end - - @testset "ts_to_date_time" begin - # Test normal timestamp - result = ts_to_date_time(1640995200123456789) - @test result !== nothing - @test result.date isa Date - @test result.time isa Dates.Time - @test result.timestamp isa DBNTimestamp - - # Test with UNDEF_TIMESTAMP - @test ts_to_date_time(UNDEF_TIMESTAMP) === nothing - - # Test epoch - result_epoch = ts_to_date_time(0) - @test result_epoch !== nothing - @test result_epoch.date == Date(1970, 1, 1) - @test result_epoch.time == Dates.Time(0, 0, 0) - end - - @testset "date_time_to_ts" begin - # Test normal date and time - date = Date(2022, 1, 1) - time = Dates.Time(12, 30, 45, 123, 456, 789) - ts = date_time_to_ts(date, time) - @test ts > 0 - - # Test epoch - epoch_date = Date(1970, 1, 1) - epoch_time = Dates.Time(0, 0, 0) - epoch_ts = date_time_to_ts(epoch_date, epoch_time) - @test epoch_ts == 0 - - # Test midnight - midnight = Dates.Time(0, 0, 0) - ts_midnight = date_time_to_ts(date, midnight) - @test ts_midnight % (24 * 60 * 60 * 1_000_000_000) == 0 - end - - @testset "Timestamp conversion round-trips" begin - # Test datetime round-trip - original_ts = 1640995200123456789 - result = ts_to_datetime(original_ts) - if result !== nothing - # Note: We lose nanosecond precision in DateTime, so we add it back - reconstructed_ts = datetime_to_ts(result.datetime, result.nanoseconds) - @test reconstructed_ts == original_ts - end - - # Test date_time round-trip - original_ts2 = 1640995200123456789 - result2 = ts_to_date_time(original_ts2) - if result2 !== nothing - reconstructed_ts2 = date_time_to_ts(result2.date, result2.time) - @test reconstructed_ts2 == original_ts2 - end - end - end - - @testset "Constants validation" begin - @test DBN_VERSION == 3 - @test FIXED_PRICE_SCALE == Int32(1_000_000_000) - @test UNDEF_PRICE == typemax(Int64) - @test UNDEF_ORDER_SIZE == typemax(UInt32) - @test UNDEF_TIMESTAMP == typemax(Int64) - - # Test that constants are sensible - @test FIXED_PRICE_SCALE > 0 - @test UNDEF_PRICE > 0 - @test UNDEF_ORDER_SIZE > 0 - @test UNDEF_TIMESTAMP > 0 - end - - @testset "Edge cases and boundary values" begin - @testset "Price conversion edge cases" begin - # Test very small values - @test price_to_float(1) ≈ 1e-9 - @test float_to_price(1e-9) == 1 - - # Test precision limits - small_price = 1000000 # 0.001 dollars - @test price_to_float(small_price) ≈ 0.001 - @test float_to_price(price_to_float(small_price)) == small_price - end - - @testset "Timestamp edge cases" begin - # Test large timestamps near type limits (but not max which is UNDEF) - large_ts = typemax(Int64) ÷ 2 - dbn_ts = DBNTimestamp(large_ts) - @test to_nanoseconds(dbn_ts) == large_ts - - # Test negative timestamps (before Unix epoch) - negative_ts = -1000000000 # 1 second before epoch - dbn_ts_neg = DBNTimestamp(negative_ts) - @test to_nanoseconds(dbn_ts_neg) == negative_ts - end - end +# Phase 3: Utility Function Testing +using Dates + + +@testset "Phase 3: Utility Function Testing" begin + + @testset "Price conversion functions" begin + @testset "price_to_float" begin + # Test normal prices + @test price_to_float(10050000000) ≈ 10.05 + @test price_to_float(1000000000) ≈ 1.0 + @test price_to_float(500000000) ≈ 0.5 + @test price_to_float(0) ≈ 0.0 + @test price_to_float(-1000000000) ≈ -1.0 + + # Test with custom scale + @test price_to_float(10050, Int32(1000)) ≈ 10.05 + @test price_to_float(1000, Int32(100)) ≈ 10.0 + + # Test with UNDEF_PRICE + @test isnan(price_to_float(UNDEF_PRICE)) + + # Test edge cases (avoid typemax since it's UNDEF_PRICE) + large_price = typemax(Int64) ÷ 2 + @test price_to_float(large_price, FIXED_PRICE_SCALE) ≈ Float64(large_price) / Float64(FIXED_PRICE_SCALE) + @test price_to_float(typemin(Int64), FIXED_PRICE_SCALE) ≈ Float64(typemin(Int64)) / Float64(FIXED_PRICE_SCALE) + end + + @testset "float_to_price" begin + # Test normal values + @test float_to_price(10.05) == 10050000000 + @test float_to_price(1.0) == 1000000000 + @test float_to_price(0.5) == 500000000 + @test float_to_price(0.0) == 0 + @test float_to_price(-1.0) == -1000000000 + + # Test with custom scale + @test float_to_price(10.05, Int32(1000)) == 10050 + @test float_to_price(10.0, Int32(100)) == 1000 + + # Test with NaN and Inf + @test float_to_price(NaN) == UNDEF_PRICE + @test float_to_price(Inf) == UNDEF_PRICE + @test float_to_price(-Inf) == UNDEF_PRICE + + # Test rounding + @test float_to_price(10.0546) == 10054600000 + @test float_to_price(10.0545) == 10054500000 + end + + @testset "price_to_float and float_to_price round-trip" begin + test_prices = [0, 1000000000, 10050000000, 999999999, -1000000000] + for price in test_prices + @test float_to_price(price_to_float(price)) == price + end + + test_floats = [0.0, 1.0, 10.05, 0.999999999, -1.0] + for value in test_floats + @test price_to_float(float_to_price(value)) ≈ value + end + end + end + + @testset "Timestamp conversion functions" begin + @testset "DBNTimestamp constructor" begin + # Test normal timestamp + ts = DBNTimestamp(1640995200123456789) + @test ts.seconds == 1640995200 + @test ts.nanoseconds == 123456789 + + # Test with UNDEF_TIMESTAMP + ts_undef = DBNTimestamp(UNDEF_TIMESTAMP) + @test ts_undef.seconds == UNDEF_TIMESTAMP + @test ts_undef.nanoseconds == 0 + + # Test zero timestamp + ts_zero = DBNTimestamp(0) + @test ts_zero.seconds == 0 + @test ts_zero.nanoseconds == 0 + + # Test edge cases + ts_max_ns = DBNTimestamp(999999999) + @test ts_max_ns.seconds == 0 + @test ts_max_ns.nanoseconds == 999999999 + + ts_one_sec = DBNTimestamp(1000000000) + @test ts_one_sec.seconds == 1 + @test ts_one_sec.nanoseconds == 0 + end + + @testset "to_nanoseconds" begin + # Test normal timestamp + ts = DBNTimestamp(1640995200, 123456789) + @test to_nanoseconds(ts) == 1640995200123456789 + + # Test with UNDEF_TIMESTAMP + ts_undef = DBNTimestamp(UNDEF_TIMESTAMP, 0) + @test to_nanoseconds(ts_undef) == UNDEF_TIMESTAMP + + # Test zero values + ts_zero = DBNTimestamp(0, 0) + @test to_nanoseconds(ts_zero) == 0 + + # Test maximum nanoseconds + ts_max = DBNTimestamp(1640995200, 999999999) + @test to_nanoseconds(ts_max) == 1640995200999999999 + end + + @testset "DBNTimestamp and to_nanoseconds round-trip" begin + test_timestamps = [ + 0, + 1000000000, # 1 second + 1640995200123456789, # Typical timestamp + 999999999, # Max nanoseconds in first second + typemax(Int64) ÷ 2 # Large but safe value + ] + + for ts_ns in test_timestamps + if ts_ns != UNDEF_TIMESTAMP + @test to_nanoseconds(DBNTimestamp(ts_ns)) == ts_ns + end + end + end + + @testset "ts_to_datetime" begin + # Test normal timestamp + result = ts_to_datetime(1640995200123456789) + @test result !== nothing + @test result.datetime isa DateTime + @test result.nanoseconds == 123456789 + + # Test with UNDEF_TIMESTAMP + @test ts_to_datetime(UNDEF_TIMESTAMP) === nothing + + # Test zero timestamp (Unix epoch) + result_epoch = ts_to_datetime(0) + @test result_epoch !== nothing + @test result_epoch.datetime == DateTime(1970, 1, 1, 0, 0, 0) + @test result_epoch.nanoseconds == 0 + + # Test timestamp with only seconds + result_sec = ts_to_datetime(1640995200000000000) + @test result_sec !== nothing + @test result_sec.nanoseconds == 0 + end + + @testset "datetime_to_ts" begin + # Test normal datetime + dt = DateTime(2022, 1, 1, 12, 0, 0) + ts = datetime_to_ts(dt, Int32(0)) + @test ts > 0 + + # Test with nanoseconds + ts_with_ns = datetime_to_ts(dt, Int32(123456789)) + @test ts_with_ns == ts + 123456789 + + # Test epoch + epoch_dt = DateTime(1970, 1, 1, 0, 0, 0) + epoch_ts = datetime_to_ts(epoch_dt, Int32(0)) + @test epoch_ts == 0 + end + + @testset "ts_to_date_time" begin + # Test normal timestamp + result = ts_to_date_time(1640995200123456789) + @test result !== nothing + @test result.date isa Date + @test result.time isa Dates.Time + @test result.timestamp isa DBNTimestamp + + # Test with UNDEF_TIMESTAMP + @test ts_to_date_time(UNDEF_TIMESTAMP) === nothing + + # Test epoch + result_epoch = ts_to_date_time(0) + @test result_epoch !== nothing + @test result_epoch.date == Date(1970, 1, 1) + @test result_epoch.time == Dates.Time(0, 0, 0) + end + + @testset "date_time_to_ts" begin + # Test normal date and time + date = Date(2022, 1, 1) + time = Dates.Time(12, 30, 45, 123, 456, 789) + ts = date_time_to_ts(date, time) + @test ts > 0 + + # Test epoch + epoch_date = Date(1970, 1, 1) + epoch_time = Dates.Time(0, 0, 0) + epoch_ts = date_time_to_ts(epoch_date, epoch_time) + @test epoch_ts == 0 + + # Test midnight + midnight = Dates.Time(0, 0, 0) + ts_midnight = date_time_to_ts(date, midnight) + @test ts_midnight % (24 * 60 * 60 * 1_000_000_000) == 0 + end + + @testset "Timestamp conversion round-trips" begin + # Test datetime round-trip + original_ts = 1640995200123456789 + result = ts_to_datetime(original_ts) + if result !== nothing + # Note: We lose nanosecond precision in DateTime, so we add it back + reconstructed_ts = datetime_to_ts(result.datetime, result.nanoseconds) + @test reconstructed_ts == original_ts + end + + # Test date_time round-trip + original_ts2 = 1640995200123456789 + result2 = ts_to_date_time(original_ts2) + if result2 !== nothing + reconstructed_ts2 = date_time_to_ts(result2.date, result2.time) + @test reconstructed_ts2 == original_ts2 + end + end + end + + @testset "Constants validation" begin + @test DBN_VERSION == 3 + @test FIXED_PRICE_SCALE == Int32(1_000_000_000) + @test UNDEF_PRICE == typemax(Int64) + @test UNDEF_ORDER_SIZE == typemax(UInt32) + @test UNDEF_TIMESTAMP == typemax(Int64) + + # Test that constants are sensible + @test FIXED_PRICE_SCALE > 0 + @test UNDEF_PRICE > 0 + @test UNDEF_ORDER_SIZE > 0 + @test UNDEF_TIMESTAMP > 0 + end + + @testset "Edge cases and boundary values" begin + @testset "Price conversion edge cases" begin + # Test very small values + @test price_to_float(1) ≈ 1e-9 + @test float_to_price(1e-9) == 1 + + # Test precision limits + small_price = 1000000 # 0.001 dollars + @test price_to_float(small_price) ≈ 0.001 + @test float_to_price(price_to_float(small_price)) == small_price + end + + @testset "Timestamp edge cases" begin + # Test large timestamps near type limits (but not max which is UNDEF) + large_ts = typemax(Int64) ÷ 2 + dbn_ts = DBNTimestamp(large_ts) + @test to_nanoseconds(dbn_ts) == large_ts + + # Test negative timestamps (before Unix epoch) + negative_ts = -1000000000 # 1 second before epoch + dbn_ts_neg = DBNTimestamp(negative_ts) + @test to_nanoseconds(dbn_ts_neg) == negative_ts + end + end end \ No newline at end of file diff --git a/test/test_phase4.jl b/test/test_phase4.jl index 9d42397a..30e1e958 100644 --- a/test/test_phase4.jl +++ b/test/test_phase4.jl @@ -1,371 +1,371 @@ -# Phase 4: Basic Read/Write Testing (No Compression) -using Dates - -@testset "Phase 4: Basic Read/Write Testing (No Compression)" begin - - # Helper function to create test metadata - function create_test_metadata() - return Metadata( - UInt8(DBN_VERSION), # version - "TEST.ITCH", # dataset - Schema.TRADES, # schema - 1640995200000000000, # start_ts - 1640995260000000000, # end_ts - UInt64(1000), # limit - SType.RAW_SYMBOL, # stype_in - SType.RAW_SYMBOL, # stype_out - false, # ts_out - ["AAPL", "MSFT"], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings - ) - end - - # Helper function to create test TradeMsg - function create_test_trade_msg() - hd = RecordHeader(10, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) # 10 units = 40 bytes - return TradeMsg( - hd, # hd - 10055000000, # price - 250, # size - Action.TRADE, # action - Side.NONE, # side - 0x02, # flags - 0, # depth - 1640995200000000001, # ts_recv - 1000, # ts_in_delta - 12346 # sequence - ) - end - - @testset "Minimal DBN file writer test" begin - temp_file = tempname() * ".dbn" - - try - @testset "Write DBN header" begin - metadata = create_test_metadata() - - # Test writing header - open(temp_file, "w") do f - encoder = DBNEncoder(f, metadata) - write_header(encoder) - finalize_encoder(encoder) - end - - @test isfile(temp_file) - @test filesize(temp_file) > 0 - end - - @testset "Write a single TradeMsg record" begin - metadata = create_test_metadata() - trade_msg = create_test_trade_msg() - - # Write header and single record - write_dbn(temp_file, metadata, [trade_msg]) - - @test isfile(temp_file) - @test filesize(temp_file) > 100 # Should be more than just header - end - - @testset "Verify file is created" begin - @test isfile(temp_file) - - # Check that file has proper DBN magic bytes - open(temp_file, "r") do f - magic = read(f, 3) - @test magic == b"DBN" - - version = read(f, UInt8) - @test version == DBN_VERSION - end - end - - finally - # Clean up - safe_rm(temp_file) - end - end - - @testset "Minimal DBN file reader test" begin - temp_file = tempname() * ".dbn" - - try - # First write a test file - metadata = create_test_metadata() - trade_msg = create_test_trade_msg() - write_dbn(temp_file, metadata, [trade_msg]) - - @testset "Read the file created above" begin - records = read_dbn(temp_file) - @test length(records) == 1 - @test records[1] isa TradeMsg - end - - @testset "Verify header is parsed correctly" begin - open(temp_file, "r") do f - decoder = DBNDecoder(f) - read_header!(decoder) - - @test decoder.metadata !== nothing - @test decoder.metadata.version == DBN_VERSION - @test decoder.metadata.dataset == "TEST.ITCH" - @test decoder.metadata.schema == Schema.TRADES - @test length(decoder.metadata.symbols) == 2 - @test decoder.metadata.symbols[1] == "AAPL" - @test decoder.metadata.symbols[2] == "MSFT" - end - end - - @testset "Verify record is read correctly" begin - records = read_dbn(temp_file) - trade_record = records[1] - original_trade = create_test_trade_msg() - - @test DBN.record_length_bytes(trade_record.hd) == DBN.record_length_bytes(original_trade.hd) - @test trade_record.hd.rtype == original_trade.hd.rtype - @test trade_record.hd.publisher_id == original_trade.hd.publisher_id - @test trade_record.hd.instrument_id == original_trade.hd.instrument_id - @test trade_record.hd.ts_event == original_trade.hd.ts_event - - @test trade_record.price == original_trade.price - @test trade_record.size == original_trade.size - @test trade_record.action == original_trade.action - @test trade_record.side == original_trade.side - @test trade_record.flags == original_trade.flags - @test trade_record.depth == original_trade.depth - @test trade_record.ts_recv == original_trade.ts_recv - @test trade_record.ts_in_delta == original_trade.ts_in_delta - @test trade_record.sequence == original_trade.sequence - end - - finally - safe_rm(temp_file) - end - end - - @testset "Round-trip testing (write then read)" begin - temp_file = tempname() * ".dbn" - - try - @testset "Write multiple record types" begin - metadata = create_test_metadata() - - # Create different message types - hd1 = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) - trade_msg = TradeMsg(hd1, 10055000000, 250, Action.TRADE, Side.NONE, 0x02, 0, 1640995200000000001, 1000, 12346) - - hd2 = RecordHeader(50, RType.MBO_MSG, 1, 12345, 1640995200000000002) - mbo_msg = MBOMsg(hd2, 9876543210, 10050000000, 100, 0x01, 1, Action.ADD, Side.BID, 1640995200000000003, 2000, 12347) - - hd3 = RecordHeader(30, RType.OHLCV_1S_MSG, 1, 12345, 1640995200000000004) - ohlcv_msg = OHLCVMsg(hd3, 10050000000, 10070000000, 10040000000, 10065000000, 125000) - - records = [trade_msg, mbo_msg, ohlcv_msg] - write_dbn(temp_file, metadata, records) - - @test isfile(temp_file) - @test filesize(temp_file) > 200 # Should be substantial - end - - @testset "Read them back" begin - records = read_dbn(temp_file) - @test length(records) == 3 - @test records[1] isa TradeMsg - @test records[2] isa MBOMsg - @test records[3] isa OHLCVMsg - end - - @testset "Verify data integrity" begin - records = read_dbn(temp_file) - - # Test TradeMsg integrity - trade_record = records[1] - @test trade_record.hd.rtype == RType.MBP_0_MSG - @test trade_record.price == 10055000000 - @test trade_record.size == 250 - @test trade_record.action == Action.TRADE - @test trade_record.side == Side.NONE - - # Test MBOMsg integrity - mbo_record = records[2] - @test mbo_record.hd.rtype == RType.MBO_MSG - @test mbo_record.order_id == 9876543210 - @test mbo_record.price == 10050000000 - @test mbo_record.size == 100 - @test mbo_record.action == Action.ADD - @test mbo_record.side == Side.BID - - # Test OHLCVMsg integrity - ohlcv_record = records[3] - @test ohlcv_record.hd.rtype == RType.OHLCV_1S_MSG - @test ohlcv_record.open == 10050000000 - @test ohlcv_record.high == 10070000000 - @test ohlcv_record.low == 10040000000 - @test ohlcv_record.close == 10065000000 - @test ohlcv_record.volume == 125000 - end - - finally - safe_rm(temp_file) - end - end - - @testset "DBNEncoder and DBNDecoder direct testing" begin - temp_file = tempname() * ".dbn" - - try - @testset "Direct encoder usage" begin - metadata = create_test_metadata() - trade_msg = create_test_trade_msg() - - open(temp_file, "w") do f - encoder = DBNEncoder(f, metadata) - write_header(encoder) - write_record(encoder, trade_msg) - finalize_encoder(encoder) - end - - @test isfile(temp_file) - @test filesize(temp_file) > 50 - end - - @testset "Direct decoder usage" begin - open(temp_file, "r") do f - decoder = DBNDecoder(f) - read_header!(decoder) - - # Verify metadata was read correctly - @test decoder.metadata.dataset == "TEST.ITCH" - @test decoder.metadata.schema == Schema.TRADES - - # Read the record - record = read_record(decoder) - @test record isa TradeMsg - @test record.price == 10055000000 - - # Try to read another record (should be nothing) - next_record = read_record(decoder) - @test next_record === nothing - end - end - - finally - safe_rm(temp_file) - end - end - - @testset "Error handling and edge cases" begin - @testset "Writing to invalid path" begin - invalid_path = "/invalid/nonexistent/path/test.dbn" - metadata = create_test_metadata() - trade_msg = create_test_trade_msg() - - @test_throws Exception write_dbn(invalid_path, metadata, [trade_msg]) - end - - @testset "Reading nonexistent file" begin - nonexistent_file = "nonexistent_file.dbn" - @test_throws Exception read_dbn(nonexistent_file) - end - - @testset "Reading invalid DBN file" begin - temp_file = tempname() * ".dbn" - - try - # Write invalid magic bytes - open(temp_file, "w") do f - write(f, b"INVALID") - end - - @test_throws Exception read_dbn(temp_file) - - finally - safe_rm(temp_file) - end - end - - @testset "Empty file handling" begin - temp_file = tempname() * ".dbn" - - try - # Create empty file - touch(temp_file) - @test_throws Exception read_dbn(temp_file) - - finally - safe_rm(temp_file) - end - end - end - - @testset "Different metadata configurations" begin - temp_file = tempname() * ".dbn" - - try - @testset "Different schema types" begin - schemas_to_test = [Schema.MBO, Schema.TRADES, Schema.OHLCV_1S, Schema.STATUS] - - for schema in schemas_to_test - metadata = Metadata( - UInt8(DBN_VERSION), "TEST.DATA", schema, - 1640995200000000000, 1640995260000000000, UInt64(1000), - SType.RAW_SYMBOL, SType.RAW_SYMBOL, - false, ["TEST"], String[], String[], - Tuple{String,String,Int64,Int64}[] - ) - - trade_msg = create_test_trade_msg() - write_dbn(temp_file, metadata, [trade_msg]) - - # Read it back and verify schema - open(temp_file, "r") do f - decoder = DBNDecoder(f) - read_header!(decoder) - @test decoder.metadata.schema == schema - end - end - end - - @testset "Different symbol configurations" begin - # Test with no symbols - metadata_no_symbols = Metadata( - UInt8(DBN_VERSION), "TEST.DATA", Schema.TRADES, - 1640995200000000000, 1640995260000000000, UInt64(1000), - SType.RAW_SYMBOL, SType.RAW_SYMBOL, - false, String[], String[], String[], - Tuple{String,String,Int64,Int64}[] - ) - - trade_msg = create_test_trade_msg() - write_dbn(temp_file, metadata_no_symbols, [trade_msg]) - - records = read_dbn(temp_file) - @test length(records) == 1 - - # Test with many symbols - many_symbols = ["SYM$i" for i in 1:50] - metadata_many_symbols = Metadata( - UInt8(DBN_VERSION), "TEST.DATA", Schema.TRADES, - 1640995200000000000, 1640995260000000000, UInt64(1000), - SType.RAW_SYMBOL, SType.RAW_SYMBOL, - false, many_symbols, String[], String[], - Tuple{String,String,Int64,Int64}[] - ) - - write_dbn(temp_file, metadata_many_symbols, [trade_msg]) - - open(temp_file, "r") do f - decoder = DBNDecoder(f) - read_header!(decoder) - @test length(decoder.metadata.symbols) == 50 - @test decoder.metadata.symbols[1] == "SYM1" - @test decoder.metadata.symbols[50] == "SYM50" - end - end - - finally - safe_rm(temp_file) - end - end +# Phase 4: Basic Read/Write Testing (No Compression) +using Dates + +@testset "Phase 4: Basic Read/Write Testing (No Compression)" begin + + # Helper function to create test metadata + function create_test_metadata() + return Metadata( + UInt8(DBN_VERSION), # version + "TEST.ITCH", # dataset + Schema.TRADES, # schema + 1640995200000000000, # start_ts + 1640995260000000000, # end_ts + UInt64(1000), # limit + SType.RAW_SYMBOL, # stype_in + SType.RAW_SYMBOL, # stype_out + false, # ts_out + ["AAPL", "MSFT"], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings + ) + end + + # Helper function to create test TradeMsg + function create_test_trade_msg() + hd = RecordHeader(10, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) # 10 units = 40 bytes + return TradeMsg( + hd, # hd + 10055000000, # price + 250, # size + Action.TRADE, # action + Side.NONE, # side + 0x02, # flags + 0, # depth + 1640995200000000001, # ts_recv + 1000, # ts_in_delta + 12346 # sequence + ) + end + + @testset "Minimal DBN file writer test" begin + temp_file = tempname() * ".dbn" + + try + @testset "Write DBN header" begin + metadata = create_test_metadata() + + # Test writing header + open(temp_file, "w") do f + encoder = DBNEncoder(f, metadata) + write_header(encoder) + finalize_encoder(encoder) + end + + @test isfile(temp_file) + @test filesize(temp_file) > 0 + end + + @testset "Write a single TradeMsg record" begin + metadata = create_test_metadata() + trade_msg = create_test_trade_msg() + + # Write header and single record + write_dbn(temp_file, metadata, [trade_msg]) + + @test isfile(temp_file) + @test filesize(temp_file) > 100 # Should be more than just header + end + + @testset "Verify file is created" begin + @test isfile(temp_file) + + # Check that file has proper DBN magic bytes + open(temp_file, "r") do f + magic = read(f, 3) + @test magic == b"DBN" + + version = read(f, UInt8) + @test version == DBN_VERSION + end + end + + finally + # Clean up + safe_rm(temp_file) + end + end + + @testset "Minimal DBN file reader test" begin + temp_file = tempname() * ".dbn" + + try + # First write a test file + metadata = create_test_metadata() + trade_msg = create_test_trade_msg() + write_dbn(temp_file, metadata, [trade_msg]) + + @testset "Read the file created above" begin + records = read_dbn(temp_file) + @test length(records) == 1 + @test records[1] isa TradeMsg + end + + @testset "Verify header is parsed correctly" begin + open(temp_file, "r") do f + decoder = DBNDecoder(f) + read_header!(decoder) + + @test decoder.metadata !== nothing + @test decoder.metadata.version == DBN_VERSION + @test decoder.metadata.dataset == "TEST.ITCH" + @test decoder.metadata.schema == Schema.TRADES + @test length(decoder.metadata.symbols) == 2 + @test decoder.metadata.symbols[1] == "AAPL" + @test decoder.metadata.symbols[2] == "MSFT" + end + end + + @testset "Verify record is read correctly" begin + records = read_dbn(temp_file) + trade_record = records[1] + original_trade = create_test_trade_msg() + + @test DBN.record_length_bytes(trade_record.hd) == DBN.record_length_bytes(original_trade.hd) + @test trade_record.hd.rtype == original_trade.hd.rtype + @test trade_record.hd.publisher_id == original_trade.hd.publisher_id + @test trade_record.hd.instrument_id == original_trade.hd.instrument_id + @test trade_record.hd.ts_event == original_trade.hd.ts_event + + @test trade_record.price == original_trade.price + @test trade_record.size == original_trade.size + @test trade_record.action == original_trade.action + @test trade_record.side == original_trade.side + @test trade_record.flags == original_trade.flags + @test trade_record.depth == original_trade.depth + @test trade_record.ts_recv == original_trade.ts_recv + @test trade_record.ts_in_delta == original_trade.ts_in_delta + @test trade_record.sequence == original_trade.sequence + end + + finally + safe_rm(temp_file) + end + end + + @testset "Round-trip testing (write then read)" begin + temp_file = tempname() * ".dbn" + + try + @testset "Write multiple record types" begin + metadata = create_test_metadata() + + # Create different message types + hd1 = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000) + trade_msg = TradeMsg(hd1, 10055000000, 250, Action.TRADE, Side.NONE, 0x02, 0, 1640995200000000001, 1000, 12346) + + hd2 = RecordHeader(50, RType.MBO_MSG, 1, 12345, 1640995200000000002) + mbo_msg = MBOMsg(hd2, 9876543210, 10050000000, 100, 0x01, 1, Action.ADD, Side.BID, 1640995200000000003, 2000, 12347) + + hd3 = RecordHeader(30, RType.OHLCV_1S_MSG, 1, 12345, 1640995200000000004) + ohlcv_msg = OHLCVMsg(hd3, 10050000000, 10070000000, 10040000000, 10065000000, 125000) + + records = [trade_msg, mbo_msg, ohlcv_msg] + write_dbn(temp_file, metadata, records) + + @test isfile(temp_file) + @test filesize(temp_file) > 200 # Should be substantial + end + + @testset "Read them back" begin + records = read_dbn(temp_file) + @test length(records) == 3 + @test records[1] isa TradeMsg + @test records[2] isa MBOMsg + @test records[3] isa OHLCVMsg + end + + @testset "Verify data integrity" begin + records = read_dbn(temp_file) + + # Test TradeMsg integrity + trade_record = records[1] + @test trade_record.hd.rtype == RType.MBP_0_MSG + @test trade_record.price == 10055000000 + @test trade_record.size == 250 + @test trade_record.action == Action.TRADE + @test trade_record.side == Side.NONE + + # Test MBOMsg integrity + mbo_record = records[2] + @test mbo_record.hd.rtype == RType.MBO_MSG + @test mbo_record.order_id == 9876543210 + @test mbo_record.price == 10050000000 + @test mbo_record.size == 100 + @test mbo_record.action == Action.ADD + @test mbo_record.side == Side.BID + + # Test OHLCVMsg integrity + ohlcv_record = records[3] + @test ohlcv_record.hd.rtype == RType.OHLCV_1S_MSG + @test ohlcv_record.open == 10050000000 + @test ohlcv_record.high == 10070000000 + @test ohlcv_record.low == 10040000000 + @test ohlcv_record.close == 10065000000 + @test ohlcv_record.volume == 125000 + end + + finally + safe_rm(temp_file) + end + end + + @testset "DBNEncoder and DBNDecoder direct testing" begin + temp_file = tempname() * ".dbn" + + try + @testset "Direct encoder usage" begin + metadata = create_test_metadata() + trade_msg = create_test_trade_msg() + + open(temp_file, "w") do f + encoder = DBNEncoder(f, metadata) + write_header(encoder) + write_record(encoder, trade_msg) + finalize_encoder(encoder) + end + + @test isfile(temp_file) + @test filesize(temp_file) > 50 + end + + @testset "Direct decoder usage" begin + open(temp_file, "r") do f + decoder = DBNDecoder(f) + read_header!(decoder) + + # Verify metadata was read correctly + @test decoder.metadata.dataset == "TEST.ITCH" + @test decoder.metadata.schema == Schema.TRADES + + # Read the record + record = read_record(decoder) + @test record isa TradeMsg + @test record.price == 10055000000 + + # Try to read another record (should be nothing) + next_record = read_record(decoder) + @test next_record === nothing + end + end + + finally + safe_rm(temp_file) + end + end + + @testset "Error handling and edge cases" begin + @testset "Writing to invalid path" begin + invalid_path = "/invalid/nonexistent/path/test.dbn" + metadata = create_test_metadata() + trade_msg = create_test_trade_msg() + + @test_throws Exception write_dbn(invalid_path, metadata, [trade_msg]) + end + + @testset "Reading nonexistent file" begin + nonexistent_file = "nonexistent_file.dbn" + @test_throws Exception read_dbn(nonexistent_file) + end + + @testset "Reading invalid DBN file" begin + temp_file = tempname() * ".dbn" + + try + # Write invalid magic bytes + open(temp_file, "w") do f + write(f, b"INVALID") + end + + @test_throws Exception read_dbn(temp_file) + + finally + safe_rm(temp_file) + end + end + + @testset "Empty file handling" begin + temp_file = tempname() * ".dbn" + + try + # Create empty file + touch(temp_file) + @test_throws Exception read_dbn(temp_file) + + finally + safe_rm(temp_file) + end + end + end + + @testset "Different metadata configurations" begin + temp_file = tempname() * ".dbn" + + try + @testset "Different schema types" begin + schemas_to_test = [Schema.MBO, Schema.TRADES, Schema.OHLCV_1S, Schema.STATUS] + + for schema in schemas_to_test + metadata = Metadata( + UInt8(DBN_VERSION), "TEST.DATA", schema, + 1640995200000000000, 1640995260000000000, UInt64(1000), + SType.RAW_SYMBOL, SType.RAW_SYMBOL, + false, ["TEST"], String[], String[], + Tuple{String,String,Int64,Int64}[] + ) + + trade_msg = create_test_trade_msg() + write_dbn(temp_file, metadata, [trade_msg]) + + # Read it back and verify schema + open(temp_file, "r") do f + decoder = DBNDecoder(f) + read_header!(decoder) + @test decoder.metadata.schema == schema + end + end + end + + @testset "Different symbol configurations" begin + # Test with no symbols + metadata_no_symbols = Metadata( + UInt8(DBN_VERSION), "TEST.DATA", Schema.TRADES, + 1640995200000000000, 1640995260000000000, UInt64(1000), + SType.RAW_SYMBOL, SType.RAW_SYMBOL, + false, String[], String[], String[], + Tuple{String,String,Int64,Int64}[] + ) + + trade_msg = create_test_trade_msg() + write_dbn(temp_file, metadata_no_symbols, [trade_msg]) + + records = read_dbn(temp_file) + @test length(records) == 1 + + # Test with many symbols + many_symbols = ["SYM$i" for i in 1:50] + metadata_many_symbols = Metadata( + UInt8(DBN_VERSION), "TEST.DATA", Schema.TRADES, + 1640995200000000000, 1640995260000000000, UInt64(1000), + SType.RAW_SYMBOL, SType.RAW_SYMBOL, + false, many_symbols, String[], String[], + Tuple{String,String,Int64,Int64}[] + ) + + write_dbn(temp_file, metadata_many_symbols, [trade_msg]) + + open(temp_file, "r") do f + decoder = DBNDecoder(f) + read_header!(decoder) + @test length(decoder.metadata.symbols) == 50 + @test decoder.metadata.symbols[1] == "SYM1" + @test decoder.metadata.symbols[50] == "SYM50" + end + end + + finally + safe_rm(temp_file) + end + end end \ No newline at end of file diff --git a/test/test_phase5.jl b/test/test_phase5.jl index f5aa428d..28b38bed 100644 --- a/test/test_phase5.jl +++ b/test/test_phase5.jl @@ -1,24 +1,24 @@ -# Phase 5: Record Type Read/Write Testing -using Dates - -@testset "Phase 5: Record Type Read/Write Testing" begin - - # Helper function to create test metadata +# Phase 5: Record Type Read/Write Testing +using Dates + +@testset "Phase 5: Record Type Read/Write Testing" begin + + # Helper function to create test metadata function create_test_metadata(schema::Schema.T) return Metadata( - UInt8(DBN_VERSION), # version - "TEST.DATA", # dataset - schema, # schema - 1640995200000000000, # start_ts - 1640995260000000000, # end_ts - UInt64(1000), # limit - SType.RAW_SYMBOL, # stype_in - SType.RAW_SYMBOL, # stype_out - false, # ts_out - ["AAPL", "MSFT"], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings + UInt8(DBN_VERSION), # version + "TEST.DATA", # dataset + schema, # schema + 1640995200000000000, # start_ts + 1640995260000000000, # end_ts + UInt64(1000), # limit + SType.RAW_SYMBOL, # stype_in + SType.RAW_SYMBOL, # stype_out + false, # ts_out + ["AAPL", "MSFT"], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings ) end @@ -34,505 +34,505 @@ using Dates write(io, value) return take!(io) end - - @testset "MBOMsg serialization/deserialization" begin - metadata = create_test_metadata(Schema.MBO) - temp_file = tempname() * ".dbn" - - try - # Create test MBO message - hd = RecordHeader(55, RType.MBO_MSG, 1, 12345, 1640995200000000000) - original_msg = MBOMsg( - hd, # hd - 9876543210987654, # order_id - 10055000000, # price - 500, # size - 0x01, # flags - 2, # channel_id - Action.ADD, # action - Side.BID, # side - 1640995200000000001, # ts_recv - 1500, # ts_in_delta - 98765 # sequence - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa MBOMsg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.MBO_MSG - @test read_msg.order_id == original_msg.order_id - @test read_msg.price == original_msg.price - @test read_msg.size == original_msg.size - @test read_msg.flags == original_msg.flags - @test read_msg.channel_id == original_msg.channel_id - @test read_msg.action == original_msg.action - @test read_msg.side == original_msg.side - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.ts_in_delta == original_msg.ts_in_delta - @test read_msg.sequence == original_msg.sequence - - finally - safe_rm(temp_file) - end - end - - @testset "TradeMsg serialization/deserialization" begin - metadata = create_test_metadata(Schema.TRADES) - temp_file = tempname() * ".dbn" - - try - # Create test Trade message - hd = RecordHeader(45, RType.MBP_0_MSG, 2, 54321, 1640995200000000100) - original_msg = TradeMsg( - hd, # hd - 10075000000, # price - 1000, # size - Action.TRADE, # action - Side.NONE, # side - 0x04, # flags - 1, # depth - 1640995200000000101, # ts_recv - 2000, # ts_in_delta - 87654 # sequence - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa TradeMsg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.MBP_0_MSG - @test read_msg.price == original_msg.price - @test read_msg.size == original_msg.size - @test read_msg.action == original_msg.action - @test read_msg.side == original_msg.side - @test read_msg.flags == original_msg.flags - @test read_msg.depth == original_msg.depth - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.ts_in_delta == original_msg.ts_in_delta - @test read_msg.sequence == original_msg.sequence - - finally - safe_rm(temp_file) - end - end - - @testset "MBP1Msg serialization/deserialization" begin - metadata = create_test_metadata(Schema.MBP_1) - temp_file = tempname() * ".dbn" - - try - # Create test MBP1 message - hd = RecordHeader(65, RType.MBP_1_MSG, 1, 11111, 1640995200000000200) - levels = BidAskPair(10050000000, 10060000000, 200, 300, 8, 5) - - original_msg = MBP1Msg( - hd, # hd - 10055000000, # price - 250, # size - Action.MODIFY, # action - Side.ASK, # side - 0x08, # flags - 2, # depth - 1640995200000000201, # ts_recv - 3000, # ts_in_delta - 76543, # sequence - levels # levels - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa MBP1Msg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.MBP_1_MSG - @test read_msg.price == original_msg.price - @test read_msg.size == original_msg.size - @test read_msg.action == original_msg.action - @test read_msg.side == original_msg.side - @test read_msg.flags == original_msg.flags - @test read_msg.depth == original_msg.depth - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.ts_in_delta == original_msg.ts_in_delta - @test read_msg.sequence == original_msg.sequence - - # Test BidAskPair levels - @test read_msg.levels.bid_px == levels.bid_px - @test read_msg.levels.ask_px == levels.ask_px - @test read_msg.levels.bid_sz == levels.bid_sz - @test read_msg.levels.ask_sz == levels.ask_sz - @test read_msg.levels.bid_ct == levels.bid_ct - @test read_msg.levels.ask_ct == levels.ask_ct - - finally - safe_rm(temp_file) - end - end - - @testset "MBP10Msg serialization/deserialization" begin - metadata = create_test_metadata(Schema.MBP_10) - temp_file = tempname() * ".dbn" - - try - # Create test MBP10 message with 10 levels - hd = RecordHeader(250, RType.MBP_10_MSG, 1, 22222, 1640995200000000300) - levels = ntuple(10) do i - BidAskPair( - 10050000000 - (i-1)*1000000, # bid decreasing - 10060000000 + (i-1)*1000000, # ask increasing - 100 + i*50, # bid size - 150 + i*60, # ask size - 3 + i, # bid count - 2 + i # ask count - ) - end - - original_msg = MBP10Msg( - hd, # hd - 10055000000, # price - 400, # size - Action.CLEAR, # action - Side.BID, # side - 0x10, # flags - 5, # depth - 1640995200000000301, # ts_recv - 4000, # ts_in_delta - 65432, # sequence - levels # levels - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa MBP10Msg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.MBP_10_MSG - @test read_msg.price == original_msg.price - @test read_msg.size == original_msg.size - @test read_msg.action == original_msg.action - @test read_msg.side == original_msg.side - @test read_msg.flags == original_msg.flags - @test read_msg.depth == original_msg.depth - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.ts_in_delta == original_msg.ts_in_delta - @test read_msg.sequence == original_msg.sequence - - # Test all 10 levels - @test length(read_msg.levels) == 10 - for i in 1:10 - @test read_msg.levels[i].bid_px == levels[i].bid_px - @test read_msg.levels[i].ask_px == levels[i].ask_px - @test read_msg.levels[i].bid_sz == levels[i].bid_sz - @test read_msg.levels[i].ask_sz == levels[i].ask_sz - @test read_msg.levels[i].bid_ct == levels[i].bid_ct - @test read_msg.levels[i].ask_ct == levels[i].ask_ct - end - - finally - safe_rm(temp_file) - end - end - - @testset "OHLCVMsg serialization/deserialization" begin - metadata = create_test_metadata(Schema.OHLCV_1S) - temp_file = tempname() * ".dbn" - - try - # Create test OHLCV message - hd = RecordHeader(14, RType.OHLCV_1S_MSG, 3, 33333, 1640995200000000400) - original_msg = OHLCVMsg( - hd, # hd - 10040000000, # open - 10080000000, # high - 10030000000, # low - 10070000000, # close - 250000 # volume - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa OHLCVMsg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.OHLCV_1S_MSG - @test read_msg.open == original_msg.open - @test read_msg.high == original_msg.high - @test read_msg.low == original_msg.low - @test read_msg.close == original_msg.close - @test read_msg.volume == original_msg.volume - - finally - safe_rm(temp_file) - end - end - - @testset "StatusMsg serialization/deserialization" begin - metadata = create_test_metadata(Schema.STATUS) - temp_file = tempname() * ".dbn" - - try - # Create test Status message - hd = RecordHeader(10, RType.STATUS_MSG, 1, 44444, 1640995200000000500) - original_msg = StatusMsg( - hd, # hd - 1640995200000000501, # ts_recv - 5, # action - 2, # reason - 10, # trading_event - UInt8('Y'), # is_trading (c_char) - UInt8('N'), # is_quoting (c_char) - UInt8('Y') # is_short_sell_restricted (c_char) - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa StatusMsg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.STATUS_MSG - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.action == original_msg.action - @test read_msg.reason == original_msg.reason - @test read_msg.trading_event == original_msg.trading_event - @test read_msg.is_trading == original_msg.is_trading - @test read_msg.is_quoting == original_msg.is_quoting - @test read_msg.is_short_sell_restricted == original_msg.is_short_sell_restricted - - finally - safe_rm(temp_file) - end - end - - @testset "ImbalanceMsg serialization/deserialization" begin - metadata = create_test_metadata(Schema.IMBALANCE) - temp_file = tempname() * ".dbn" - - try - # Create test Imbalance message with full DBN v3 structure - hd = RecordHeader(28, RType.IMBALANCE_MSG, 2, 55555, 1640995200000000600) - original_msg = ImbalanceMsg( - hd, # hd - 1640995200000000601, # ts_recv - 10065000000, # ref_price - 1640995230000000000, # auction_time - 10066000000, # cont_book_clr_price - 10067000000, # auct_interest_clr_price - 10068000000, # ssr_filling_price - 10069000000, # ind_match_price - 10070000000, # upper_collar - 10060000000, # lower_collar - 5000, # paired_qty - 15000, # total_imbalance_qty - 8000, # market_imbalance_qty - 2000, # unpaired_qty - UInt8('O'), # auction_type - Side.ASK, # side - UInt8(1), # auction_status - UInt8(0), # freeze_status - UInt8(0), # num_extensions - UInt8('A'), # unpaired_side - UInt8('N') # significant_imbalance - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa ImbalanceMsg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.IMBALANCE_MSG - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.ref_price == original_msg.ref_price - @test read_msg.auction_time == original_msg.auction_time - @test read_msg.cont_book_clr_price == original_msg.cont_book_clr_price - @test read_msg.auct_interest_clr_price == original_msg.auct_interest_clr_price - @test read_msg.total_imbalance_qty == original_msg.total_imbalance_qty - @test read_msg.side == original_msg.side - - finally - safe_rm(temp_file) - end - end - - @testset "StatMsg serialization/deserialization (DBN v3)" begin - metadata = create_test_metadata(Schema.STATISTICS) - temp_file = tempname() * ".dbn" - - try - # Create test Stat message with v3 features - hd = RecordHeader(20, RType.STAT_MSG, 1, 66666, 1640995200000000700) - original_msg = StatMsg( - hd, # hd - 1640995200000000701, # ts_recv - 1640995200000000000, # ts_ref - 10055000000, # price - 123456789012345678, # quantity (64-bit in v3) - 54321, # sequence - 5500, # ts_in_delta - 8, # stat_type - 3, # channel_id - 1, # update_action - 0x20 # stat_flags - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa StatMsg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.STAT_MSG - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.ts_ref == original_msg.ts_ref - @test read_msg.price == original_msg.price - @test read_msg.quantity == original_msg.quantity # Test 64-bit quantity - @test read_msg.sequence == original_msg.sequence - @test read_msg.ts_in_delta == original_msg.ts_in_delta - @test read_msg.stat_type == original_msg.stat_type - @test read_msg.channel_id == original_msg.channel_id - @test read_msg.update_action == original_msg.update_action - @test read_msg.stat_flags == original_msg.stat_flags - - finally - safe_rm(temp_file) - end - end - - @testset "InstrumentDefMsg serialization/deserialization (DBN v3)" begin - metadata = create_test_metadata(Schema.DEFINITION) - temp_file = tempname() * ".dbn" - - try - # Create test InstrumentDef message with v3 features - hd = RecordHeader(130, RType.INSTRUMENT_DEF_MSG, 1, 77777, 1640995200000000800) - original_msg = InstrumentDefMsg( - hd, # hd - 1640995200000000801, # ts_recv - 1000000, # min_price_increment - 1000000000, # display_factor - 1672531200000000000, # expiration - 1640995200000000000, # activation - 15000000000, # high_limit_price - 5000000000, # low_limit_price - 1000000000, # max_price_variation - 0, # trading_reference_price (v2 only) - 100, # unit_of_measure_qty - 1000000, # min_price_increment_amount - 1000000000, # price_ratio - 5, # inst_attrib_value - 12345, # underlying_id - 987654321098765432, # raw_instrument_id (64-bit in v3) - 0, # market_depth_implied - 10, # market_depth - 2, # market_segment_id - 1000000, # max_trade_vol - 1, # min_lot_size - 100, # min_lot_size_block - 1, # min_lot_size_round_lot - 10, # min_trade_vol - 100, # contract_multiplier - 0, # decay_quantity - 100, # original_contract_size - 0, # trading_reference_date (v2 only) - 10, # appl_id - 2025, # maturity_year - 0, # decay_start_date - 5, # channel_id - "USD", # currency - "USD", # settl_currency - "CS", # secsubtype + + @testset "MBOMsg serialization/deserialization" begin + metadata = create_test_metadata(Schema.MBO) + temp_file = tempname() * ".dbn" + + try + # Create test MBO message + hd = RecordHeader(55, RType.MBO_MSG, 1, 12345, 1640995200000000000) + original_msg = MBOMsg( + hd, # hd + 9876543210987654, # order_id + 10055000000, # price + 500, # size + 0x01, # flags + 2, # channel_id + Action.ADD, # action + Side.BID, # side + 1640995200000000001, # ts_recv + 1500, # ts_in_delta + 98765 # sequence + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa MBOMsg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.MBO_MSG + @test read_msg.order_id == original_msg.order_id + @test read_msg.price == original_msg.price + @test read_msg.size == original_msg.size + @test read_msg.flags == original_msg.flags + @test read_msg.channel_id == original_msg.channel_id + @test read_msg.action == original_msg.action + @test read_msg.side == original_msg.side + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.ts_in_delta == original_msg.ts_in_delta + @test read_msg.sequence == original_msg.sequence + + finally + safe_rm(temp_file) + end + end + + @testset "TradeMsg serialization/deserialization" begin + metadata = create_test_metadata(Schema.TRADES) + temp_file = tempname() * ".dbn" + + try + # Create test Trade message + hd = RecordHeader(45, RType.MBP_0_MSG, 2, 54321, 1640995200000000100) + original_msg = TradeMsg( + hd, # hd + 10075000000, # price + 1000, # size + Action.TRADE, # action + Side.NONE, # side + 0x04, # flags + 1, # depth + 1640995200000000101, # ts_recv + 2000, # ts_in_delta + 87654 # sequence + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa TradeMsg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.MBP_0_MSG + @test read_msg.price == original_msg.price + @test read_msg.size == original_msg.size + @test read_msg.action == original_msg.action + @test read_msg.side == original_msg.side + @test read_msg.flags == original_msg.flags + @test read_msg.depth == original_msg.depth + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.ts_in_delta == original_msg.ts_in_delta + @test read_msg.sequence == original_msg.sequence + + finally + safe_rm(temp_file) + end + end + + @testset "MBP1Msg serialization/deserialization" begin + metadata = create_test_metadata(Schema.MBP_1) + temp_file = tempname() * ".dbn" + + try + # Create test MBP1 message + hd = RecordHeader(65, RType.MBP_1_MSG, 1, 11111, 1640995200000000200) + levels = BidAskPair(10050000000, 10060000000, 200, 300, 8, 5) + + original_msg = MBP1Msg( + hd, # hd + 10055000000, # price + 250, # size + Action.MODIFY, # action + Side.ASK, # side + 0x08, # flags + 2, # depth + 1640995200000000201, # ts_recv + 3000, # ts_in_delta + 76543, # sequence + levels # levels + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa MBP1Msg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.MBP_1_MSG + @test read_msg.price == original_msg.price + @test read_msg.size == original_msg.size + @test read_msg.action == original_msg.action + @test read_msg.side == original_msg.side + @test read_msg.flags == original_msg.flags + @test read_msg.depth == original_msg.depth + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.ts_in_delta == original_msg.ts_in_delta + @test read_msg.sequence == original_msg.sequence + + # Test BidAskPair levels + @test read_msg.levels.bid_px == levels.bid_px + @test read_msg.levels.ask_px == levels.ask_px + @test read_msg.levels.bid_sz == levels.bid_sz + @test read_msg.levels.ask_sz == levels.ask_sz + @test read_msg.levels.bid_ct == levels.bid_ct + @test read_msg.levels.ask_ct == levels.ask_ct + + finally + safe_rm(temp_file) + end + end + + @testset "MBP10Msg serialization/deserialization" begin + metadata = create_test_metadata(Schema.MBP_10) + temp_file = tempname() * ".dbn" + + try + # Create test MBP10 message with 10 levels + hd = RecordHeader(250, RType.MBP_10_MSG, 1, 22222, 1640995200000000300) + levels = ntuple(10) do i + BidAskPair( + 10050000000 - (i-1)*1000000, # bid decreasing + 10060000000 + (i-1)*1000000, # ask increasing + 100 + i*50, # bid size + 150 + i*60, # ask size + 3 + i, # bid count + 2 + i # ask count + ) + end + + original_msg = MBP10Msg( + hd, # hd + 10055000000, # price + 400, # size + Action.CLEAR, # action + Side.BID, # side + 0x10, # flags + 5, # depth + 1640995200000000301, # ts_recv + 4000, # ts_in_delta + 65432, # sequence + levels # levels + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa MBP10Msg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.MBP_10_MSG + @test read_msg.price == original_msg.price + @test read_msg.size == original_msg.size + @test read_msg.action == original_msg.action + @test read_msg.side == original_msg.side + @test read_msg.flags == original_msg.flags + @test read_msg.depth == original_msg.depth + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.ts_in_delta == original_msg.ts_in_delta + @test read_msg.sequence == original_msg.sequence + + # Test all 10 levels + @test length(read_msg.levels) == 10 + for i in 1:10 + @test read_msg.levels[i].bid_px == levels[i].bid_px + @test read_msg.levels[i].ask_px == levels[i].ask_px + @test read_msg.levels[i].bid_sz == levels[i].bid_sz + @test read_msg.levels[i].ask_sz == levels[i].ask_sz + @test read_msg.levels[i].bid_ct == levels[i].bid_ct + @test read_msg.levels[i].ask_ct == levels[i].ask_ct + end + + finally + safe_rm(temp_file) + end + end + + @testset "OHLCVMsg serialization/deserialization" begin + metadata = create_test_metadata(Schema.OHLCV_1S) + temp_file = tempname() * ".dbn" + + try + # Create test OHLCV message + hd = RecordHeader(14, RType.OHLCV_1S_MSG, 3, 33333, 1640995200000000400) + original_msg = OHLCVMsg( + hd, # hd + 10040000000, # open + 10080000000, # high + 10030000000, # low + 10070000000, # close + 250000 # volume + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa OHLCVMsg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.OHLCV_1S_MSG + @test read_msg.open == original_msg.open + @test read_msg.high == original_msg.high + @test read_msg.low == original_msg.low + @test read_msg.close == original_msg.close + @test read_msg.volume == original_msg.volume + + finally + safe_rm(temp_file) + end + end + + @testset "StatusMsg serialization/deserialization" begin + metadata = create_test_metadata(Schema.STATUS) + temp_file = tempname() * ".dbn" + + try + # Create test Status message + hd = RecordHeader(10, RType.STATUS_MSG, 1, 44444, 1640995200000000500) + original_msg = StatusMsg( + hd, # hd + 1640995200000000501, # ts_recv + 5, # action + 2, # reason + 10, # trading_event + UInt8('Y'), # is_trading (c_char) + UInt8('N'), # is_quoting (c_char) + UInt8('Y') # is_short_sell_restricted (c_char) + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa StatusMsg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.STATUS_MSG + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.action == original_msg.action + @test read_msg.reason == original_msg.reason + @test read_msg.trading_event == original_msg.trading_event + @test read_msg.is_trading == original_msg.is_trading + @test read_msg.is_quoting == original_msg.is_quoting + @test read_msg.is_short_sell_restricted == original_msg.is_short_sell_restricted + + finally + safe_rm(temp_file) + end + end + + @testset "ImbalanceMsg serialization/deserialization" begin + metadata = create_test_metadata(Schema.IMBALANCE) + temp_file = tempname() * ".dbn" + + try + # Create test Imbalance message with full DBN v3 structure + hd = RecordHeader(28, RType.IMBALANCE_MSG, 2, 55555, 1640995200000000600) + original_msg = ImbalanceMsg( + hd, # hd + 1640995200000000601, # ts_recv + 10065000000, # ref_price + 1640995230000000000, # auction_time + 10066000000, # cont_book_clr_price + 10067000000, # auct_interest_clr_price + 10068000000, # ssr_filling_price + 10069000000, # ind_match_price + 10070000000, # upper_collar + 10060000000, # lower_collar + 5000, # paired_qty + 15000, # total_imbalance_qty + 8000, # market_imbalance_qty + 2000, # unpaired_qty + UInt8('O'), # auction_type + Side.ASK, # side + UInt8(1), # auction_status + UInt8(0), # freeze_status + UInt8(0), # num_extensions + UInt8('A'), # unpaired_side + UInt8('N') # significant_imbalance + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa ImbalanceMsg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.IMBALANCE_MSG + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.ref_price == original_msg.ref_price + @test read_msg.auction_time == original_msg.auction_time + @test read_msg.cont_book_clr_price == original_msg.cont_book_clr_price + @test read_msg.auct_interest_clr_price == original_msg.auct_interest_clr_price + @test read_msg.total_imbalance_qty == original_msg.total_imbalance_qty + @test read_msg.side == original_msg.side + + finally + safe_rm(temp_file) + end + end + + @testset "StatMsg serialization/deserialization (DBN v3)" begin + metadata = create_test_metadata(Schema.STATISTICS) + temp_file = tempname() * ".dbn" + + try + # Create test Stat message with v3 features + hd = RecordHeader(20, RType.STAT_MSG, 1, 66666, 1640995200000000700) + original_msg = StatMsg( + hd, # hd + 1640995200000000701, # ts_recv + 1640995200000000000, # ts_ref + 10055000000, # price + 123456789012345678, # quantity (64-bit in v3) + 54321, # sequence + 5500, # ts_in_delta + 8, # stat_type + 3, # channel_id + 1, # update_action + 0x20 # stat_flags + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa StatMsg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.STAT_MSG + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.ts_ref == original_msg.ts_ref + @test read_msg.price == original_msg.price + @test read_msg.quantity == original_msg.quantity # Test 64-bit quantity + @test read_msg.sequence == original_msg.sequence + @test read_msg.ts_in_delta == original_msg.ts_in_delta + @test read_msg.stat_type == original_msg.stat_type + @test read_msg.channel_id == original_msg.channel_id + @test read_msg.update_action == original_msg.update_action + @test read_msg.stat_flags == original_msg.stat_flags + + finally + safe_rm(temp_file) + end + end + + @testset "InstrumentDefMsg serialization/deserialization (DBN v3)" begin + metadata = create_test_metadata(Schema.DEFINITION) + temp_file = tempname() * ".dbn" + + try + # Create test InstrumentDef message with v3 features + hd = RecordHeader(130, RType.INSTRUMENT_DEF_MSG, 1, 77777, 1640995200000000800) + original_msg = InstrumentDefMsg( + hd, # hd + 1640995200000000801, # ts_recv + 1000000, # min_price_increment + 1000000000, # display_factor + 1672531200000000000, # expiration + 1640995200000000000, # activation + 15000000000, # high_limit_price + 5000000000, # low_limit_price + 1000000000, # max_price_variation + 0, # trading_reference_price (v2 only) + 100, # unit_of_measure_qty + 1000000, # min_price_increment_amount + 1000000000, # price_ratio + 5, # inst_attrib_value + 12345, # underlying_id + 987654321098765432, # raw_instrument_id (64-bit in v3) + 0, # market_depth_implied + 10, # market_depth + 2, # market_segment_id + 1000000, # max_trade_vol + 1, # min_lot_size + 100, # min_lot_size_block + 1, # min_lot_size_round_lot + 10, # min_trade_vol + 100, # contract_multiplier + 0, # decay_quantity + 100, # original_contract_size + 0, # trading_reference_date (v2 only) + 10, # appl_id + 2025, # maturity_year + 0, # decay_start_date + 5, # channel_id + "USD", # currency + "USD", # settl_currency + "CS", # secsubtype "TSLA240119C00100000.EXTRA", # raw_symbol longer than DBN v1 length - "AUTO", # group - "XNAS", # exchange - "TSLA.NASDAQ", # asset (11 bytes in v3) - "ESTVPS", # cfi - "CS", # security_type - "Shares", # unit_of_measure - "", # underlying - "", # strike_price_currency - InstrumentClass.STOCK, # instrument_class - 0, # strike_price - 'P', # match_algorithm - 0, # md_security_trading_status (v2 only) - 2, # main_fraction - 0, # price_display_format - 0, # settl_price_type (v2 only) - 0, # sub_fraction - 0, # underlying_product - 'A', # security_update_action - 0, # maturity_month - 0, # maturity_day - 0, # maturity_week - false, # user_defined_instrument - 0, # contract_multiplier_unit - 0, # flow_schedule_type - 0, # tick_rule - # New strategy leg fields in DBN v3 - 2, # leg_count - 1, # leg_index - 88888, # leg_instrument_id + "AUTO", # group + "XNAS", # exchange + "TSLA.NASDAQ", # asset (11 bytes in v3) + "ESTVPS", # cfi + "CS", # security_type + "Shares", # unit_of_measure + "", # underlying + "", # strike_price_currency + InstrumentClass.STOCK, # instrument_class + 0, # strike_price + 'P', # match_algorithm + 0, # md_security_trading_status (v2 only) + 2, # main_fraction + 0, # price_display_format + 0, # settl_price_type (v2 only) + 0, # sub_fraction + 0, # underlying_product + 'A', # security_update_action + 0, # maturity_month + 0, # maturity_day + 0, # maturity_week + false, # user_defined_instrument + 0, # contract_multiplier_unit + 0, # flow_schedule_type + 0, # tick_rule + # New strategy leg fields in DBN v3 + 2, # leg_count + 1, # leg_index + 88888, # leg_instrument_id "LEG-SYMBOL-LONGER-THAN-20", # leg_raw_symbol uses DBN v2/v3 length - Side.BID, # leg_side - 99999, # leg_underlying_id + Side.BID, # leg_side + 99999, # leg_underlying_id InstrumentClass.CALL, # leg_instrument_class - 1, # leg_ratio_qty_numerator - 2, # leg_ratio_qty_denominator - 3, # leg_ratio_price_numerator - 4, # leg_ratio_price_denominator - 10025000000, # leg_price - 500000000 # leg_delta - ) - - # Write and read back - write_dbn(temp_file, metadata, [original_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa InstrumentDefMsg - - read_msg = records[1] - @test read_msg.hd.rtype == RType.INSTRUMENT_DEF_MSG - @test read_msg.ts_recv == original_msg.ts_recv - @test read_msg.raw_instrument_id == original_msg.raw_instrument_id # Test 64-bit field - @test read_msg.currency == original_msg.currency - @test read_msg.raw_symbol == original_msg.raw_symbol - @test read_msg.asset == original_msg.asset # Test expanded 11-byte field - @test read_msg.instrument_class == original_msg.instrument_class - - # Test new v3 strategy leg fields - @test read_msg.leg_count == original_msg.leg_count - @test read_msg.leg_index == original_msg.leg_index - @test read_msg.leg_instrument_id == original_msg.leg_instrument_id - @test read_msg.leg_raw_symbol == original_msg.leg_raw_symbol - @test read_msg.leg_side == original_msg.leg_side - @test read_msg.leg_underlying_id == original_msg.leg_underlying_id - @test read_msg.leg_instrument_class == original_msg.leg_instrument_class - @test read_msg.leg_ratio_qty_numerator == original_msg.leg_ratio_qty_numerator - @test read_msg.leg_ratio_qty_denominator == original_msg.leg_ratio_qty_denominator + 1, # leg_ratio_qty_numerator + 2, # leg_ratio_qty_denominator + 3, # leg_ratio_price_numerator + 4, # leg_ratio_price_denominator + 10025000000, # leg_price + 500000000 # leg_delta + ) + + # Write and read back + write_dbn(temp_file, metadata, [original_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa InstrumentDefMsg + + read_msg = records[1] + @test read_msg.hd.rtype == RType.INSTRUMENT_DEF_MSG + @test read_msg.ts_recv == original_msg.ts_recv + @test read_msg.raw_instrument_id == original_msg.raw_instrument_id # Test 64-bit field + @test read_msg.currency == original_msg.currency + @test read_msg.raw_symbol == original_msg.raw_symbol + @test read_msg.asset == original_msg.asset # Test expanded 11-byte field + @test read_msg.instrument_class == original_msg.instrument_class + + # Test new v3 strategy leg fields + @test read_msg.leg_count == original_msg.leg_count + @test read_msg.leg_index == original_msg.leg_index + @test read_msg.leg_instrument_id == original_msg.leg_instrument_id + @test read_msg.leg_raw_symbol == original_msg.leg_raw_symbol + @test read_msg.leg_side == original_msg.leg_side + @test read_msg.leg_underlying_id == original_msg.leg_underlying_id + @test read_msg.leg_instrument_class == original_msg.leg_instrument_class + @test read_msg.leg_ratio_qty_numerator == original_msg.leg_ratio_qty_numerator + @test read_msg.leg_ratio_qty_denominator == original_msg.leg_ratio_qty_denominator @test read_msg.leg_ratio_price_numerator == original_msg.leg_ratio_price_numerator @test read_msg.leg_ratio_price_denominator == original_msg.leg_ratio_price_denominator @test read_msg.leg_price == original_msg.leg_price @@ -552,1059 +552,1059 @@ using Dates finally safe_rm(temp_file) end - end - - @testset "Mixed record types in single file" begin - metadata = create_test_metadata(Schema.MBO) # Use MBO schema for mixed content - temp_file = tempname() * ".dbn" - - try - # Create multiple different message types - hd1 = RecordHeader(55, RType.MBO_MSG, 1, 11111, 1640995200000000000) - mbo_msg = MBOMsg(hd1, 1111111111, 10050000000, 100, 0x01, 1, Action.ADD, Side.BID, 1640995200000000001, 1000, 1001) - - hd2 = RecordHeader(45, RType.MBP_0_MSG, 1, 11111, 1640995200000000002) - trade_msg = TradeMsg(hd2, 10055000000, 200, Action.TRADE, Side.NONE, 0x02, 0, 1640995200000000003, 2000, 1002) - - hd3 = RecordHeader(14, RType.OHLCV_1S_MSG, 1, 11111, 1640995200000000004) - ohlcv_msg = OHLCVMsg(hd3, 10050000000, 10060000000, 10040000000, 10055000000, 50000) - - records_to_write = [mbo_msg, trade_msg, ohlcv_msg] - - # Write and read back - write_dbn(temp_file, metadata, records_to_write) - read_records = read_dbn(temp_file) - - @test length(read_records) == 3 - @test read_records[1] isa MBOMsg - @test read_records[2] isa TradeMsg - @test read_records[3] isa OHLCVMsg - - # Verify record order and basic data - @test read_records[1].order_id == mbo_msg.order_id - @test read_records[2].price == trade_msg.price - @test read_records[3].volume == ohlcv_msg.volume - - finally - safe_rm(temp_file) - end - end - - @testset "Large dataset with many records" begin - metadata = create_test_metadata(Schema.TRADES) - temp_file = tempname() * ".dbn" - - try - # Create 1000 trade records - records_to_write = [] - base_ts = 1640995200000000000 - - for i in 1:1000 - hd = RecordHeader(45, RType.MBP_0_MSG, 1, 12345, base_ts + i * 1000000) - trade_msg = TradeMsg( - hd, - 10050000000 + i * 1000, # varying price - 100 + i, # varying size - Action.TRADE, - Side.NONE, - 0x01, - 0, - base_ts + i * 1000000 + 500000, - 1000 + i, - i - ) - push!(records_to_write, trade_msg) - end - - # Write and read back - write_dbn(temp_file, metadata, records_to_write) - read_records = read_dbn(temp_file) - - @test length(read_records) == 1000 - - # Verify first and last records - @test read_records[1].price == 10050001000 - @test read_records[1].size == 101 - @test read_records[1].sequence == 1 - - @test read_records[1000].price == 10051000000 - @test read_records[1000].size == 1100 - @test read_records[1000].sequence == 1000 - - # Verify all records are TradeMsg - @test all(r isa TradeMsg for r in read_records) - - finally - safe_rm(temp_file) - end - end - - @testset "MBO v3 file reading" begin - # Test reading actual MBO v3 files from test data - @testset "Uncompressed MBO v3" begin - file = joinpath(@__DIR__, "data", "test_data.mbo.v3.dbn") - if isfile(file) - records = read_dbn(file) - @test length(records) == 2 - @test all(r isa MBOMsg for r in records) - - # First record - r1 = records[1] - @test r1.order_id == 647784973705 # official value; pre-0.1.7 this held the price bytes - @test r1.price == 3722750000000 - @test r1.ts_recv == 1609160400000704060 - @test r1.action == Action.CANCEL - @test r1.side == Side.ASK - @test r1.size == 1 - @test r1.hd.publisher_id == 1 - @test r1.hd.instrument_id == 5482 - - # Second record - r2 = records[2] - @test r2.order_id == 647784973631 - @test r2.price == 3723000000000 - @test r2.action == Action.CANCEL - @test r2.side == Side.ASK - @test r2.sequence == r1.sequence + 1 - end - end - - @testset "Compressed MBO v3" begin - file = joinpath(@__DIR__, "data", "test_data.mbo.v3.dbn.zst") - if isfile(file) - records = read_dbn(file) - @test length(records) == 2 - @test all(r isa MBOMsg for r in records) - - # Should match uncompressed data - r1 = records[1] - @test r1.order_id == 647784973705 # official value; pre-0.1.7 this held the price bytes - @test r1.price == 3722750000000 - @test r1.ts_recv == 1609160400000704060 - @test r1.action == Action.CANCEL - @test r1.side == Side.ASK - end - end - end - - @testset "OHLCV message tests" begin - @testset "OHLCV structure and reading" begin - # Test all OHLCV cadences - test_files = [ - ("test_data.ohlcv-1s.dbn", RType.OHLCV_1S_MSG, Schema.OHLCV_1S), - ("test_data.ohlcv-1m.dbn", RType.OHLCV_1M_MSG, Schema.OHLCV_1M), - ("test_data.ohlcv-1h.dbn", RType.OHLCV_1H_MSG, Schema.OHLCV_1H), - ("test_data.ohlcv-1d.dbn", RType.OHLCV_1D_MSG, Schema.OHLCV_1D), - ] - - for (filename, expected_rtype, expected_schema) in test_files - file = joinpath(@__DIR__, "data", filename) - if isfile(file) - @testset "$filename" begin - records = read_dbn(file) - - if length(records) > 0 - @test all(r isa OHLCVMsg for r in records) - @test all(r.hd.rtype == expected_rtype for r in records) - - # Check first record has valid OHLCV data - r = records[1] - @test r.high >= r.low - @test r.open >= r.low && r.open <= r.high - @test r.close >= r.low && r.close <= r.high - @test r.volume >= 0 - end - end - end - end - end - - @testset "OHLCV v2 vs v3 compatibility" begin - v2_file = joinpath(@__DIR__, "data", "test_data.ohlcv-1s.dbn") - v3_file = joinpath(@__DIR__, "data", "test_data.ohlcv-1s.v3.dbn.zst") - - if isfile(v2_file) && isfile(v3_file) - v2_records = read_dbn(v2_file) - v3_records = read_dbn(v3_file) - - @test length(v2_records) == length(v3_records) - - if length(v2_records) > 0 - # Compare first record - r2 = v2_records[1] - r3 = v3_records[1] - - @test r2.open == r3.open - @test r2.high == r3.high - @test r2.low == r3.low - @test r2.close == r3.close - @test r2.volume == r3.volume - end - end - end - - @testset "OHLCV write and read back" begin - metadata = create_test_metadata(Schema.OHLCV_1S) - temp_file = tempname() * ".dbn" - - try - # Create test OHLCV messages - hd1 = RecordHeader(14, RType.OHLCV_1S_MSG, 1, 5482, 1609160400000000000) - ohlcv1 = OHLCVMsg(hd1, 100000000000, 105000000000, 99000000000, 102000000000, 1500) - - hd2 = RecordHeader(14, RType.OHLCV_1S_MSG, 1, 5482, 1609160401000000000) - ohlcv2 = OHLCVMsg(hd2, 102000000000, 103000000000, 101000000000, 101500000000, 800) - - # Write and read back - write_dbn(temp_file, metadata, [ohlcv1, ohlcv2]) - records = read_dbn(temp_file) - - @test length(records) == 2 - @test all(r isa OHLCVMsg for r in records) - - # Verify data integrity - @test records[1].open == 100000000000 - @test records[1].high == 105000000000 - @test records[1].low == 99000000000 - @test records[1].close == 102000000000 - @test records[1].volume == 1500 - - @test records[2].open == 102000000000 - @test records[2].close == 101500000000 - @test records[2].volume == 800 - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - end - - @testset "MBP message tests" begin - @testset "MBP-1 structure and reading" begin - # Test MBP-1 files - test_files = [ - ("test_data.mbp-1.dbn", RType.MBP_1_MSG, Schema.MBP_1), - ("test_data.mbp-1.v3.dbn.zst", RType.MBP_1_MSG, Schema.MBP_1), - ] - - for (filename, expected_rtype, expected_schema) in test_files - file = joinpath(@__DIR__, "data", filename) - if isfile(file) - @testset "$filename" begin - records = read_dbn(file) - - if length(records) > 0 - @test all(r isa MBP1Msg for r in records) - @test all(r.hd.rtype == expected_rtype for r in records) - - # Check first record has valid MBP data - r = records[1] - @test r.price > 0 - @test r.size > 0 - @test r.action in [Action.ADD, Action.MODIFY, Action.CANCEL, Action.CLEAR, Action.TRADE] - @test r.side in [Side.BID, Side.ASK, Side.NONE] - - # Check BidAskPair structure (32 bytes total) - level = r.levels - @test typeof(level) == BidAskPair - @test level.bid_px >= 0 - @test level.ask_px >= 0 - @test level.bid_sz >= 0 - @test level.ask_sz >= 0 - @test level.bid_ct >= 0 - @test level.ask_ct >= 0 - - # Verify record size matches length field - expected_size = DBN.record_length_bytes(r.hd) - @test expected_size == 80 # 16 (header) + 32 (MBP data) + 32 (BidAskPair) - end - end - end - end - end - - @testset "MBP-10 structure and reading" begin - # Test MBP-10 files - test_files = [ - ("test_data.mbp-10.dbn", RType.MBP_10_MSG, Schema.MBP_10), - ("test_data.mbp-10.v3.dbn.zst", RType.MBP_10_MSG, Schema.MBP_10), - ] - - for (filename, expected_rtype, expected_schema) in test_files - file = joinpath(@__DIR__, "data", filename) - if isfile(file) - @testset "$filename" begin - records = read_dbn(file) - - if length(records) > 0 - @test all(r isa MBP10Msg for r in records) - @test all(r.hd.rtype == expected_rtype for r in records) - - # Check first record has valid MBP data - r = records[1] - @test r.price > 0 - @test r.size > 0 - @test r.action in [Action.ADD, Action.MODIFY, Action.CANCEL, Action.CLEAR, Action.TRADE] - @test r.side in [Side.BID, Side.ASK, Side.NONE] - - # Check all 10 BidAskPair levels - @test length(r.levels) == 10 - for (i, level) in enumerate(r.levels) - @test typeof(level) == BidAskPair - @test level.bid_px >= 0 - @test level.ask_px >= 0 - @test level.bid_sz >= 0 - @test level.ask_sz >= 0 - @test level.bid_ct >= 0 - @test level.ask_ct >= 0 - end - - # Verify record size matches length field - expected_size = DBN.record_length_bytes(r.hd) - @test expected_size == 368 # 16 (header) + 32 (MBP data) + 320 (10 × 32-byte BidAskPairs) - end - end - end - end - end - - @testset "MBP v2 vs v3 compatibility" begin - # MBP-1 comparison - v2_file = joinpath(@__DIR__, "data", "test_data.mbp-1.dbn") - v3_file = joinpath(@__DIR__, "data", "test_data.mbp-1.v3.dbn.zst") - - if isfile(v2_file) && isfile(v3_file) - @testset "MBP-1 v2 vs v3" begin - v2_records = read_dbn(v2_file) - v3_records = read_dbn(v3_file) - - @test length(v2_records) == length(v3_records) - - if length(v2_records) > 0 - # Compare first record - r2 = v2_records[1] - r3 = v3_records[1] - - @test r2.price == r3.price - @test r2.size == r3.size - @test r2.action == r3.action - @test r2.side == r3.side - @test r2.levels.bid_px == r3.levels.bid_px - @test r2.levels.ask_px == r3.levels.ask_px - @test r2.levels.bid_sz == r3.levels.bid_sz - @test r2.levels.ask_sz == r3.levels.ask_sz - end - end - end - - # MBP-10 comparison - v2_file = joinpath(@__DIR__, "data", "test_data.mbp-10.dbn") - v3_file = joinpath(@__DIR__, "data", "test_data.mbp-10.v3.dbn.zst") - - if isfile(v2_file) && isfile(v3_file) - @testset "MBP-10 v2 vs v3" begin - v2_records = read_dbn(v2_file) - v3_records = read_dbn(v3_file) - - @test length(v2_records) == length(v3_records) - - if length(v2_records) > 0 - # Compare first record - r2 = v2_records[1] - r3 = v3_records[1] - - @test r2.price == r3.price - @test r2.size == r3.size - @test r2.action == r3.action - @test r2.side == r3.side - - # Compare all 10 levels - for i in 1:10 - @test r2.levels[i].bid_px == r3.levels[i].bid_px - @test r2.levels[i].ask_px == r3.levels[i].ask_px - @test r2.levels[i].bid_sz == r3.levels[i].bid_sz - @test r2.levels[i].ask_sz == r3.levels[i].ask_sz - end - end - end - end - end - - @testset "MBP write and read back" begin - @testset "MBP-1 roundtrip" begin - metadata = create_test_metadata(Schema.MBP_1) - temp_file = tempname() * ".dbn" - - try - # Create test MBP-1 messages - hd1 = RecordHeader(20, RType.MBP_1_MSG, 1, 5482, 1609160400000000000) - level1 = BidAskPair(3720250000000, 3720500000000, 24, 11, 15, 9) - mbp1_1 = MBP1Msg(hd1, 3720500000000, 1, Action.ADD, Side.ASK, 0x80, 0, 1609160400006136329, 17214, 1170362, level1) - - hd2 = RecordHeader(20, RType.MBP_1_MSG, 1, 5482, 1609160400000001000) - level2 = BidAskPair(3720000000000, 3720750000000, 31, 34, 12, 8) - mbp1_2 = MBP1Msg(hd2, 3720750000000, 2, Action.MODIFY, Side.BID, 0x40, 1, 1609160400006136330, 17215, 1170363, level2) - - # Write and read back - write_dbn(temp_file, metadata, [mbp1_1, mbp1_2]) - records = read_dbn(temp_file) - - @test length(records) == 2 - @test all(r isa MBP1Msg for r in records) - - # Verify data integrity - @test records[1].price == 3720500000000 - @test records[1].levels.bid_px == 3720250000000 - @test records[1].levels.ask_px == 3720500000000 - @test records[1].levels.bid_sz == 24 - @test records[1].levels.ask_sz == 11 - - @test records[2].price == 3720750000000 - @test records[2].action == Action.MODIFY - @test records[2].side == Side.BID - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - - @testset "MBP-10 roundtrip" begin - metadata = create_test_metadata(Schema.MBP_10) - temp_file = tempname() * ".dbn" - - try - # Create test MBP-10 message with realistic levels - hd = RecordHeader(92, RType.MBP_10_MSG, 1, 5482, 1609160400000000000) - levels = ntuple(10) do i - bid_px = 3720250000000 - (i-1) * 250000000 # decreasing bids - ask_px = 3720500000000 + (i-1) * 250000000 # increasing asks - bid_sz = UInt32(20 + i * 5) - ask_sz = UInt32(15 + i * 3) - bid_ct = UInt32(10 + i) - ask_ct = UInt32(8 + i) - BidAskPair(bid_px, ask_px, bid_sz, ask_sz, bid_ct, ask_ct) - end - - mbp10 = MBP10Msg(hd, 3722750000000, 1, Action.CANCEL, Side.ASK, 0x80, 9, 1609160400000704060, 22993, 1170352, levels) - - # Write and read back - write_dbn(temp_file, metadata, [mbp10]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa MBP10Msg - - # Verify data integrity - r = records[1] - @test r.price == 3722750000000 - @test r.action == Action.CANCEL - @test r.side == Side.ASK - @test length(r.levels) == 10 - - # Check first and last levels - @test r.levels[1].bid_px == 3720250000000 - @test r.levels[1].ask_px == 3720500000000 - @test r.levels[10].bid_px == 3720250000000 - 9 * 250000000 - @test r.levels[10].ask_px == 3720500000000 + 9 * 250000000 - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - end - end - - @testset "Trades message tests" begin - @testset "Trades structure and reading" begin - # Test Trades files - test_files = [ - ("test_data.trades.dbn", RType.MBP_0_MSG, Schema.TRADES), - ("test_data.trades.v1.dbn.zst", RType.MBP_0_MSG, Schema.TRADES), - ("test_data.trades.v2.dbn.zst", RType.MBP_0_MSG, Schema.TRADES), - ("test_data.trades.v3.dbn.zst", RType.MBP_0_MSG, Schema.TRADES), - ] - - for (filename, expected_rtype, expected_schema) in test_files - file = joinpath(@__DIR__, "data", filename) - if isfile(file) - @testset "$filename" begin - records = read_dbn(file) - - if length(records) > 0 - @test all(r isa TradeMsg for r in records) - @test all(r.hd.rtype == expected_rtype for r in records) - - # Check first record has valid Trade data - r = records[1] - @test r.price > 0 - @test r.size > 0 - @test r.action == Action.TRADE # Trades should always have TRADE action - @test r.side in [Side.BID, Side.ASK, Side.NONE] - @test r.depth == 0 # Trades typically have depth 0 - - # Verify record size matches length field (48 bytes total) - expected_size = DBN.record_length_bytes(r.hd) - @test expected_size == 48 # 16 (header) + 32 (trade data) - end - end - end - end - end - - @testset "Trades v1/v2/v3 compatibility" begin - # Test compatibility across versions - test_pairs = [ - ("test_data.trades.dbn", "test_data.trades.v2.dbn.zst"), - ("test_data.trades.v2.dbn.zst", "test_data.trades.v3.dbn.zst"), - ] - - for (file1, file2) in test_pairs - path1 = joinpath(@__DIR__, "data", file1) - path2 = joinpath(@__DIR__, "data", file2) - - if isfile(path1) && isfile(path2) - @testset "$file1 vs $file2" begin - records1 = read_dbn(path1) - records2 = read_dbn(path2) - - @test length(records1) == length(records2) - - if length(records1) > 0 - # Compare first record - r1 = records1[1] - r2 = records2[1] - - @test r1.price == r2.price - @test r1.size == r2.size - @test r1.action == r2.action - @test r1.side == r2.side - @test r1.flags == r2.flags - @test r1.depth == r2.depth - @test r1.ts_recv == r2.ts_recv - @test r1.ts_in_delta == r2.ts_in_delta - @test r1.sequence == r2.sequence - end - end - end - end - end - - @testset "Trades write and read back" begin - @testset "Single trade roundtrip" begin - metadata = create_test_metadata(Schema.TRADES) - temp_file = tempname() * ".dbn" - - try - # Create test Trade message matching real data format - hd = RecordHeader(12, RType.MBP_0_MSG, 1, 5482, 1609160400098821953) - trade_msg = TradeMsg( - hd, # hd - 3720250000000, # price ($3720.25) - 5, # size - Action.TRADE, # action - Side.ASK, # side - 0x81, # flags (129) - 0, # depth - 1609160400099150057, # ts_recv - 19251, # ts_in_delta - 1170380 # sequence - ) - - # Write and read back - write_dbn(temp_file, metadata, [trade_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa TradeMsg - - # Verify data integrity - r = records[1] - @test r.price == trade_msg.price - @test r.size == trade_msg.size - @test r.action == trade_msg.action - @test r.side == trade_msg.side - @test r.flags == trade_msg.flags - @test r.depth == trade_msg.depth - @test r.ts_recv == trade_msg.ts_recv - @test r.ts_in_delta == trade_msg.ts_in_delta - @test r.sequence == trade_msg.sequence - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - - @testset "Multiple trades roundtrip" begin - metadata = create_test_metadata(Schema.TRADES) - temp_file = tempname() * ".dbn" - - try - # Create multiple trade messages with varying data - trades = [] - base_ts = 1609160400098821953 - base_price = 3720000000000 # $3720.00 - - for i in 1:10 - hd = RecordHeader(12, RType.MBP_0_MSG, 1, 5482, base_ts + i * 1000000) - trade = TradeMsg( - hd, - base_price + i * 250000000, # increment price by $0.25 - 10 + i * 5, # varying size - Action.TRADE, - i % 2 == 0 ? Side.BID : Side.ASK, # alternate sides - UInt8(0x80 + i % 8), # varying flags - 0, # depth always 0 for trades - base_ts + i * 1000000 + 100000, # ts_recv slightly after ts_event - 15000 + i * 1000, # varying ts_in_delta - 1170000 + i # incrementing sequence - ) - push!(trades, trade) - end - - # Write and read back - write_dbn(temp_file, metadata, trades) - records = read_dbn(temp_file) - - @test length(records) == 10 - @test all(r isa TradeMsg for r in records) - - # Verify all data matches - for (orig, read) in zip(trades, records) - @test orig.price == read.price - @test orig.size == read.size - @test orig.action == read.action - @test orig.side == read.side - @test orig.flags == read.flags - @test orig.sequence == read.sequence - end - - # Verify price progression - @test records[1].price == base_price + 250000000 # $3720.25 - @test records[10].price == base_price + 10 * 250000000 # $3722.50 - - # Verify size progression - @test records[1].size == 15 - @test records[10].size == 60 - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - - @testset "Trades with different actions and sides" begin - metadata = create_test_metadata(Schema.TRADES) - temp_file = tempname() * ".dbn" - - try - # Test different combinations of actions and sides - test_cases = [ - (Action.TRADE, Side.BID), - (Action.TRADE, Side.ASK), - (Action.TRADE, Side.NONE), - (Action.FILL, Side.BID), - (Action.FILL, Side.ASK), - ] - - trades = [] - for (i, (action, side)) in enumerate(test_cases) - hd = RecordHeader(12, RType.MBP_0_MSG, 1, 5482, 1609160400000000000 + i * 1000000) - trade = TradeMsg( - hd, - 3720000000000 + i * 1000000000, # varying prices - 100 + i * 10, # varying sizes - action, - side, - UInt8(0x80), # standard flags - 0, # depth - 1609160400000000000 + i * 1000000 + 50000, - 20000, - 1000000 + i - ) - push!(trades, trade) - end - - # Write and read back - write_dbn(temp_file, metadata, trades) - records = read_dbn(temp_file) - - @test length(records) == length(test_cases) - - # Verify each action/side combination - for (i, ((expected_action, expected_side), record)) in enumerate(zip(test_cases, records)) - @test record.action == expected_action - @test record.side == expected_side - @test record.size == 100 + i * 10 - end - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - end - end - - @testset "BBO message tests" begin - @testset "BBO structure and reading" begin - # Test different BBO file types - test_files = [ - ("test_data.tbbo.dbn", RType.MBP_1_MSG, Schema.TBBO), - ("test_data.tbbo.v2.dbn.zst", RType.MBP_1_MSG, Schema.TBBO), - ("test_data.cbbo-1s.dbn", RType.CBBO_1S_MSG, Schema.CBBO_1S), - ("test_data.cbbo-1s.v2.dbn.zst", RType.CBBO_1S_MSG, Schema.CBBO_1S), - ("test_data.bbo-1s.dbn", RType.BBO_1S_MSG, Schema.BBO_1S), - ("test_data.bbo-1s.v2.dbn.zst", RType.BBO_1S_MSG, Schema.BBO_1S), - ("test_data.bbo-1s.v3.dbn.zst", RType.BBO_1S_MSG, Schema.BBO_1S), - ("test_data.bbo-1m.v3.dbn.zst", RType.BBO_1M_MSG, Schema.BBO_1M), - ] - - for (filename, expected_rtype, expected_schema) in test_files - file = joinpath(@__DIR__, "data", filename) - if isfile(file) - @testset "$filename" begin - records = read_dbn(file) - - if length(records) > 0 - # Check record type based on RType - if expected_rtype == RType.MBP_1_MSG - @test all(r isa MBP1Msg for r in records) - elseif expected_rtype == RType.CBBO_1S_MSG - @test all(r isa CBBO1sMsg for r in records) - elseif expected_rtype == RType.BBO_1S_MSG - @test all(r isa BBO1sMsg for r in records) - elseif expected_rtype == RType.BBO_1M_MSG - @test all(r isa BBO1mMsg for r in records) - end - - @test all(r.hd.rtype == expected_rtype for r in records) - - # Check first record has valid BBO data - r = records[1] - @test r.price > 0 - @test r.size > 0 - @test r.side in [Side.BID, Side.ASK, Side.NONE] - - # BBO messages typically use NONE action - if hasproperty(r, :action) - @test r.action in [Action.NONE, Action.TRADE, Action.MODIFY] - end - - # Check BidAskPair levels - if hasproperty(r, :levels) - @test typeof(r.levels) == BidAskPair - @test r.levels.bid_px >= 0 - @test r.levels.ask_px >= 0 - @test r.levels.bid_sz >= 0 - @test r.levels.ask_sz >= 0 - end - - # Verify record size matches length field (80 bytes total) - expected_size = DBN.record_length_bytes(r.hd) - @test expected_size == 80 # 16 (header) + 32 (BBO data) + 32 (BidAskPair) - end - end - end - end - end - - @testset "BBO v2 vs v3 compatibility" begin - # Test BBO compatibility across versions - v2_file = joinpath(@__DIR__, "data", "test_data.bbo-1s.v2.dbn.zst") - v3_file = joinpath(@__DIR__, "data", "test_data.bbo-1s.v3.dbn.zst") - - if isfile(v2_file) && isfile(v3_file) - @testset "BBO-1s v2 vs v3" begin - v2_records = read_dbn(v2_file) - v3_records = read_dbn(v3_file) - - @test length(v2_records) == length(v3_records) - - if length(v2_records) > 0 - # Compare first record - r2 = v2_records[1] - r3 = v3_records[1] - - @test r2.price == r3.price - @test r2.size == r3.size - @test r2.side == r3.side - @test r2.flags == r3.flags - @test r2.ts_recv == r3.ts_recv - @test r2.sequence == r3.sequence - @test r2.levels.bid_px == r3.levels.bid_px - @test r2.levels.ask_px == r3.levels.ask_px - @test r2.levels.bid_sz == r3.levels.bid_sz - @test r2.levels.ask_sz == r3.levels.ask_sz - end - end - end - end - - @testset "BBO write and read back" begin - @testset "CBBO-1s roundtrip" begin - metadata = create_test_metadata(Schema.CBBO_1S) - temp_file = tempname() * ".dbn" - - try - # Create test CBBO-1s message - hd = RecordHeader(20, RType.CBBO_1S_MSG, 1, 5482, 1609113599045849637) - level = BidAskPair(3702250000000, 3702750000000, 18, 13, 12, 8) - cbbo_msg = CBBO1sMsg( - hd, # hd - 3702500000000, # price ($3702.5) - 2, # size - Action.NONE, # action - Side.ASK, # side - 0xa8, # flags (168) - 0, # depth - 1609113600000000000, # ts_recv - 500000000, # ts_in_delta - 145799, # sequence - level # levels - ) - - # Write and read back - write_dbn(temp_file, metadata, [cbbo_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa CBBO1sMsg - - # Verify data integrity - r = records[1] - @test r.price == cbbo_msg.price - @test r.size == cbbo_msg.size - @test r.action == cbbo_msg.action - @test r.side == cbbo_msg.side - @test r.flags == cbbo_msg.flags - @test r.ts_recv == cbbo_msg.ts_recv - @test r.sequence == cbbo_msg.sequence - @test r.levels.bid_px == cbbo_msg.levels.bid_px - @test r.levels.ask_px == cbbo_msg.levels.ask_px - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - - @testset "BBO-1s roundtrip" begin - metadata = create_test_metadata(Schema.BBO_1S) - temp_file = tempname() * ".dbn" - - try - # Create test BBO-1s message - hd = RecordHeader(20, RType.BBO_1S_MSG, 1, 5482, 1609113599045849637) - level = BidAskPair(3702250000000, 3702750000000, 18, 13, 12, 8) - bbo_msg = BBO1sMsg( - hd, # hd - 3702500000000, # price ($3702.5) - 2, # size - Action.NONE, # action - Side.ASK, # side - 0xa8, # flags (168) - 0, # depth - 1609113600000000000, # ts_recv - 500000000, # ts_in_delta - 145799, # sequence - level # levels - ) - - # Write and read back - write_dbn(temp_file, metadata, [bbo_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa BBO1sMsg - - # Verify data integrity - r = records[1] - @test r.price == bbo_msg.price - @test r.size == bbo_msg.size - @test r.action == bbo_msg.action - @test r.side == bbo_msg.side - @test r.flags == bbo_msg.flags - @test r.ts_recv == bbo_msg.ts_recv - @test r.sequence == bbo_msg.sequence - @test r.levels.bid_px == bbo_msg.levels.bid_px - @test r.levels.ask_px == bbo_msg.levels.ask_px - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - - @testset "TBBO roundtrip (MBP1Msg)" begin - metadata = create_test_metadata(Schema.TBBO) - temp_file = tempname() * ".dbn" - - try - # TBBO uses MBP1Msg structure - hd = RecordHeader(20, RType.MBP_1_MSG, 1, 5482, 1609160400098821953) - level = BidAskPair(3720250000000, 3720500000000, 26, 7, 15, 9) - tbbo_msg = MBP1Msg( - hd, # hd - 3720250000000, # price ($3720.25) - 5, # size - Action.TRADE, # action - Side.ASK, # side - 0x81, # flags (129) - 0, # depth - 1609160400099150057, # ts_recv - 19251, # ts_in_delta - 1170380, # sequence - level # levels - ) - - # Write and read back - write_dbn(temp_file, metadata, [tbbo_msg]) - records = read_dbn(temp_file) - - @test length(records) == 1 - @test records[1] isa MBP1Msg - - # Verify data integrity - r = records[1] - @test r.price == tbbo_msg.price - @test r.size == tbbo_msg.size - @test r.action == tbbo_msg.action - @test r.side == tbbo_msg.side - @test r.levels.bid_px == tbbo_msg.levels.bid_px - @test r.levels.ask_px == tbbo_msg.levels.ask_px - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - - @testset "Multiple BBO types in one file" begin - metadata = create_test_metadata(Schema.BBO_1S) - temp_file = tempname() * ".dbn" - - try - # Create multiple BBO messages with different timestamps - bbo_msgs = [] - base_ts = 1609113599000000000 - base_price = 3702000000000 - - for i in 1:5 - hd = RecordHeader(20, RType.BBO_1S_MSG, 1, 5482, base_ts + i * 1000000000) - level = BidAskPair( - base_price + i * 250000000, # bid price - base_price + (i + 1) * 250000000, # ask price - 15 + i * 2, # bid size - 10 + i * 3, # ask size - 8 + i, # bid count - 6 + i # ask count - ) - - bbo_msg = BBO1sMsg( - hd, - base_price + i * 500000000, # price between bid/ask - UInt32(i * 2), # varying size - Action.NONE, - i % 2 == 0 ? Side.BID : Side.ASK, # alternate sides - UInt8(0xa0 + i), # varying flags - UInt8(0), # depth - base_ts + i * 1000000000 + 100000000, - Int32(500000000 + i * 100000000), # varying ts_in_delta - UInt32(145000 + i), # incrementing sequence - level - ) - push!(bbo_msgs, bbo_msg) - end - - # Write and read back - write_dbn(temp_file, metadata, bbo_msgs) - records = read_dbn(temp_file) - - @test length(records) == 5 - @test all(r isa BBO1sMsg for r in records) - - # Verify progression - for (i, (orig, read)) in enumerate(zip(bbo_msgs, records)) - @test orig.price == read.price - @test orig.size == read.size - @test orig.side == read.side - @test orig.sequence == read.sequence - end - - # Check price progression - @test records[1].price == base_price + 500000000 # $3702.50 - @test records[5].price == base_price + 5 * 500000000 # $3705.00 - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - end - end - - @testset "Zstd compression tests" begin - @testset "Auto-detection of compressed files" begin - # Test that we can read compressed files with and without .zst extension - compressed_file = joinpath(@__DIR__, "data", "test_data.mbo.v3.dbn.zst") - - if isfile(compressed_file) - # Copy to a file without .zst extension - temp_file = tempname() * ".dbn" - try - cp(compressed_file, temp_file) - - # Should auto-detect compression by magic bytes - records = read_dbn(temp_file) - @test length(records) == 2 - @test all(r isa MBOMsg for r in records) - - finally - if isfile(temp_file) - safe_rm(temp_file) - end - end - end - end - - @testset "Streaming decompression" begin - # Test that we can stream large compressed files - compressed_file = joinpath(@__DIR__, "data", "test_data.mbp-10.v3.dbn.zst") - - if isfile(compressed_file) - # Use DBNDecoder directly to test streaming - decoder = DBNDecoder(compressed_file) - try - count = 0 - while !eof(decoder.io) - record = read_record(decoder) - if record !== nothing - count += 1 - @test record isa MBP10Msg - end - end - @test count > 0 - finally - if decoder.io !== decoder.base_io - close(decoder.io) - end - close(decoder.base_io) - end - end - end - end + end + + @testset "Mixed record types in single file" begin + metadata = create_test_metadata(Schema.MBO) # Use MBO schema for mixed content + temp_file = tempname() * ".dbn" + + try + # Create multiple different message types + hd1 = RecordHeader(55, RType.MBO_MSG, 1, 11111, 1640995200000000000) + mbo_msg = MBOMsg(hd1, 1111111111, 10050000000, 100, 0x01, 1, Action.ADD, Side.BID, 1640995200000000001, 1000, 1001) + + hd2 = RecordHeader(45, RType.MBP_0_MSG, 1, 11111, 1640995200000000002) + trade_msg = TradeMsg(hd2, 10055000000, 200, Action.TRADE, Side.NONE, 0x02, 0, 1640995200000000003, 2000, 1002) + + hd3 = RecordHeader(14, RType.OHLCV_1S_MSG, 1, 11111, 1640995200000000004) + ohlcv_msg = OHLCVMsg(hd3, 10050000000, 10060000000, 10040000000, 10055000000, 50000) + + records_to_write = [mbo_msg, trade_msg, ohlcv_msg] + + # Write and read back + write_dbn(temp_file, metadata, records_to_write) + read_records = read_dbn(temp_file) + + @test length(read_records) == 3 + @test read_records[1] isa MBOMsg + @test read_records[2] isa TradeMsg + @test read_records[3] isa OHLCVMsg + + # Verify record order and basic data + @test read_records[1].order_id == mbo_msg.order_id + @test read_records[2].price == trade_msg.price + @test read_records[3].volume == ohlcv_msg.volume + + finally + safe_rm(temp_file) + end + end + + @testset "Large dataset with many records" begin + metadata = create_test_metadata(Schema.TRADES) + temp_file = tempname() * ".dbn" + + try + # Create 1000 trade records + records_to_write = [] + base_ts = 1640995200000000000 + + for i in 1:1000 + hd = RecordHeader(45, RType.MBP_0_MSG, 1, 12345, base_ts + i * 1000000) + trade_msg = TradeMsg( + hd, + 10050000000 + i * 1000, # varying price + 100 + i, # varying size + Action.TRADE, + Side.NONE, + 0x01, + 0, + base_ts + i * 1000000 + 500000, + 1000 + i, + i + ) + push!(records_to_write, trade_msg) + end + + # Write and read back + write_dbn(temp_file, metadata, records_to_write) + read_records = read_dbn(temp_file) + + @test length(read_records) == 1000 + + # Verify first and last records + @test read_records[1].price == 10050001000 + @test read_records[1].size == 101 + @test read_records[1].sequence == 1 + + @test read_records[1000].price == 10051000000 + @test read_records[1000].size == 1100 + @test read_records[1000].sequence == 1000 + + # Verify all records are TradeMsg + @test all(r isa TradeMsg for r in read_records) + + finally + safe_rm(temp_file) + end + end + + @testset "MBO v3 file reading" begin + # Test reading actual MBO v3 files from test data + @testset "Uncompressed MBO v3" begin + file = joinpath(@__DIR__, "data", "test_data.mbo.v3.dbn") + if isfile(file) + records = read_dbn(file) + @test length(records) == 2 + @test all(r isa MBOMsg for r in records) + + # First record + r1 = records[1] + @test r1.order_id == 647784973705 # official value; pre-0.1.7 this held the price bytes + @test r1.price == 3722750000000 + @test r1.ts_recv == 1609160400000704060 + @test r1.action == Action.CANCEL + @test r1.side == Side.ASK + @test r1.size == 1 + @test r1.hd.publisher_id == 1 + @test r1.hd.instrument_id == 5482 + + # Second record + r2 = records[2] + @test r2.order_id == 647784973631 + @test r2.price == 3723000000000 + @test r2.action == Action.CANCEL + @test r2.side == Side.ASK + @test r2.sequence == r1.sequence + 1 + end + end + + @testset "Compressed MBO v3" begin + file = joinpath(@__DIR__, "data", "test_data.mbo.v3.dbn.zst") + if isfile(file) + records = read_dbn(file) + @test length(records) == 2 + @test all(r isa MBOMsg for r in records) + + # Should match uncompressed data + r1 = records[1] + @test r1.order_id == 647784973705 # official value; pre-0.1.7 this held the price bytes + @test r1.price == 3722750000000 + @test r1.ts_recv == 1609160400000704060 + @test r1.action == Action.CANCEL + @test r1.side == Side.ASK + end + end + end + + @testset "OHLCV message tests" begin + @testset "OHLCV structure and reading" begin + # Test all OHLCV cadences + test_files = [ + ("test_data.ohlcv-1s.dbn", RType.OHLCV_1S_MSG, Schema.OHLCV_1S), + ("test_data.ohlcv-1m.dbn", RType.OHLCV_1M_MSG, Schema.OHLCV_1M), + ("test_data.ohlcv-1h.dbn", RType.OHLCV_1H_MSG, Schema.OHLCV_1H), + ("test_data.ohlcv-1d.dbn", RType.OHLCV_1D_MSG, Schema.OHLCV_1D), + ] + + for (filename, expected_rtype, expected_schema) in test_files + file = joinpath(@__DIR__, "data", filename) + if isfile(file) + @testset "$filename" begin + records = read_dbn(file) + + if length(records) > 0 + @test all(r isa OHLCVMsg for r in records) + @test all(r.hd.rtype == expected_rtype for r in records) + + # Check first record has valid OHLCV data + r = records[1] + @test r.high >= r.low + @test r.open >= r.low && r.open <= r.high + @test r.close >= r.low && r.close <= r.high + @test r.volume >= 0 + end + end + end + end + end + + @testset "OHLCV v2 vs v3 compatibility" begin + v2_file = joinpath(@__DIR__, "data", "test_data.ohlcv-1s.dbn") + v3_file = joinpath(@__DIR__, "data", "test_data.ohlcv-1s.v3.dbn.zst") + + if isfile(v2_file) && isfile(v3_file) + v2_records = read_dbn(v2_file) + v3_records = read_dbn(v3_file) + + @test length(v2_records) == length(v3_records) + + if length(v2_records) > 0 + # Compare first record + r2 = v2_records[1] + r3 = v3_records[1] + + @test r2.open == r3.open + @test r2.high == r3.high + @test r2.low == r3.low + @test r2.close == r3.close + @test r2.volume == r3.volume + end + end + end + + @testset "OHLCV write and read back" begin + metadata = create_test_metadata(Schema.OHLCV_1S) + temp_file = tempname() * ".dbn" + + try + # Create test OHLCV messages + hd1 = RecordHeader(14, RType.OHLCV_1S_MSG, 1, 5482, 1609160400000000000) + ohlcv1 = OHLCVMsg(hd1, 100000000000, 105000000000, 99000000000, 102000000000, 1500) + + hd2 = RecordHeader(14, RType.OHLCV_1S_MSG, 1, 5482, 1609160401000000000) + ohlcv2 = OHLCVMsg(hd2, 102000000000, 103000000000, 101000000000, 101500000000, 800) + + # Write and read back + write_dbn(temp_file, metadata, [ohlcv1, ohlcv2]) + records = read_dbn(temp_file) + + @test length(records) == 2 + @test all(r isa OHLCVMsg for r in records) + + # Verify data integrity + @test records[1].open == 100000000000 + @test records[1].high == 105000000000 + @test records[1].low == 99000000000 + @test records[1].close == 102000000000 + @test records[1].volume == 1500 + + @test records[2].open == 102000000000 + @test records[2].close == 101500000000 + @test records[2].volume == 800 + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + end + + @testset "MBP message tests" begin + @testset "MBP-1 structure and reading" begin + # Test MBP-1 files + test_files = [ + ("test_data.mbp-1.dbn", RType.MBP_1_MSG, Schema.MBP_1), + ("test_data.mbp-1.v3.dbn.zst", RType.MBP_1_MSG, Schema.MBP_1), + ] + + for (filename, expected_rtype, expected_schema) in test_files + file = joinpath(@__DIR__, "data", filename) + if isfile(file) + @testset "$filename" begin + records = read_dbn(file) + + if length(records) > 0 + @test all(r isa MBP1Msg for r in records) + @test all(r.hd.rtype == expected_rtype for r in records) + + # Check first record has valid MBP data + r = records[1] + @test r.price > 0 + @test r.size > 0 + @test r.action in [Action.ADD, Action.MODIFY, Action.CANCEL, Action.CLEAR, Action.TRADE] + @test r.side in [Side.BID, Side.ASK, Side.NONE] + + # Check BidAskPair structure (32 bytes total) + level = r.levels + @test typeof(level) == BidAskPair + @test level.bid_px >= 0 + @test level.ask_px >= 0 + @test level.bid_sz >= 0 + @test level.ask_sz >= 0 + @test level.bid_ct >= 0 + @test level.ask_ct >= 0 + + # Verify record size matches length field + expected_size = DBN.record_length_bytes(r.hd) + @test expected_size == 80 # 16 (header) + 32 (MBP data) + 32 (BidAskPair) + end + end + end + end + end + + @testset "MBP-10 structure and reading" begin + # Test MBP-10 files + test_files = [ + ("test_data.mbp-10.dbn", RType.MBP_10_MSG, Schema.MBP_10), + ("test_data.mbp-10.v3.dbn.zst", RType.MBP_10_MSG, Schema.MBP_10), + ] + + for (filename, expected_rtype, expected_schema) in test_files + file = joinpath(@__DIR__, "data", filename) + if isfile(file) + @testset "$filename" begin + records = read_dbn(file) + + if length(records) > 0 + @test all(r isa MBP10Msg for r in records) + @test all(r.hd.rtype == expected_rtype for r in records) + + # Check first record has valid MBP data + r = records[1] + @test r.price > 0 + @test r.size > 0 + @test r.action in [Action.ADD, Action.MODIFY, Action.CANCEL, Action.CLEAR, Action.TRADE] + @test r.side in [Side.BID, Side.ASK, Side.NONE] + + # Check all 10 BidAskPair levels + @test length(r.levels) == 10 + for (i, level) in enumerate(r.levels) + @test typeof(level) == BidAskPair + @test level.bid_px >= 0 + @test level.ask_px >= 0 + @test level.bid_sz >= 0 + @test level.ask_sz >= 0 + @test level.bid_ct >= 0 + @test level.ask_ct >= 0 + end + + # Verify record size matches length field + expected_size = DBN.record_length_bytes(r.hd) + @test expected_size == 368 # 16 (header) + 32 (MBP data) + 320 (10 × 32-byte BidAskPairs) + end + end + end + end + end + + @testset "MBP v2 vs v3 compatibility" begin + # MBP-1 comparison + v2_file = joinpath(@__DIR__, "data", "test_data.mbp-1.dbn") + v3_file = joinpath(@__DIR__, "data", "test_data.mbp-1.v3.dbn.zst") + + if isfile(v2_file) && isfile(v3_file) + @testset "MBP-1 v2 vs v3" begin + v2_records = read_dbn(v2_file) + v3_records = read_dbn(v3_file) + + @test length(v2_records) == length(v3_records) + + if length(v2_records) > 0 + # Compare first record + r2 = v2_records[1] + r3 = v3_records[1] + + @test r2.price == r3.price + @test r2.size == r3.size + @test r2.action == r3.action + @test r2.side == r3.side + @test r2.levels.bid_px == r3.levels.bid_px + @test r2.levels.ask_px == r3.levels.ask_px + @test r2.levels.bid_sz == r3.levels.bid_sz + @test r2.levels.ask_sz == r3.levels.ask_sz + end + end + end + + # MBP-10 comparison + v2_file = joinpath(@__DIR__, "data", "test_data.mbp-10.dbn") + v3_file = joinpath(@__DIR__, "data", "test_data.mbp-10.v3.dbn.zst") + + if isfile(v2_file) && isfile(v3_file) + @testset "MBP-10 v2 vs v3" begin + v2_records = read_dbn(v2_file) + v3_records = read_dbn(v3_file) + + @test length(v2_records) == length(v3_records) + + if length(v2_records) > 0 + # Compare first record + r2 = v2_records[1] + r3 = v3_records[1] + + @test r2.price == r3.price + @test r2.size == r3.size + @test r2.action == r3.action + @test r2.side == r3.side + + # Compare all 10 levels + for i in 1:10 + @test r2.levels[i].bid_px == r3.levels[i].bid_px + @test r2.levels[i].ask_px == r3.levels[i].ask_px + @test r2.levels[i].bid_sz == r3.levels[i].bid_sz + @test r2.levels[i].ask_sz == r3.levels[i].ask_sz + end + end + end + end + end + + @testset "MBP write and read back" begin + @testset "MBP-1 roundtrip" begin + metadata = create_test_metadata(Schema.MBP_1) + temp_file = tempname() * ".dbn" + + try + # Create test MBP-1 messages + hd1 = RecordHeader(20, RType.MBP_1_MSG, 1, 5482, 1609160400000000000) + level1 = BidAskPair(3720250000000, 3720500000000, 24, 11, 15, 9) + mbp1_1 = MBP1Msg(hd1, 3720500000000, 1, Action.ADD, Side.ASK, 0x80, 0, 1609160400006136329, 17214, 1170362, level1) + + hd2 = RecordHeader(20, RType.MBP_1_MSG, 1, 5482, 1609160400000001000) + level2 = BidAskPair(3720000000000, 3720750000000, 31, 34, 12, 8) + mbp1_2 = MBP1Msg(hd2, 3720750000000, 2, Action.MODIFY, Side.BID, 0x40, 1, 1609160400006136330, 17215, 1170363, level2) + + # Write and read back + write_dbn(temp_file, metadata, [mbp1_1, mbp1_2]) + records = read_dbn(temp_file) + + @test length(records) == 2 + @test all(r isa MBP1Msg for r in records) + + # Verify data integrity + @test records[1].price == 3720500000000 + @test records[1].levels.bid_px == 3720250000000 + @test records[1].levels.ask_px == 3720500000000 + @test records[1].levels.bid_sz == 24 + @test records[1].levels.ask_sz == 11 + + @test records[2].price == 3720750000000 + @test records[2].action == Action.MODIFY + @test records[2].side == Side.BID + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + + @testset "MBP-10 roundtrip" begin + metadata = create_test_metadata(Schema.MBP_10) + temp_file = tempname() * ".dbn" + + try + # Create test MBP-10 message with realistic levels + hd = RecordHeader(92, RType.MBP_10_MSG, 1, 5482, 1609160400000000000) + levels = ntuple(10) do i + bid_px = 3720250000000 - (i-1) * 250000000 # decreasing bids + ask_px = 3720500000000 + (i-1) * 250000000 # increasing asks + bid_sz = UInt32(20 + i * 5) + ask_sz = UInt32(15 + i * 3) + bid_ct = UInt32(10 + i) + ask_ct = UInt32(8 + i) + BidAskPair(bid_px, ask_px, bid_sz, ask_sz, bid_ct, ask_ct) + end + + mbp10 = MBP10Msg(hd, 3722750000000, 1, Action.CANCEL, Side.ASK, 0x80, 9, 1609160400000704060, 22993, 1170352, levels) + + # Write and read back + write_dbn(temp_file, metadata, [mbp10]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa MBP10Msg + + # Verify data integrity + r = records[1] + @test r.price == 3722750000000 + @test r.action == Action.CANCEL + @test r.side == Side.ASK + @test length(r.levels) == 10 + + # Check first and last levels + @test r.levels[1].bid_px == 3720250000000 + @test r.levels[1].ask_px == 3720500000000 + @test r.levels[10].bid_px == 3720250000000 - 9 * 250000000 + @test r.levels[10].ask_px == 3720500000000 + 9 * 250000000 + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + end + end + + @testset "Trades message tests" begin + @testset "Trades structure and reading" begin + # Test Trades files + test_files = [ + ("test_data.trades.dbn", RType.MBP_0_MSG, Schema.TRADES), + ("test_data.trades.v1.dbn.zst", RType.MBP_0_MSG, Schema.TRADES), + ("test_data.trades.v2.dbn.zst", RType.MBP_0_MSG, Schema.TRADES), + ("test_data.trades.v3.dbn.zst", RType.MBP_0_MSG, Schema.TRADES), + ] + + for (filename, expected_rtype, expected_schema) in test_files + file = joinpath(@__DIR__, "data", filename) + if isfile(file) + @testset "$filename" begin + records = read_dbn(file) + + if length(records) > 0 + @test all(r isa TradeMsg for r in records) + @test all(r.hd.rtype == expected_rtype for r in records) + + # Check first record has valid Trade data + r = records[1] + @test r.price > 0 + @test r.size > 0 + @test r.action == Action.TRADE # Trades should always have TRADE action + @test r.side in [Side.BID, Side.ASK, Side.NONE] + @test r.depth == 0 # Trades typically have depth 0 + + # Verify record size matches length field (48 bytes total) + expected_size = DBN.record_length_bytes(r.hd) + @test expected_size == 48 # 16 (header) + 32 (trade data) + end + end + end + end + end + + @testset "Trades v1/v2/v3 compatibility" begin + # Test compatibility across versions + test_pairs = [ + ("test_data.trades.dbn", "test_data.trades.v2.dbn.zst"), + ("test_data.trades.v2.dbn.zst", "test_data.trades.v3.dbn.zst"), + ] + + for (file1, file2) in test_pairs + path1 = joinpath(@__DIR__, "data", file1) + path2 = joinpath(@__DIR__, "data", file2) + + if isfile(path1) && isfile(path2) + @testset "$file1 vs $file2" begin + records1 = read_dbn(path1) + records2 = read_dbn(path2) + + @test length(records1) == length(records2) + + if length(records1) > 0 + # Compare first record + r1 = records1[1] + r2 = records2[1] + + @test r1.price == r2.price + @test r1.size == r2.size + @test r1.action == r2.action + @test r1.side == r2.side + @test r1.flags == r2.flags + @test r1.depth == r2.depth + @test r1.ts_recv == r2.ts_recv + @test r1.ts_in_delta == r2.ts_in_delta + @test r1.sequence == r2.sequence + end + end + end + end + end + + @testset "Trades write and read back" begin + @testset "Single trade roundtrip" begin + metadata = create_test_metadata(Schema.TRADES) + temp_file = tempname() * ".dbn" + + try + # Create test Trade message matching real data format + hd = RecordHeader(12, RType.MBP_0_MSG, 1, 5482, 1609160400098821953) + trade_msg = TradeMsg( + hd, # hd + 3720250000000, # price ($3720.25) + 5, # size + Action.TRADE, # action + Side.ASK, # side + 0x81, # flags (129) + 0, # depth + 1609160400099150057, # ts_recv + 19251, # ts_in_delta + 1170380 # sequence + ) + + # Write and read back + write_dbn(temp_file, metadata, [trade_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa TradeMsg + + # Verify data integrity + r = records[1] + @test r.price == trade_msg.price + @test r.size == trade_msg.size + @test r.action == trade_msg.action + @test r.side == trade_msg.side + @test r.flags == trade_msg.flags + @test r.depth == trade_msg.depth + @test r.ts_recv == trade_msg.ts_recv + @test r.ts_in_delta == trade_msg.ts_in_delta + @test r.sequence == trade_msg.sequence + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + + @testset "Multiple trades roundtrip" begin + metadata = create_test_metadata(Schema.TRADES) + temp_file = tempname() * ".dbn" + + try + # Create multiple trade messages with varying data + trades = [] + base_ts = 1609160400098821953 + base_price = 3720000000000 # $3720.00 + + for i in 1:10 + hd = RecordHeader(12, RType.MBP_0_MSG, 1, 5482, base_ts + i * 1000000) + trade = TradeMsg( + hd, + base_price + i * 250000000, # increment price by $0.25 + 10 + i * 5, # varying size + Action.TRADE, + i % 2 == 0 ? Side.BID : Side.ASK, # alternate sides + UInt8(0x80 + i % 8), # varying flags + 0, # depth always 0 for trades + base_ts + i * 1000000 + 100000, # ts_recv slightly after ts_event + 15000 + i * 1000, # varying ts_in_delta + 1170000 + i # incrementing sequence + ) + push!(trades, trade) + end + + # Write and read back + write_dbn(temp_file, metadata, trades) + records = read_dbn(temp_file) + + @test length(records) == 10 + @test all(r isa TradeMsg for r in records) + + # Verify all data matches + for (orig, read) in zip(trades, records) + @test orig.price == read.price + @test orig.size == read.size + @test orig.action == read.action + @test orig.side == read.side + @test orig.flags == read.flags + @test orig.sequence == read.sequence + end + + # Verify price progression + @test records[1].price == base_price + 250000000 # $3720.25 + @test records[10].price == base_price + 10 * 250000000 # $3722.50 + + # Verify size progression + @test records[1].size == 15 + @test records[10].size == 60 + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + + @testset "Trades with different actions and sides" begin + metadata = create_test_metadata(Schema.TRADES) + temp_file = tempname() * ".dbn" + + try + # Test different combinations of actions and sides + test_cases = [ + (Action.TRADE, Side.BID), + (Action.TRADE, Side.ASK), + (Action.TRADE, Side.NONE), + (Action.FILL, Side.BID), + (Action.FILL, Side.ASK), + ] + + trades = [] + for (i, (action, side)) in enumerate(test_cases) + hd = RecordHeader(12, RType.MBP_0_MSG, 1, 5482, 1609160400000000000 + i * 1000000) + trade = TradeMsg( + hd, + 3720000000000 + i * 1000000000, # varying prices + 100 + i * 10, # varying sizes + action, + side, + UInt8(0x80), # standard flags + 0, # depth + 1609160400000000000 + i * 1000000 + 50000, + 20000, + 1000000 + i + ) + push!(trades, trade) + end + + # Write and read back + write_dbn(temp_file, metadata, trades) + records = read_dbn(temp_file) + + @test length(records) == length(test_cases) + + # Verify each action/side combination + for (i, ((expected_action, expected_side), record)) in enumerate(zip(test_cases, records)) + @test record.action == expected_action + @test record.side == expected_side + @test record.size == 100 + i * 10 + end + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + end + end + + @testset "BBO message tests" begin + @testset "BBO structure and reading" begin + # Test different BBO file types + test_files = [ + ("test_data.tbbo.dbn", RType.MBP_1_MSG, Schema.TBBO), + ("test_data.tbbo.v2.dbn.zst", RType.MBP_1_MSG, Schema.TBBO), + ("test_data.cbbo-1s.dbn", RType.CBBO_1S_MSG, Schema.CBBO_1S), + ("test_data.cbbo-1s.v2.dbn.zst", RType.CBBO_1S_MSG, Schema.CBBO_1S), + ("test_data.bbo-1s.dbn", RType.BBO_1S_MSG, Schema.BBO_1S), + ("test_data.bbo-1s.v2.dbn.zst", RType.BBO_1S_MSG, Schema.BBO_1S), + ("test_data.bbo-1s.v3.dbn.zst", RType.BBO_1S_MSG, Schema.BBO_1S), + ("test_data.bbo-1m.v3.dbn.zst", RType.BBO_1M_MSG, Schema.BBO_1M), + ] + + for (filename, expected_rtype, expected_schema) in test_files + file = joinpath(@__DIR__, "data", filename) + if isfile(file) + @testset "$filename" begin + records = read_dbn(file) + + if length(records) > 0 + # Check record type based on RType + if expected_rtype == RType.MBP_1_MSG + @test all(r isa MBP1Msg for r in records) + elseif expected_rtype == RType.CBBO_1S_MSG + @test all(r isa CBBO1sMsg for r in records) + elseif expected_rtype == RType.BBO_1S_MSG + @test all(r isa BBO1sMsg for r in records) + elseif expected_rtype == RType.BBO_1M_MSG + @test all(r isa BBO1mMsg for r in records) + end + + @test all(r.hd.rtype == expected_rtype for r in records) + + # Check first record has valid BBO data + r = records[1] + @test r.price > 0 + @test r.size > 0 + @test r.side in [Side.BID, Side.ASK, Side.NONE] + + # BBO messages typically use NONE action + if hasproperty(r, :action) + @test r.action in [Action.NONE, Action.TRADE, Action.MODIFY] + end + + # Check BidAskPair levels + if hasproperty(r, :levels) + @test typeof(r.levels) == BidAskPair + @test r.levels.bid_px >= 0 + @test r.levels.ask_px >= 0 + @test r.levels.bid_sz >= 0 + @test r.levels.ask_sz >= 0 + end + + # Verify record size matches length field (80 bytes total) + expected_size = DBN.record_length_bytes(r.hd) + @test expected_size == 80 # 16 (header) + 32 (BBO data) + 32 (BidAskPair) + end + end + end + end + end + + @testset "BBO v2 vs v3 compatibility" begin + # Test BBO compatibility across versions + v2_file = joinpath(@__DIR__, "data", "test_data.bbo-1s.v2.dbn.zst") + v3_file = joinpath(@__DIR__, "data", "test_data.bbo-1s.v3.dbn.zst") + + if isfile(v2_file) && isfile(v3_file) + @testset "BBO-1s v2 vs v3" begin + v2_records = read_dbn(v2_file) + v3_records = read_dbn(v3_file) + + @test length(v2_records) == length(v3_records) + + if length(v2_records) > 0 + # Compare first record + r2 = v2_records[1] + r3 = v3_records[1] + + @test r2.price == r3.price + @test r2.size == r3.size + @test r2.side == r3.side + @test r2.flags == r3.flags + @test r2.ts_recv == r3.ts_recv + @test r2.sequence == r3.sequence + @test r2.levels.bid_px == r3.levels.bid_px + @test r2.levels.ask_px == r3.levels.ask_px + @test r2.levels.bid_sz == r3.levels.bid_sz + @test r2.levels.ask_sz == r3.levels.ask_sz + end + end + end + end + + @testset "BBO write and read back" begin + @testset "CBBO-1s roundtrip" begin + metadata = create_test_metadata(Schema.CBBO_1S) + temp_file = tempname() * ".dbn" + + try + # Create test CBBO-1s message + hd = RecordHeader(20, RType.CBBO_1S_MSG, 1, 5482, 1609113599045849637) + level = BidAskPair(3702250000000, 3702750000000, 18, 13, 12, 8) + cbbo_msg = CBBO1sMsg( + hd, # hd + 3702500000000, # price ($3702.5) + 2, # size + Action.NONE, # action + Side.ASK, # side + 0xa8, # flags (168) + 0, # depth + 1609113600000000000, # ts_recv + 500000000, # ts_in_delta + 145799, # sequence + level # levels + ) + + # Write and read back + write_dbn(temp_file, metadata, [cbbo_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa CBBO1sMsg + + # Verify data integrity + r = records[1] + @test r.price == cbbo_msg.price + @test r.size == cbbo_msg.size + @test r.action == cbbo_msg.action + @test r.side == cbbo_msg.side + @test r.flags == cbbo_msg.flags + @test r.ts_recv == cbbo_msg.ts_recv + @test r.sequence == cbbo_msg.sequence + @test r.levels.bid_px == cbbo_msg.levels.bid_px + @test r.levels.ask_px == cbbo_msg.levels.ask_px + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + + @testset "BBO-1s roundtrip" begin + metadata = create_test_metadata(Schema.BBO_1S) + temp_file = tempname() * ".dbn" + + try + # Create test BBO-1s message + hd = RecordHeader(20, RType.BBO_1S_MSG, 1, 5482, 1609113599045849637) + level = BidAskPair(3702250000000, 3702750000000, 18, 13, 12, 8) + bbo_msg = BBO1sMsg( + hd, # hd + 3702500000000, # price ($3702.5) + 2, # size + Action.NONE, # action + Side.ASK, # side + 0xa8, # flags (168) + 0, # depth + 1609113600000000000, # ts_recv + 500000000, # ts_in_delta + 145799, # sequence + level # levels + ) + + # Write and read back + write_dbn(temp_file, metadata, [bbo_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa BBO1sMsg + + # Verify data integrity + r = records[1] + @test r.price == bbo_msg.price + @test r.size == bbo_msg.size + @test r.action == bbo_msg.action + @test r.side == bbo_msg.side + @test r.flags == bbo_msg.flags + @test r.ts_recv == bbo_msg.ts_recv + @test r.sequence == bbo_msg.sequence + @test r.levels.bid_px == bbo_msg.levels.bid_px + @test r.levels.ask_px == bbo_msg.levels.ask_px + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + + @testset "TBBO roundtrip (MBP1Msg)" begin + metadata = create_test_metadata(Schema.TBBO) + temp_file = tempname() * ".dbn" + + try + # TBBO uses MBP1Msg structure + hd = RecordHeader(20, RType.MBP_1_MSG, 1, 5482, 1609160400098821953) + level = BidAskPair(3720250000000, 3720500000000, 26, 7, 15, 9) + tbbo_msg = MBP1Msg( + hd, # hd + 3720250000000, # price ($3720.25) + 5, # size + Action.TRADE, # action + Side.ASK, # side + 0x81, # flags (129) + 0, # depth + 1609160400099150057, # ts_recv + 19251, # ts_in_delta + 1170380, # sequence + level # levels + ) + + # Write and read back + write_dbn(temp_file, metadata, [tbbo_msg]) + records = read_dbn(temp_file) + + @test length(records) == 1 + @test records[1] isa MBP1Msg + + # Verify data integrity + r = records[1] + @test r.price == tbbo_msg.price + @test r.size == tbbo_msg.size + @test r.action == tbbo_msg.action + @test r.side == tbbo_msg.side + @test r.levels.bid_px == tbbo_msg.levels.bid_px + @test r.levels.ask_px == tbbo_msg.levels.ask_px + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + + @testset "Multiple BBO types in one file" begin + metadata = create_test_metadata(Schema.BBO_1S) + temp_file = tempname() * ".dbn" + + try + # Create multiple BBO messages with different timestamps + bbo_msgs = [] + base_ts = 1609113599000000000 + base_price = 3702000000000 + + for i in 1:5 + hd = RecordHeader(20, RType.BBO_1S_MSG, 1, 5482, base_ts + i * 1000000000) + level = BidAskPair( + base_price + i * 250000000, # bid price + base_price + (i + 1) * 250000000, # ask price + 15 + i * 2, # bid size + 10 + i * 3, # ask size + 8 + i, # bid count + 6 + i # ask count + ) + + bbo_msg = BBO1sMsg( + hd, + base_price + i * 500000000, # price between bid/ask + UInt32(i * 2), # varying size + Action.NONE, + i % 2 == 0 ? Side.BID : Side.ASK, # alternate sides + UInt8(0xa0 + i), # varying flags + UInt8(0), # depth + base_ts + i * 1000000000 + 100000000, + Int32(500000000 + i * 100000000), # varying ts_in_delta + UInt32(145000 + i), # incrementing sequence + level + ) + push!(bbo_msgs, bbo_msg) + end + + # Write and read back + write_dbn(temp_file, metadata, bbo_msgs) + records = read_dbn(temp_file) + + @test length(records) == 5 + @test all(r isa BBO1sMsg for r in records) + + # Verify progression + for (i, (orig, read)) in enumerate(zip(bbo_msgs, records)) + @test orig.price == read.price + @test orig.size == read.size + @test orig.side == read.side + @test orig.sequence == read.sequence + end + + # Check price progression + @test records[1].price == base_price + 500000000 # $3702.50 + @test records[5].price == base_price + 5 * 500000000 # $3705.00 + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + end + end + + @testset "Zstd compression tests" begin + @testset "Auto-detection of compressed files" begin + # Test that we can read compressed files with and without .zst extension + compressed_file = joinpath(@__DIR__, "data", "test_data.mbo.v3.dbn.zst") + + if isfile(compressed_file) + # Copy to a file without .zst extension + temp_file = tempname() * ".dbn" + try + cp(compressed_file, temp_file) + + # Should auto-detect compression by magic bytes + records = read_dbn(temp_file) + @test length(records) == 2 + @test all(r isa MBOMsg for r in records) + + finally + if isfile(temp_file) + safe_rm(temp_file) + end + end + end + end + + @testset "Streaming decompression" begin + # Test that we can stream large compressed files + compressed_file = joinpath(@__DIR__, "data", "test_data.mbp-10.v3.dbn.zst") + + if isfile(compressed_file) + # Use DBNDecoder directly to test streaming + decoder = DBNDecoder(compressed_file) + try + count = 0 + while !eof(decoder.io) + record = read_record(decoder) + if record !== nothing + count += 1 + @test record isa MBP10Msg + end + end + @test count > 0 + finally + if decoder.io !== decoder.base_io + close(decoder.io) + end + close(decoder.base_io) + end + end + end + end end diff --git a/test/test_phase6.jl b/test/test_phase6.jl index c65e93bb..e5091e61 100644 --- a/test/test_phase6.jl +++ b/test/test_phase6.jl @@ -1,467 +1,467 @@ -# Phase 6: Compression Testing -using Dates - -@testset "Phase 6: Compression Features" begin - - @testset "Compressed File Reading" begin - @testset "Auto-detection of compressed files" begin - # Test automatic detection by file extension - compressed_files = [ - "test/data/test_data.trades.v3.dbn.zst", - "test/data/test_data.mbp-1.v3.dbn.zst", - "test/data/test_data.mbo.v3.dbn.zst", - "test/data/test_data.ohlcv-1s.v3.dbn.zst" - ] - - for file in compressed_files - if isfile(file) - @test_nowarn begin - records = read_dbn(file) - @test length(records) > 0 - println("Successfully read $(length(records)) records from $file") - end - end - end - end - - @testset "Zstd magic number detection" begin - # Test detection by magic bytes regardless of extension - test_file = "test/data/test_data.trades.v3.dbn.zst" - if isfile(test_file) - # Copy to file without .zst extension to test magic byte detection - temp_file = tempname() - cp(test_file, temp_file) - - try - @test_nowarn begin - records = read_dbn(temp_file) - @test length(records) > 0 - end - finally - safe_rm(temp_file) - end - end - end - - @testset "Compare compressed vs uncompressed content" begin - # Test that compressed and uncompressed versions have identical content - test_pairs = [ - ("test/data/test_data.trades.dbn", "test/data/test_data.trades.v3.dbn.zst"), - ("test/data/test_data.mbp-1.dbn", "test/data/test_data.mbp-1.v3.dbn.zst"), - ("test/data/test_data.ohlcv-1s.dbn", "test/data/test_data.ohlcv-1s.v3.dbn.zst") - ] - - for (uncompressed, compressed) in test_pairs - if isfile(uncompressed) && isfile(compressed) - records_uncomp = read_dbn(uncompressed) - records_comp = read_dbn(compressed) - - @test length(records_uncomp) == length(records_comp) - - # Compare first few records in detail - for i in 1:min(5, length(records_uncomp)) - @test typeof(records_uncomp[i]) == typeof(records_comp[i]) - # Test that timestamps match - if hasproperty(records_uncomp[i], :hd) && hasproperty(records_uncomp[i].hd, :ts_event) - @test records_uncomp[i].hd.ts_event == records_comp[i].hd.ts_event - end - end - end - end - end - end - - @testset "compress_dbn_file Function" begin - @testset "Basic compression functionality" begin - # Create test data - test_input = tempname() - test_output = tempname() - - try - # Create simple test file - metadata = Metadata( - 3, "XNAS", Schema.TRADES, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["AAPL"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - - # Create sample records - records = [] - for i in 1:10 - hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000 + i*1000000000) - trade = TradeMsg(hd, 150000000000, 100, Action.TRADE, Side.BID, 0, 0, - 1640995200000000000 + i*1000000000, 0, UInt32(i)) - push!(records, trade) - end - - # Write test file - write_dbn(test_input, metadata, records) - - # Test compression - @test_nowarn begin - stats = compress_dbn_file(test_input, test_output) - - @test haskey(stats, :original_size) - @test haskey(stats, :compressed_size) - @test haskey(stats, :compression_ratio) - @test haskey(stats, :space_saved) - - @test stats.original_size > 0 - @test stats.compressed_size > 0 - @test stats.space_saved >= 0 - @test 0.0 <= stats.compression_ratio <= 1.0 - - println("Compression stats: $(stats)") - end - - # Verify compressed file can be read - @test_nowarn begin - compressed_records = read_dbn(test_output) - @test length(compressed_records) == length(records) - end - - finally - safe_rm(test_input) - safe_rm(test_output) - end - end - - @testset "Compression with delete_original option" begin - test_input = tempname() - test_output = tempname() - - try - # Create minimal test file - metadata = Metadata( - 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - write_dbn(test_input, metadata, []) - - @test isfile(test_input) - - # Compress with delete_original=true - stats = compress_dbn_file(test_input, test_output, delete_original=true) - - @test !isfile(test_input) # Original should be deleted - @test isfile(test_output) # Compressed file should exist - - finally - safe_rm(test_input) - safe_rm(test_output) - end - end - - @testset "Compression error handling" begin - # Test with non-existent input file - @test_throws SystemError compress_dbn_file("nonexistent.dbn", "output.dbn.zst") - - # Test with invalid output directory - test_input = tempname() - try - metadata = Metadata( - 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - write_dbn(test_input, metadata, []) - - @test_throws SystemError compress_dbn_file(test_input, "/invalid/path/output.dbn.zst") - - finally - safe_rm(test_input) - end - end - end - - @testset "compress_daily_files Function" begin - @testset "Batch compression functionality" begin - temp_dir = mktempdir() - - try - # Create test files for a specific date - test_date = Date("2024-01-15") - date_str = "2024-01-15" - - test_files = [ - joinpath(temp_dir, "$(date_str)_trades.dbn"), - joinpath(temp_dir, "$(date_str)_mbp1.dbn"), - joinpath(temp_dir, "$(date_str)_ohlcv.dbn") - ] - - # Create test metadata and records - metadata = Metadata( - 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - - records = [ - TradeMsg( - RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000), - 150000000000, 100, Action.TRADE, Side.BID, 0, 0, - 1640995200000000000, 0, UInt32(1) - ) - ] - - # Write test files - for file in test_files - write_dbn(file, metadata, records) - end - - # Test batch compression - results = compress_daily_files(test_date, temp_dir) - - @test length(results) == length(test_files) - - for (i, result) in enumerate(results) - if result !== nothing - @test haskey(result, :original_size) - @test haskey(result, :compressed_size) - @test result.original_size > 0 - - # Check compressed file exists - compressed_file = replace(test_files[i], ".dbn" => ".dbn.zst") - @test isfile(compressed_file) - - # Original should be deleted (delete_original=true by default) - @test !isfile(test_files[i]) - end - end - - finally - rm(temp_dir, recursive=true, force=true) - end - end - - @testset "Pattern matching for daily files" begin - temp_dir = mktempdir() - - try - test_date = Date("2024-02-20") - date_str = "2024-02-20" - - # Create files that should match - matching_files = [ - joinpath(temp_dir, "$(date_str)_data.dbn"), - joinpath(temp_dir, "symbols_$(date_str).dbn") - ] - - # Create files that should NOT match - non_matching_files = [ - joinpath(temp_dir, "$(date_str)_data.txt"), # Wrong extension - joinpath(temp_dir, "2024-02-21_data.dbn"), # Wrong date - joinpath(temp_dir, "other_$(date_str).csv") # Wrong extension - ] - - metadata = Metadata( - 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - - # Create all files - for file in vcat(matching_files, non_matching_files) - if endswith(file, ".dbn") - write_dbn(file, metadata, []) - else - touch(file) - end - end - - # Test compression - should only process matching files - results = compress_daily_files(test_date, temp_dir) - - # Should only process .dbn files with the date - @test length(results) == length(matching_files) - - # Check that only matching files were processed - for file in matching_files - compressed_file = replace(file, ".dbn" => ".dbn.zst") - @test isfile(compressed_file) - @test !isfile(file) # Original deleted - end - - # Non-matching files should still exist - for file in non_matching_files - @test isfile(file) - end - - finally - rm(temp_dir, recursive=true, force=true) - end - end - end - - @testset "Compression Stats and File Size Verification" begin - @testset "Compression ratio calculations" begin - test_input = tempname() - test_output = tempname() - - try - # Create test file with repetitive data (should compress well) - metadata = Metadata( - 3, "REPEAT", Schema.TRADES, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["AAPL"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - - # Create many similar records (should compress well) - records = [] - for i in 1:1000 - hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000 + i*1000000) - trade = TradeMsg(hd, 150000000000, 100, Action.TRADE, Side.BID, 0, 0, - 1640995200000000000 + i*1000000, 0, UInt32(i)) - push!(records, trade) - end - - write_dbn(test_input, metadata, records) - - original_size = filesize(test_input) - @test original_size > 0 - - stats = compress_dbn_file(test_input, test_output) - - # Verify stats accuracy - @test stats.original_size == original_size - @test stats.compressed_size == filesize(test_output) - @test stats.space_saved == (original_size - stats.compressed_size) - @test stats.compression_ratio ≈ (1.0 - stats.compressed_size / original_size) - - # For repetitive data, we should get decent compression - @test stats.compression_ratio > 0.1 # At least 10% compression - @test stats.space_saved > 0 - - println("Compression achieved: $(round(stats.compression_ratio * 100, digits=2))%") - println("Space saved: $(stats.space_saved) bytes") - - finally - safe_rm(test_input) - safe_rm(test_output) - end - end - - @testset "File size comparisons" begin - # Test with existing compressed test data - test_pairs = [ - ("test/data/test_data.trades.dbn", "test/data/test_data.trades.v3.dbn.zst"), - ("test/data/test_data.ohlcv-1s.dbn", "test/data/test_data.ohlcv-1s.v3.dbn.zst") - ] - - for (uncompressed_file, compressed_file) in test_pairs - if isfile(uncompressed_file) && isfile(compressed_file) - original_size = filesize(uncompressed_file) - compressed_size = filesize(compressed_file) - - @test original_size > 0 - @test compressed_size > 0 - @test compressed_size < original_size # Compression should reduce size - - ratio = 1.0 - (compressed_size / original_size) - @test ratio > 0.0 # Should achieve some compression - - println("File: $(basename(uncompressed_file))") - println(" Original: $(original_size) bytes") - println(" Compressed: $(compressed_size) bytes") - println(" Ratio: $(round(ratio * 100, digits=2))%") - end - end - end - end - - @testset "Error Handling and Edge Cases" begin - @testset "Corrupted compressed files" begin - # Create a file with invalid zstd header - corrupted_file = tempname() - try - open(corrupted_file, "w") do io - # Write fake zstd magic bytes followed by garbage - write(io, UInt8[0x28, 0xB5, 0x2F, 0xFD]) # Zstd magic - write(io, rand(UInt8, 100)) # Random garbage - end - - @test_throws Exception read_dbn(corrupted_file) - - finally - safe_rm(corrupted_file) - end - end - - @testset "Empty file compression" begin - test_input = tempname() - test_output = tempname() - - try - # Create empty DBN file - metadata = Metadata( - 3, "EMPTY", Schema.TRADES, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - [], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - write_dbn(test_input, metadata, []) - - @test_nowarn begin - stats = compress_dbn_file(test_input, test_output) - @test stats.original_size > 0 # Should have header at least - @test stats.compressed_size > 0 - end - - # Should be able to read compressed empty file - records = read_dbn(test_output) - @test length(records) == 0 - - finally - safe_rm(test_input) - safe_rm(test_output) - end - end - - @testset "Mixed record type compression" begin - test_input = tempname() - test_output = tempname() - - try - metadata = Metadata( - 3, "MIXED", Schema.MBO, 1640995200000000000, 1640995260000000000, - nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - ["MIXED"], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - - # Create mixed record types - records = [] - - # MBO record - mbo_hd = RecordHeader(56, RType.MBO_MSG, 1, 12345, 1640995200000000000) - mbo = MBOMsg(mbo_hd, 98765, 150000000000, 100, 0, 1, Action.ADD, Side.BID, - 1640995200000000000, 0, UInt32(1)) - push!(records, mbo) - - # Trade record - trade_hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995201000000000) - trade = TradeMsg(trade_hd, 150000000000, 100, Action.TRADE, Side.BID, 0, 0, - 1640995201000000000, 0, UInt32(2)) - push!(records, trade) - - write_dbn(test_input, metadata, records) - - @test_nowarn begin - stats = compress_dbn_file(test_input, test_output) - @test stats.original_size > 0 - @test stats.compressed_size > 0 - end - - # Verify all record types preserved - compressed_records = read_dbn(test_output) - @test length(compressed_records) == 2 - @test isa(compressed_records[1], MBOMsg) - @test isa(compressed_records[2], TradeMsg) - - finally - safe_rm(test_input) - safe_rm(test_output) - end - end - end - +# Phase 6: Compression Testing +using Dates + +@testset "Phase 6: Compression Features" begin + + @testset "Compressed File Reading" begin + @testset "Auto-detection of compressed files" begin + # Test automatic detection by file extension + compressed_files = [ + "test/data/test_data.trades.v3.dbn.zst", + "test/data/test_data.mbp-1.v3.dbn.zst", + "test/data/test_data.mbo.v3.dbn.zst", + "test/data/test_data.ohlcv-1s.v3.dbn.zst" + ] + + for file in compressed_files + if isfile(file) + @test_nowarn begin + records = read_dbn(file) + @test length(records) > 0 + println("Successfully read $(length(records)) records from $file") + end + end + end + end + + @testset "Zstd magic number detection" begin + # Test detection by magic bytes regardless of extension + test_file = "test/data/test_data.trades.v3.dbn.zst" + if isfile(test_file) + # Copy to file without .zst extension to test magic byte detection + temp_file = tempname() + cp(test_file, temp_file) + + try + @test_nowarn begin + records = read_dbn(temp_file) + @test length(records) > 0 + end + finally + safe_rm(temp_file) + end + end + end + + @testset "Compare compressed vs uncompressed content" begin + # Test that compressed and uncompressed versions have identical content + test_pairs = [ + ("test/data/test_data.trades.dbn", "test/data/test_data.trades.v3.dbn.zst"), + ("test/data/test_data.mbp-1.dbn", "test/data/test_data.mbp-1.v3.dbn.zst"), + ("test/data/test_data.ohlcv-1s.dbn", "test/data/test_data.ohlcv-1s.v3.dbn.zst") + ] + + for (uncompressed, compressed) in test_pairs + if isfile(uncompressed) && isfile(compressed) + records_uncomp = read_dbn(uncompressed) + records_comp = read_dbn(compressed) + + @test length(records_uncomp) == length(records_comp) + + # Compare first few records in detail + for i in 1:min(5, length(records_uncomp)) + @test typeof(records_uncomp[i]) == typeof(records_comp[i]) + # Test that timestamps match + if hasproperty(records_uncomp[i], :hd) && hasproperty(records_uncomp[i].hd, :ts_event) + @test records_uncomp[i].hd.ts_event == records_comp[i].hd.ts_event + end + end + end + end + end + end + + @testset "compress_dbn_file Function" begin + @testset "Basic compression functionality" begin + # Create test data + test_input = tempname() + test_output = tempname() + + try + # Create simple test file + metadata = Metadata( + 3, "XNAS", Schema.TRADES, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["AAPL"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + + # Create sample records + records = [] + for i in 1:10 + hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000 + i*1000000000) + trade = TradeMsg(hd, 150000000000, 100, Action.TRADE, Side.BID, 0, 0, + 1640995200000000000 + i*1000000000, 0, UInt32(i)) + push!(records, trade) + end + + # Write test file + write_dbn(test_input, metadata, records) + + # Test compression + @test_nowarn begin + stats = compress_dbn_file(test_input, test_output) + + @test haskey(stats, :original_size) + @test haskey(stats, :compressed_size) + @test haskey(stats, :compression_ratio) + @test haskey(stats, :space_saved) + + @test stats.original_size > 0 + @test stats.compressed_size > 0 + @test stats.space_saved >= 0 + @test 0.0 <= stats.compression_ratio <= 1.0 + + println("Compression stats: $(stats)") + end + + # Verify compressed file can be read + @test_nowarn begin + compressed_records = read_dbn(test_output) + @test length(compressed_records) == length(records) + end + + finally + safe_rm(test_input) + safe_rm(test_output) + end + end + + @testset "Compression with delete_original option" begin + test_input = tempname() + test_output = tempname() + + try + # Create minimal test file + metadata = Metadata( + 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + write_dbn(test_input, metadata, []) + + @test isfile(test_input) + + # Compress with delete_original=true + stats = compress_dbn_file(test_input, test_output, delete_original=true) + + @test !isfile(test_input) # Original should be deleted + @test isfile(test_output) # Compressed file should exist + + finally + safe_rm(test_input) + safe_rm(test_output) + end + end + + @testset "Compression error handling" begin + # Test with non-existent input file + @test_throws SystemError compress_dbn_file("nonexistent.dbn", "output.dbn.zst") + + # Test with invalid output directory + test_input = tempname() + try + metadata = Metadata( + 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + write_dbn(test_input, metadata, []) + + @test_throws SystemError compress_dbn_file(test_input, "/invalid/path/output.dbn.zst") + + finally + safe_rm(test_input) + end + end + end + + @testset "compress_daily_files Function" begin + @testset "Batch compression functionality" begin + temp_dir = mktempdir() + + try + # Create test files for a specific date + test_date = Date("2024-01-15") + date_str = "2024-01-15" + + test_files = [ + joinpath(temp_dir, "$(date_str)_trades.dbn"), + joinpath(temp_dir, "$(date_str)_mbp1.dbn"), + joinpath(temp_dir, "$(date_str)_ohlcv.dbn") + ] + + # Create test metadata and records + metadata = Metadata( + 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + + records = [ + TradeMsg( + RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000), + 150000000000, 100, Action.TRADE, Side.BID, 0, 0, + 1640995200000000000, 0, UInt32(1) + ) + ] + + # Write test files + for file in test_files + write_dbn(file, metadata, records) + end + + # Test batch compression + results = compress_daily_files(test_date, temp_dir) + + @test length(results) == length(test_files) + + for (i, result) in enumerate(results) + if result !== nothing + @test haskey(result, :original_size) + @test haskey(result, :compressed_size) + @test result.original_size > 0 + + # Check compressed file exists + compressed_file = replace(test_files[i], ".dbn" => ".dbn.zst") + @test isfile(compressed_file) + + # Original should be deleted (delete_original=true by default) + @test !isfile(test_files[i]) + end + end + + finally + rm(temp_dir, recursive=true, force=true) + end + end + + @testset "Pattern matching for daily files" begin + temp_dir = mktempdir() + + try + test_date = Date("2024-02-20") + date_str = "2024-02-20" + + # Create files that should match + matching_files = [ + joinpath(temp_dir, "$(date_str)_data.dbn"), + joinpath(temp_dir, "symbols_$(date_str).dbn") + ] + + # Create files that should NOT match + non_matching_files = [ + joinpath(temp_dir, "$(date_str)_data.txt"), # Wrong extension + joinpath(temp_dir, "2024-02-21_data.dbn"), # Wrong date + joinpath(temp_dir, "other_$(date_str).csv") # Wrong extension + ] + + metadata = Metadata( + 3, "TEST", Schema.TRADES, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["TEST"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + + # Create all files + for file in vcat(matching_files, non_matching_files) + if endswith(file, ".dbn") + write_dbn(file, metadata, []) + else + touch(file) + end + end + + # Test compression - should only process matching files + results = compress_daily_files(test_date, temp_dir) + + # Should only process .dbn files with the date + @test length(results) == length(matching_files) + + # Check that only matching files were processed + for file in matching_files + compressed_file = replace(file, ".dbn" => ".dbn.zst") + @test isfile(compressed_file) + @test !isfile(file) # Original deleted + end + + # Non-matching files should still exist + for file in non_matching_files + @test isfile(file) + end + + finally + rm(temp_dir, recursive=true, force=true) + end + end + end + + @testset "Compression Stats and File Size Verification" begin + @testset "Compression ratio calculations" begin + test_input = tempname() + test_output = tempname() + + try + # Create test file with repetitive data (should compress well) + metadata = Metadata( + 3, "REPEAT", Schema.TRADES, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["AAPL"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + + # Create many similar records (should compress well) + records = [] + for i in 1:1000 + hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995200000000000 + i*1000000) + trade = TradeMsg(hd, 150000000000, 100, Action.TRADE, Side.BID, 0, 0, + 1640995200000000000 + i*1000000, 0, UInt32(i)) + push!(records, trade) + end + + write_dbn(test_input, metadata, records) + + original_size = filesize(test_input) + @test original_size > 0 + + stats = compress_dbn_file(test_input, test_output) + + # Verify stats accuracy + @test stats.original_size == original_size + @test stats.compressed_size == filesize(test_output) + @test stats.space_saved == (original_size - stats.compressed_size) + @test stats.compression_ratio ≈ (1.0 - stats.compressed_size / original_size) + + # For repetitive data, we should get decent compression + @test stats.compression_ratio > 0.1 # At least 10% compression + @test stats.space_saved > 0 + + println("Compression achieved: $(round(stats.compression_ratio * 100, digits=2))%") + println("Space saved: $(stats.space_saved) bytes") + + finally + safe_rm(test_input) + safe_rm(test_output) + end + end + + @testset "File size comparisons" begin + # Test with existing compressed test data + test_pairs = [ + ("test/data/test_data.trades.dbn", "test/data/test_data.trades.v3.dbn.zst"), + ("test/data/test_data.ohlcv-1s.dbn", "test/data/test_data.ohlcv-1s.v3.dbn.zst") + ] + + for (uncompressed_file, compressed_file) in test_pairs + if isfile(uncompressed_file) && isfile(compressed_file) + original_size = filesize(uncompressed_file) + compressed_size = filesize(compressed_file) + + @test original_size > 0 + @test compressed_size > 0 + @test compressed_size < original_size # Compression should reduce size + + ratio = 1.0 - (compressed_size / original_size) + @test ratio > 0.0 # Should achieve some compression + + println("File: $(basename(uncompressed_file))") + println(" Original: $(original_size) bytes") + println(" Compressed: $(compressed_size) bytes") + println(" Ratio: $(round(ratio * 100, digits=2))%") + end + end + end + end + + @testset "Error Handling and Edge Cases" begin + @testset "Corrupted compressed files" begin + # Create a file with invalid zstd header + corrupted_file = tempname() + try + open(corrupted_file, "w") do io + # Write fake zstd magic bytes followed by garbage + write(io, UInt8[0x28, 0xB5, 0x2F, 0xFD]) # Zstd magic + write(io, rand(UInt8, 100)) # Random garbage + end + + @test_throws Exception read_dbn(corrupted_file) + + finally + safe_rm(corrupted_file) + end + end + + @testset "Empty file compression" begin + test_input = tempname() + test_output = tempname() + + try + # Create empty DBN file + metadata = Metadata( + 3, "EMPTY", Schema.TRADES, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + [], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + write_dbn(test_input, metadata, []) + + @test_nowarn begin + stats = compress_dbn_file(test_input, test_output) + @test stats.original_size > 0 # Should have header at least + @test stats.compressed_size > 0 + end + + # Should be able to read compressed empty file + records = read_dbn(test_output) + @test length(records) == 0 + + finally + safe_rm(test_input) + safe_rm(test_output) + end + end + + @testset "Mixed record type compression" begin + test_input = tempname() + test_output = tempname() + + try + metadata = Metadata( + 3, "MIXED", Schema.MBO, 1640995200000000000, 1640995260000000000, + nothing, SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + ["MIXED"], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + + # Create mixed record types + records = [] + + # MBO record + mbo_hd = RecordHeader(56, RType.MBO_MSG, 1, 12345, 1640995200000000000) + mbo = MBOMsg(mbo_hd, 98765, 150000000000, 100, 0, 1, Action.ADD, Side.BID, + 1640995200000000000, 0, UInt32(1)) + push!(records, mbo) + + # Trade record + trade_hd = RecordHeader(40, RType.MBP_0_MSG, 1, 12345, 1640995201000000000) + trade = TradeMsg(trade_hd, 150000000000, 100, Action.TRADE, Side.BID, 0, 0, + 1640995201000000000, 0, UInt32(2)) + push!(records, trade) + + write_dbn(test_input, metadata, records) + + @test_nowarn begin + stats = compress_dbn_file(test_input, test_output) + @test stats.original_size > 0 + @test stats.compressed_size > 0 + end + + # Verify all record types preserved + compressed_records = read_dbn(test_output) + @test length(compressed_records) == 2 + @test isa(compressed_records[1], MBOMsg) + @test isa(compressed_records[2], TradeMsg) + + finally + safe_rm(test_input) + safe_rm(test_output) + end + end + end + end \ No newline at end of file diff --git a/test/test_phase7.jl b/test/test_phase7.jl index 84c9bfd3..b1b5f3d9 100644 --- a/test/test_phase7.jl +++ b/test/test_phase7.jl @@ -1,352 +1,352 @@ - - -@testset "Phase 7: Streaming Writer Testing" begin - - @testset "Test DBNStreamWriter creation" begin - # Test basic creation - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "stream_writer_test.dbn") - - # Create writer with default settings - writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) - @test writer.record_count == 0 - @test writer.first_ts == typemax(Int64) - @test writer.last_ts == 0 - @test writer.auto_flush == true - @test writer.flush_interval == 1000 - @test writer.last_flush_count == 0 - - # Close the writer - DBN.close_writer!(writer) - - # Verify file was created - @test isfile(output_file) - - # Test creation with custom parameters - output_file2 = joinpath(tmpdir, "stream_writer_custom.dbn") - writer2 = DBN.DBNStreamWriter(output_file2, "XBTS", DBN.Schema.MBO, - symbols=["AAPL", "MSFT"], - auto_flush=false, - flush_interval=500) - @test writer2.auto_flush == false - @test writer2.flush_interval == 500 - @test writer2.encoder.metadata.symbols == ["AAPL", "MSFT"] - @test writer2.encoder.metadata.dataset == "XBTS" - @test writer2.encoder.metadata.schema == DBN.Schema.MBO - - DBN.close_writer!(writer2) - end - end - - @testset "Test write_record! with timestamp tracking" begin - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "timestamp_tracking.dbn") - - writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) - - # Create test trade messages with different timestamps - ts1 = Int64(1700000000000000000) # First timestamp - ts2 = Int64(1700000001000000000) # Second timestamp (1 second later) - ts3 = Int64(1700000002000000000) # Third timestamp (2 seconds later) - - trade1 = DBN.TradeMsg( - DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts1), - 100000000, # price (10.0) - 100, # size - DBN.Action.TRADE, - DBN.Side.ASK, - 0x00, # flags - 0, # depth - ts1, # ts_recv - 10000, # ts_in_delta - 1 # sequence - ) - - trade2 = DBN.TradeMsg( - DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts2), - 101000000, # price (10.1) - 200, - DBN.Action.TRADE, - DBN.Side.BID, - 0x00, - 0, - ts2, - 10000, - 2 - ) - - trade3 = DBN.TradeMsg( - DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts3), - 102000000, # price (10.2) - 150, - DBN.Action.TRADE, - DBN.Side.ASK, - 0x00, - 0, - ts3, - 10000, - 3 - ) - - # Write records and verify timestamp tracking - DBN.write_record!(writer, trade1) - @test writer.record_count == 1 - @test writer.first_ts == ts1 - @test writer.last_ts == ts1 - - DBN.write_record!(writer, trade2) - @test writer.record_count == 2 - @test writer.first_ts == ts1 # Should remain the first - @test writer.last_ts == ts2 # Should update to the latest - - DBN.write_record!(writer, trade3) - @test writer.record_count == 3 - @test writer.first_ts == ts1 # Should still be the first - @test writer.last_ts == ts3 # Should be the latest - - DBN.close_writer!(writer) - - # Read back and verify metadata timestamps - records = DBN.read_dbn(output_file) - @test length(records) == 3 - - # Read metadata - open(output_file, "r") do io - decoder = DBN.DBNDecoder(io) - DBN.read_header!(decoder) - @test decoder.metadata.start_ts == ts1 - @test decoder.metadata.end_ts == ts3 - @test decoder.metadata.limit == 3 - end - end - end - - @testset "Test auto-flush functionality" begin - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "auto_flush_test.dbn") - - # Create writer with small flush interval for testing - writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.MBO, - auto_flush=true, - flush_interval=5) - - # Create a test MBO message - mbo_msg = DBN.MBOMsg( - DBN.RecordHeader(48, DBN.RType.MBO_MSG, 1, 0x00, Int64(1700000000000000000)), - 123456, # order_id - 100000000, # price - 100, # size - 0x00, # flags - 0, # channel_id - DBN.Action.ADD, - DBN.Side.BID, - Int64(1700000000000000000), # ts_recv - 10000, # ts_in_delta - 1 # sequence - ) - - # Write 4 records - should not trigger flush yet - for i in 1:4 - DBN.write_record!(writer, mbo_msg) - end - @test writer.last_flush_count == 0 - - # Write 5th record - should trigger flush - DBN.write_record!(writer, mbo_msg) - @test writer.last_flush_count == 5 - - # Write 4 more records - for i in 1:4 - DBN.write_record!(writer, mbo_msg) - end - @test writer.last_flush_count == 5 # Should not have flushed again - - # Write 10th record - should trigger another flush - DBN.write_record!(writer, mbo_msg) - @test writer.last_flush_count == 10 - - DBN.close_writer!(writer) - - # Verify all records were written - records = DBN.read_dbn(output_file) - @test length(records) == 10 - end - end - - @testset "Test close_writer! and header update" begin - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "header_update_test.dbn") - - # Test with symbols - the parsing issue is now fixed - writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES, - symbols=["AAPL", "MSFT", "GOOGL"]) - - # Create trades with different timestamps - timestamps = [ - Int64(1700000000000000000), - Int64(1700000005000000000), - Int64(1700000003000000000), # Out of order - Int64(1700000010000000000), - Int64(1700000001000000000) # Very early timestamp - ] - - for (i, ts) in enumerate(timestamps) - trade = DBN.TradeMsg( - DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts), - 100000000 + i * 1000000, - 100 + i, - DBN.Action.TRADE, - i % 2 == 0 ? DBN.Side.ASK : DBN.Side.BID, - 0x00, - 0, - ts, - 10000, - UInt32(i) - ) - DBN.write_record!(writer, trade) - end - - # Verify timestamps before closing - @test writer.first_ts == Int64(1700000000000000000) # Earliest - @test writer.last_ts == Int64(1700000010000000000) # Latest - @test writer.record_count == 5 - - # Close and update header - DBN.close_writer!(writer) - - # Read back and verify header was updated correctly - open(output_file, "r") do io - decoder = DBN.DBNDecoder(io) - DBN.read_header!(decoder) - metadata = decoder.metadata - - @test metadata.start_ts == Int64(1700000000000000000) - @test metadata.end_ts == Int64(1700000010000000000) - @test metadata.limit == 5 - @test metadata.symbols == ["AAPL", "MSFT", "GOOGL"] - @test metadata.dataset == "XNAS" - @test metadata.schema == DBN.Schema.TRADES - end - - # Verify records are intact - records = DBN.read_dbn(output_file) - @test length(records) == 5 - - # Verify timestamps in the read records - timestamps_read = [r.hd.ts_event for r in records] - @test sort(timestamps_read) == sort(timestamps) - end - end - - @testset "Test mixed record types with streaming writer" begin - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "mixed_records_stream.dbn") - - writer = DBN.DBNStreamWriter(output_file, "GLBX", DBN.Schema.MBO) - - # Create different record types - trade = DBN.TradeMsg( - DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, Int64(1700000000000000000)), - 100000000, 100, DBN.Action.TRADE, DBN.Side.ASK, 0x00, 0, - Int64(1700000000000000000), 10000, 1 - ) - - mbo = DBN.MBOMsg( - DBN.RecordHeader(48, DBN.RType.MBO_MSG, 1, 0x00, Int64(1700000001000000000)), - 12345, # order_id - 101000000, # price - 200, # size - 0x00, # flags - 0, # channel_id - DBN.Action.ADD, - DBN.Side.BID, - Int64(1700000001000000000), # ts_recv - 10000, # ts_in_delta - 2 # sequence - ) - - mbp1 = DBN.MBP1Msg( - DBN.RecordHeader(112, DBN.RType.MBP_1_MSG, 1, 0x00, Int64(1700000002000000000)), - 102000000, 300, DBN.Action.TRADE, DBN.Side.ASK, 0x00, 0, - Int64(1700000002000000000), 10000, 3, - DBN.BidAskPair(100000000, 105000000, 100, 150, 1, 2) - ) - - # Write mixed records - DBN.write_record!(writer, trade) - DBN.write_record!(writer, mbo) - DBN.write_record!(writer, mbp1) - DBN.write_record!(writer, trade) # Another trade - - @test writer.record_count == 4 - @test writer.first_ts == Int64(1700000000000000000) - @test writer.last_ts == Int64(1700000002000000000) - - DBN.close_writer!(writer) - - # Read back and verify - records = DBN.read_dbn(output_file) - @test length(records) == 4 - @test isa(records[1], DBN.TradeMsg) - @test isa(records[2], DBN.MBOMsg) - @test isa(records[3], DBN.MBP1Msg) - @test isa(records[4], DBN.TradeMsg) - end - end - - @testset "Test streaming writer with no records" begin - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "empty_stream.dbn") - - writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) - - # Close immediately without writing any records - DBN.close_writer!(writer) - - # File should still be created with header - @test isfile(output_file) - - # Read back - should have no records but valid metadata - open(output_file, "r") do io - decoder = DBN.DBNDecoder(io) - DBN.read_header!(decoder) - @test decoder.metadata !== nothing - @test decoder.metadata.limit === nothing # 0 limit is treated as unlimited/nothing - @test decoder.metadata.start_ts == 0 # Should be 0 for empty files - @test decoder.metadata.end_ts === nothing # Should be nothing for empty files - end - - records = DBN.read_dbn(output_file) - @test isempty(records) - end - end - - @testset "Test streaming writer error handling" begin - mktempdir() do tmpdir - output_file = joinpath(tmpdir, "error_test.dbn") - - writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) - - # Write a valid record - trade = DBN.TradeMsg( - DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, Int64(1700000000000000000)), - 100000000, 100, DBN.Action.TRADE, DBN.Side.ASK, 0x00, 0, - Int64(1700000000000000000), 10000, 1 - ) - DBN.write_record!(writer, trade) - - # Close the writer - DBN.close_writer!(writer) - - # Try to write after closing - should now throw IOError - @test_throws Base.IOError DBN.write_record!(writer, trade) - - # Verify the first record was written correctly - records = DBN.read_dbn(output_file) - @test length(records) == 1 - @test isa(records[1], DBN.TradeMsg) - end - end -end - + + +@testset "Phase 7: Streaming Writer Testing" begin + + @testset "Test DBNStreamWriter creation" begin + # Test basic creation + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "stream_writer_test.dbn") + + # Create writer with default settings + writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) + @test writer.record_count == 0 + @test writer.first_ts == typemax(Int64) + @test writer.last_ts == 0 + @test writer.auto_flush == true + @test writer.flush_interval == 1000 + @test writer.last_flush_count == 0 + + # Close the writer + DBN.close_writer!(writer) + + # Verify file was created + @test isfile(output_file) + + # Test creation with custom parameters + output_file2 = joinpath(tmpdir, "stream_writer_custom.dbn") + writer2 = DBN.DBNStreamWriter(output_file2, "XBTS", DBN.Schema.MBO, + symbols=["AAPL", "MSFT"], + auto_flush=false, + flush_interval=500) + @test writer2.auto_flush == false + @test writer2.flush_interval == 500 + @test writer2.encoder.metadata.symbols == ["AAPL", "MSFT"] + @test writer2.encoder.metadata.dataset == "XBTS" + @test writer2.encoder.metadata.schema == DBN.Schema.MBO + + DBN.close_writer!(writer2) + end + end + + @testset "Test write_record! with timestamp tracking" begin + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "timestamp_tracking.dbn") + + writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) + + # Create test trade messages with different timestamps + ts1 = Int64(1700000000000000000) # First timestamp + ts2 = Int64(1700000001000000000) # Second timestamp (1 second later) + ts3 = Int64(1700000002000000000) # Third timestamp (2 seconds later) + + trade1 = DBN.TradeMsg( + DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts1), + 100000000, # price (10.0) + 100, # size + DBN.Action.TRADE, + DBN.Side.ASK, + 0x00, # flags + 0, # depth + ts1, # ts_recv + 10000, # ts_in_delta + 1 # sequence + ) + + trade2 = DBN.TradeMsg( + DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts2), + 101000000, # price (10.1) + 200, + DBN.Action.TRADE, + DBN.Side.BID, + 0x00, + 0, + ts2, + 10000, + 2 + ) + + trade3 = DBN.TradeMsg( + DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts3), + 102000000, # price (10.2) + 150, + DBN.Action.TRADE, + DBN.Side.ASK, + 0x00, + 0, + ts3, + 10000, + 3 + ) + + # Write records and verify timestamp tracking + DBN.write_record!(writer, trade1) + @test writer.record_count == 1 + @test writer.first_ts == ts1 + @test writer.last_ts == ts1 + + DBN.write_record!(writer, trade2) + @test writer.record_count == 2 + @test writer.first_ts == ts1 # Should remain the first + @test writer.last_ts == ts2 # Should update to the latest + + DBN.write_record!(writer, trade3) + @test writer.record_count == 3 + @test writer.first_ts == ts1 # Should still be the first + @test writer.last_ts == ts3 # Should be the latest + + DBN.close_writer!(writer) + + # Read back and verify metadata timestamps + records = DBN.read_dbn(output_file) + @test length(records) == 3 + + # Read metadata + open(output_file, "r") do io + decoder = DBN.DBNDecoder(io) + DBN.read_header!(decoder) + @test decoder.metadata.start_ts == ts1 + @test decoder.metadata.end_ts == ts3 + @test decoder.metadata.limit == 3 + end + end + end + + @testset "Test auto-flush functionality" begin + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "auto_flush_test.dbn") + + # Create writer with small flush interval for testing + writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.MBO, + auto_flush=true, + flush_interval=5) + + # Create a test MBO message + mbo_msg = DBN.MBOMsg( + DBN.RecordHeader(48, DBN.RType.MBO_MSG, 1, 0x00, Int64(1700000000000000000)), + 123456, # order_id + 100000000, # price + 100, # size + 0x00, # flags + 0, # channel_id + DBN.Action.ADD, + DBN.Side.BID, + Int64(1700000000000000000), # ts_recv + 10000, # ts_in_delta + 1 # sequence + ) + + # Write 4 records - should not trigger flush yet + for i in 1:4 + DBN.write_record!(writer, mbo_msg) + end + @test writer.last_flush_count == 0 + + # Write 5th record - should trigger flush + DBN.write_record!(writer, mbo_msg) + @test writer.last_flush_count == 5 + + # Write 4 more records + for i in 1:4 + DBN.write_record!(writer, mbo_msg) + end + @test writer.last_flush_count == 5 # Should not have flushed again + + # Write 10th record - should trigger another flush + DBN.write_record!(writer, mbo_msg) + @test writer.last_flush_count == 10 + + DBN.close_writer!(writer) + + # Verify all records were written + records = DBN.read_dbn(output_file) + @test length(records) == 10 + end + end + + @testset "Test close_writer! and header update" begin + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "header_update_test.dbn") + + # Test with symbols - the parsing issue is now fixed + writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES, + symbols=["AAPL", "MSFT", "GOOGL"]) + + # Create trades with different timestamps + timestamps = [ + Int64(1700000000000000000), + Int64(1700000005000000000), + Int64(1700000003000000000), # Out of order + Int64(1700000010000000000), + Int64(1700000001000000000) # Very early timestamp + ] + + for (i, ts) in enumerate(timestamps) + trade = DBN.TradeMsg( + DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, ts), + 100000000 + i * 1000000, + 100 + i, + DBN.Action.TRADE, + i % 2 == 0 ? DBN.Side.ASK : DBN.Side.BID, + 0x00, + 0, + ts, + 10000, + UInt32(i) + ) + DBN.write_record!(writer, trade) + end + + # Verify timestamps before closing + @test writer.first_ts == Int64(1700000000000000000) # Earliest + @test writer.last_ts == Int64(1700000010000000000) # Latest + @test writer.record_count == 5 + + # Close and update header + DBN.close_writer!(writer) + + # Read back and verify header was updated correctly + open(output_file, "r") do io + decoder = DBN.DBNDecoder(io) + DBN.read_header!(decoder) + metadata = decoder.metadata + + @test metadata.start_ts == Int64(1700000000000000000) + @test metadata.end_ts == Int64(1700000010000000000) + @test metadata.limit == 5 + @test metadata.symbols == ["AAPL", "MSFT", "GOOGL"] + @test metadata.dataset == "XNAS" + @test metadata.schema == DBN.Schema.TRADES + end + + # Verify records are intact + records = DBN.read_dbn(output_file) + @test length(records) == 5 + + # Verify timestamps in the read records + timestamps_read = [r.hd.ts_event for r in records] + @test sort(timestamps_read) == sort(timestamps) + end + end + + @testset "Test mixed record types with streaming writer" begin + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "mixed_records_stream.dbn") + + writer = DBN.DBNStreamWriter(output_file, "GLBX", DBN.Schema.MBO) + + # Create different record types + trade = DBN.TradeMsg( + DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, Int64(1700000000000000000)), + 100000000, 100, DBN.Action.TRADE, DBN.Side.ASK, 0x00, 0, + Int64(1700000000000000000), 10000, 1 + ) + + mbo = DBN.MBOMsg( + DBN.RecordHeader(48, DBN.RType.MBO_MSG, 1, 0x00, Int64(1700000001000000000)), + 12345, # order_id + 101000000, # price + 200, # size + 0x00, # flags + 0, # channel_id + DBN.Action.ADD, + DBN.Side.BID, + Int64(1700000001000000000), # ts_recv + 10000, # ts_in_delta + 2 # sequence + ) + + mbp1 = DBN.MBP1Msg( + DBN.RecordHeader(112, DBN.RType.MBP_1_MSG, 1, 0x00, Int64(1700000002000000000)), + 102000000, 300, DBN.Action.TRADE, DBN.Side.ASK, 0x00, 0, + Int64(1700000002000000000), 10000, 3, + DBN.BidAskPair(100000000, 105000000, 100, 150, 1, 2) + ) + + # Write mixed records + DBN.write_record!(writer, trade) + DBN.write_record!(writer, mbo) + DBN.write_record!(writer, mbp1) + DBN.write_record!(writer, trade) # Another trade + + @test writer.record_count == 4 + @test writer.first_ts == Int64(1700000000000000000) + @test writer.last_ts == Int64(1700000002000000000) + + DBN.close_writer!(writer) + + # Read back and verify + records = DBN.read_dbn(output_file) + @test length(records) == 4 + @test isa(records[1], DBN.TradeMsg) + @test isa(records[2], DBN.MBOMsg) + @test isa(records[3], DBN.MBP1Msg) + @test isa(records[4], DBN.TradeMsg) + end + end + + @testset "Test streaming writer with no records" begin + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "empty_stream.dbn") + + writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) + + # Close immediately without writing any records + DBN.close_writer!(writer) + + # File should still be created with header + @test isfile(output_file) + + # Read back - should have no records but valid metadata + open(output_file, "r") do io + decoder = DBN.DBNDecoder(io) + DBN.read_header!(decoder) + @test decoder.metadata !== nothing + @test decoder.metadata.limit === nothing # 0 limit is treated as unlimited/nothing + @test decoder.metadata.start_ts == 0 # Should be 0 for empty files + @test decoder.metadata.end_ts === nothing # Should be nothing for empty files + end + + records = DBN.read_dbn(output_file) + @test isempty(records) + end + end + + @testset "Test streaming writer error handling" begin + mktempdir() do tmpdir + output_file = joinpath(tmpdir, "error_test.dbn") + + writer = DBN.DBNStreamWriter(output_file, "XNAS", DBN.Schema.TRADES) + + # Write a valid record + trade = DBN.TradeMsg( + DBN.RecordHeader(128, DBN.RType.MBP_0_MSG, 1, 0x00, Int64(1700000000000000000)), + 100000000, 100, DBN.Action.TRADE, DBN.Side.ASK, 0x00, 0, + Int64(1700000000000000000), 10000, 1 + ) + DBN.write_record!(writer, trade) + + # Close the writer + DBN.close_writer!(writer) + + # Try to write after closing - should now throw IOError + @test_throws Base.IOError DBN.write_record!(writer, trade) + + # Verify the first record was written correctly + records = DBN.read_dbn(output_file) + @test length(records) == 1 + @test isa(records[1], DBN.TradeMsg) + end + end +end + println("Phase 7 tests completed!") \ No newline at end of file diff --git a/test/test_phase8.jl b/test/test_phase8.jl index 1a66e9c6..dbeab9f2 100644 --- a/test/test_phase8.jl +++ b/test/test_phase8.jl @@ -1,514 +1,514 @@ -using Dates - -@testset "Phase 8: Missing Functionality Testing" begin - - @testset "DBNStream Iterator Tests" begin - # Create a test file with multiple records - test_file = tempname() * ".dbn" - - # Write test data - metadata = Metadata( - UInt8(DBN_VERSION), - "TEST.BASIC", - Schema.TRADES, - 1609459200000000000, # 2021-01-01 00:00:00 UTC - 1609459260000000000, # 2021-01-01 00:01:00 UTC - UInt64(0), - SType.RAW_SYMBOL, - SType.RAW_SYMBOL, - false, - ["AAPL", "MSFT"], - String[], - String[], - Tuple{String,String,Int64,Int64}[] - ) - - records = [ - TradeMsg( - RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(1), 1609459200000000000), - 1500000000000, # $150.00 - 100, - Action.TRADE, - Side.BID, - 0x00, - 0, - 1609459200000000000, - 0, - 1 - ), - TradeMsg( - RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(2), 1609459210000000000), - 1510000000000, # $151.00 - 200, - Action.TRADE, - Side.ASK, - 0x00, - 0, - 1609459210000000000, - 0, - 2 - ), - TradeMsg( - RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(1), 1609459220000000000), - 1520000000000, # $152.00 - 150, - Action.TRADE, - Side.BID, - 0x00, - 0, - 1609459220000000000, - 0, - 3 - ) - ] - - # Write the test file - write_dbn(test_file, metadata, records) - - @testset "Basic Iterator Functionality" begin - stream = DBNStream(test_file) - - # Test that DBNStream is iterable - @test stream isa DBNStream - @test Base.IteratorSize(typeof(stream)) == Base.SizeUnknown() - @test Base.eltype(typeof(stream)) == Any - - # Test iteration - collected_records = collect(stream) - @test length(collected_records) == 3 - - # Verify records match what we wrote - for (i, record) in enumerate(collected_records) - @test record isa TradeMsg - @test record.hd.instrument_id == records[i].hd.instrument_id - @test record.hd.ts_event == records[i].hd.ts_event - @test record.price == records[i].price - @test record.size == records[i].size - @test record.action == records[i].action - @test record.side == records[i].side - end - end - - @testset "Iterator with for loop" begin - # Test using for loop syntax - record_count = 0 - total_volume = 0 - - for record in DBNStream(test_file) - record_count += 1 - if record isa TradeMsg - total_volume += record.size - end - end - - @test record_count == 3 - @test total_volume == 450 # 100 + 200 + 150 - end - - @testset "Iterator with empty file" begin - empty_file = tempname() * ".dbn" - empty_metadata = Metadata( - UInt8(DBN_VERSION), - "TEST.EMPTY", - Schema.TRADES, - 0, 0, UInt64(0), - SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, - String[], String[], String[], Tuple{String,String,Int64,Int64}[] - ) - - write_dbn(empty_file, empty_metadata, []) - - stream = DBNStream(empty_file) - collected = collect(stream) - @test length(collected) == 0 - - # Clean up - rm(empty_file, force=true) - end - - # TODO: Fix compressed file streaming - # @testset "Iterator with compressed file" begin - # # Create a compressed version - # compressed_file = test_file * ".zst" - # compress_dbn_file(test_file, compressed_file) - # - # # Test iterator works with compressed files - # stream = DBNStream(compressed_file) - # collected_records = collect(stream) - # @test length(collected_records) == 3 - # - # # Verify first record - # @test collected_records[1] isa TradeMsg - # @test collected_records[1].price == 1500000000000 - # - # # Clean up - # rm(compressed_file, force=true) - # end - - @testset "Iterator state management" begin - # Test manual iteration - stream = DBNStream(test_file) - state = iterate(stream) - @test state !== nothing - - record1, iter_state = state - @test record1 isa TradeMsg - - state = iterate(stream, iter_state) - @test state !== nothing - - record2, iter_state = state - @test record2 isa TradeMsg - @test record2.hd.ts_event != record1.hd.ts_event - - state = iterate(stream, iter_state) - @test state !== nothing - - record3, iter_state = state - @test record3 isa TradeMsg - - # Should be at end now - state = iterate(stream, iter_state) - @test state === nothing - end - - # Clean up - rm(test_file, force=true) - end - - @testset "Error/System Message Write Operations" begin - test_file = tempname() * ".dbn" - - metadata = Metadata( - UInt8(DBN_VERSION), - "TEST.MSGS", - Schema.TRADES, - 1609459200000000000, - 1609459260000000000, - UInt64(0), - SType.RAW_SYMBOL, - SType.RAW_SYMBOL, - false, - String[], - String[], - String[], - Tuple{String,String,Int64,Int64}[] - ) - - @testset "ErrorMsg Write/Read" begin - # Create an ErrorMsg. hd.length is in 4-byte units (LENGTH_MULTIPLIER). - err_text = "Connection timeout occurred" - payload_bytes = length(err_text) + 1 # message + null terminator - total_bytes = 16 + payload_bytes - padded = ((total_bytes + 3) ÷ 4) * 4 # round up to 4-byte boundary - err_length = UInt8(padded ÷ 4) - error_msg = ErrorMsg( - RecordHeader(err_length, RType.ERROR_MSG, UInt16(0), UInt32(0), 1609459200000000000), - err_text - ) - - # Write and read back - write_dbn(test_file, metadata, [error_msg]) - records = read_dbn(test_file) - - @test length(records) == 1 - @test records[1] isa ErrorMsg - @test records[1].hd.rtype == RType.ERROR_MSG - @test records[1].err == "Connection timeout occurred" - - rm(test_file, force=true) - end - - @testset "SymbolMappingMsg Write/Read" begin - # DBN v2/v3 SymbolMappingMsg layout (the encoder writes for DBN_VERSION=3): - # header(16) + stype_in(1) + sym[71] + stype_out(1) + sym[71] + start_ts(8) + end_ts(8) = 176 - # hd.length = 176 / 4 = 44 - in_symbol = "AAPL.NASDAQ" - out_symbol = "12345" - sym_length = UInt8(44) - symbol_mapping = SymbolMappingMsg( - RecordHeader(sym_length, RType.SYMBOL_MAPPING_MSG, UInt16(0), UInt32(0), 1609459200000000000), - SType.RAW_SYMBOL, - in_symbol, - SType.INSTRUMENT_ID, - out_symbol, - 1609459200000000000, - 1609459260000000000 - ) - - # Write and read back - write_dbn(test_file, metadata, [symbol_mapping]) - records = read_dbn(test_file) - - @test length(records) == 1 - @test records[1] isa SymbolMappingMsg - @test records[1].hd.rtype == RType.SYMBOL_MAPPING_MSG - @test records[1].stype_in == SType.RAW_SYMBOL - @test records[1].stype_in_symbol == "AAPL.NASDAQ" - @test records[1].stype_out == SType.INSTRUMENT_ID - @test records[1].stype_out_symbol == "12345" - - rm(test_file, force=true) - end - - @testset "SymbolMappingMsg cross-version (v1 wire → v3 file)" begin - # Live Databento gateway emits SymbolMappingMsg in v1 wire layout - # (hd.length=20, 80 bytes total) even when the consumer writes a - # v3 file. The encoder must re-derive the header length from the - # v2+ layout it is actually about to serialize (hd.length=44, - # 176 bytes total) so that the resulting record header matches - # the body and the file remains parseable. - v1_wire_length = UInt8(20) - mapping = SymbolMappingMsg( - RecordHeader(v1_wire_length, RType.SYMBOL_MAPPING_MSG, - UInt16(0), UInt32(1191182337), 1779203015494476543), - SType.INSTRUMENT_ID, - "SPX.OPT", - SType.INSTRUMENT_ID, - "SPX 271217C02800000", - Int64(-1), - Int64(-1), - ) - - # Pair with a follow-up record so we exercise the stream offset: - # if hd.length on the SymbolMappingMsg is wrong, the decoder - # mis-positions for the next read and the file is unparseable. - # TradeMsg on-wire: 16-byte header + 32-byte body = 48 bytes, hd.length = 12 - trade = TradeMsg( - RecordHeader(UInt8(12), RType.MBP_0_MSG, UInt16(0), UInt32(1191182337), - 1779203015494476600), - Int64(100_000_000_000), UInt32(10), - Action.TRADE, Side.BID, 0x00, 0x00, - Int64(1779203015494476600), Int32(0), UInt32(1), - ) - - write_dbn(test_file, metadata, [mapping, trade]) - records = read_dbn(test_file) - - @test length(records) == 2 - @test records[1] isa SymbolMappingMsg - @test records[1].stype_in_symbol == "SPX.OPT" - @test records[1].stype_out_symbol == "SPX 271217C02800000" - @test records[1].hd.instrument_id == UInt32(1191182337) - @test records[1].hd.length == UInt8(44) # rewritten to v3 layout - @test records[2] isa TradeMsg - @test records[2].hd.instrument_id == UInt32(1191182337) - - rm(test_file, force=true) - end - - @testset "SystemMsg Write/Read" begin - # Create a SystemMsg. hd.length is in 4-byte units. - msg_text = "Market open notification" - code_text = "OPEN" - payload_bytes = length(msg_text) + 1 + length(code_text) + 1 # msg + null + code + null - total_bytes = 16 + payload_bytes - padded = ((total_bytes + 3) ÷ 4) * 4 - msg_length = UInt8(padded ÷ 4) - system_msg = SystemMsg( - RecordHeader(msg_length, RType.SYSTEM_MSG, UInt16(0), UInt32(0), 1609459200000000000), - msg_text, - code_text - ) - - # Write and read back - write_dbn(test_file, metadata, [system_msg]) - records = read_dbn(test_file) - - @test length(records) == 1 - @test records[1] isa SystemMsg - @test records[1].hd.rtype == RType.SYSTEM_MSG - @test records[1].msg == "Market open notification" - @test records[1].code == "OPEN" - - rm(test_file, force=true) - end - - @testset "Mixed Message Types Write/Read" begin - # Test writing multiple message types together. hd.length is in 4-byte units. - err_payload = 10 + 1 # "Test error" + null - err_total = 16 + err_payload # 27 → pad to 28 - err_units = UInt8(((err_total + 3) ÷ 4)) # 7 - - sys_payload = 19 + 1 + 4 + 1 # msg + null + code + null - sys_total = 16 + sys_payload # 41 → pad to 44 - sys_units = UInt8(((sys_total + 3) ÷ 4)) # 11 - - messages = [ - ErrorMsg( - RecordHeader(err_units, RType.ERROR_MSG, UInt16(0), UInt32(0), 1609459200000000000), - "Test error" - ), - SystemMsg( - RecordHeader(sys_units, RType.SYSTEM_MSG, UInt16(0), UInt32(0), 1609459210000000000), - "Test system message", - "TEST" - ), - SymbolMappingMsg( - # v3 layout: 16 + 1 + 71 + 1 + 71 + 8 + 8 = 176, /4 = 44 - RecordHeader(UInt8(44), RType.SYMBOL_MAPPING_MSG, UInt16(0), UInt32(0), 1609459220000000000), - SType.RAW_SYMBOL, - "TEST", - SType.INSTRUMENT_ID, - "999", - 1609459200000000000, - 1609459260000000000 - ) - ] - - write_dbn(test_file, metadata, messages) - records = read_dbn(test_file) - - @test length(records) == 3 - @test records[1] isa ErrorMsg - @test records[2] isa SystemMsg - @test records[3] isa SymbolMappingMsg - - # Verify order is preserved - @test records[1].hd.ts_event < records[2].hd.ts_event - @test records[2].hd.ts_event < records[3].hd.ts_event - - rm(test_file, force=true) - end - end - - @testset "Batch Compression Tests" begin - # Create a temporary directory for test files - test_dir = mktempdir() - - try - # Create test files for a specific date - test_date = Date("2024-01-15") - date_str = "2024-01-15" - - # Create sample metadata - metadata = Metadata( - UInt8(DBN_VERSION), - "TEST.BATCH", - Schema.TRADES, - 1609459200000000000, - 1609459260000000000, - UInt64(0), - SType.RAW_SYMBOL, - SType.RAW_SYMBOL, - false, - ["AAPL"], - String[], - String[], - Tuple{String,String,Int64,Int64}[] - ) - - # Create test records - record = TradeMsg( - RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(1), 1609459200000000000), - 1500000000000, - 100, - Action.TRADE, - Side.BID, - 0x00, - 0, - 1609459200000000000, - 0, - 1 - ) - - # Create multiple test files with the date pattern - test_files = [ - joinpath(test_dir, "$(date_str)_trades.dbn"), - joinpath(test_dir, "$(date_str)_mbp1.dbn"), - joinpath(test_dir, "$(date_str)_ohlcv.dbn") - ] - - for file in test_files - write_dbn(file, metadata, [record]) - end - - # Also create a file that shouldn't match the pattern - other_file = joinpath(test_dir, "2024-01-16_trades.dbn") - write_dbn(other_file, metadata, [record]) - - @testset "compress_daily_files basic functionality" begin - # Test compressing files for the specific date - results = compress_daily_files(test_date, test_dir) - - # Should have compressed 3 files (not the 2024-01-16 file) - @test length(results) == 3 - - # Check that compressed files exist - for file in test_files - compressed_file = replace(file, ".dbn" => ".dbn.zst") - @test isfile(compressed_file) - @test !isfile(file) # Original should be deleted - end - - # Check that the other date file was not touched - @test isfile(other_file) - - # Verify compression results - for result in results - @test result !== nothing - @test haskey(result, :original_size) - @test haskey(result, :compressed_size) - @test haskey(result, :compression_ratio) - @test haskey(result, :space_saved) - @test result[:compression_ratio] > 0 - @test result[:space_saved] > 0 - end - end - - @testset "compress_daily_files with custom pattern" begin - # Create more test files - additional_files = [ - joinpath(test_dir, "custom_$(date_str)_data.dbn"), - joinpath(test_dir, "symbols_$(date_str).dbn") - ] - - for file in additional_files - write_dbn(file, metadata, [record]) - end - - # Test with custom pattern - need to create regex dynamically - custom_pattern = Regex(".*" * date_str * ".*\\.dbn\$") - results = compress_daily_files(test_date, test_dir, pattern=custom_pattern) - - # Should compress all files containing the date - @test length(results) == 2 - - # Check compressed files exist - for file in additional_files - compressed_file = replace(file, ".dbn" => ".dbn.zst") - @test isfile(compressed_file) - @test !isfile(file) - end - end - - @testset "compress_daily_files with no matching files" begin - # Test with a date that has no files - future_date = Date("2025-01-01") - results = compress_daily_files(future_date, test_dir) - - # Should return empty results - @test length(results) == 0 - end - - @testset "compress_daily_files error handling" begin - # Create a file with invalid content - bad_file = joinpath(test_dir, "$(date_str)_bad.dbn") - open(bad_file, "w") do f - write(f, "invalid content") - end - - # This should handle the error gracefully - results = compress_daily_files(test_date, test_dir) - - # Should have one failed result (nothing) - failed_count = count(r -> r === nothing, results) - @test failed_count == 1 - end - - finally - # Clean up the test directory - rm(test_dir, recursive=true, force=true) - end - end +using Dates + +@testset "Phase 8: Missing Functionality Testing" begin + + @testset "DBNStream Iterator Tests" begin + # Create a test file with multiple records + test_file = tempname() * ".dbn" + + # Write test data + metadata = Metadata( + UInt8(DBN_VERSION), + "TEST.BASIC", + Schema.TRADES, + 1609459200000000000, # 2021-01-01 00:00:00 UTC + 1609459260000000000, # 2021-01-01 00:01:00 UTC + UInt64(0), + SType.RAW_SYMBOL, + SType.RAW_SYMBOL, + false, + ["AAPL", "MSFT"], + String[], + String[], + Tuple{String,String,Int64,Int64}[] + ) + + records = [ + TradeMsg( + RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(1), 1609459200000000000), + 1500000000000, # $150.00 + 100, + Action.TRADE, + Side.BID, + 0x00, + 0, + 1609459200000000000, + 0, + 1 + ), + TradeMsg( + RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(2), 1609459210000000000), + 1510000000000, # $151.00 + 200, + Action.TRADE, + Side.ASK, + 0x00, + 0, + 1609459210000000000, + 0, + 2 + ), + TradeMsg( + RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(1), 1609459220000000000), + 1520000000000, # $152.00 + 150, + Action.TRADE, + Side.BID, + 0x00, + 0, + 1609459220000000000, + 0, + 3 + ) + ] + + # Write the test file + write_dbn(test_file, metadata, records) + + @testset "Basic Iterator Functionality" begin + stream = DBNStream(test_file) + + # Test that DBNStream is iterable + @test stream isa DBNStream + @test Base.IteratorSize(typeof(stream)) == Base.SizeUnknown() + @test Base.eltype(typeof(stream)) == Any + + # Test iteration + collected_records = collect(stream) + @test length(collected_records) == 3 + + # Verify records match what we wrote + for (i, record) in enumerate(collected_records) + @test record isa TradeMsg + @test record.hd.instrument_id == records[i].hd.instrument_id + @test record.hd.ts_event == records[i].hd.ts_event + @test record.price == records[i].price + @test record.size == records[i].size + @test record.action == records[i].action + @test record.side == records[i].side + end + end + + @testset "Iterator with for loop" begin + # Test using for loop syntax + record_count = 0 + total_volume = 0 + + for record in DBNStream(test_file) + record_count += 1 + if record isa TradeMsg + total_volume += record.size + end + end + + @test record_count == 3 + @test total_volume == 450 # 100 + 200 + 150 + end + + @testset "Iterator with empty file" begin + empty_file = tempname() * ".dbn" + empty_metadata = Metadata( + UInt8(DBN_VERSION), + "TEST.EMPTY", + Schema.TRADES, + 0, 0, UInt64(0), + SType.RAW_SYMBOL, SType.RAW_SYMBOL, false, + String[], String[], String[], Tuple{String,String,Int64,Int64}[] + ) + + write_dbn(empty_file, empty_metadata, []) + + stream = DBNStream(empty_file) + collected = collect(stream) + @test length(collected) == 0 + + # Clean up + rm(empty_file, force=true) + end + + # TODO: Fix compressed file streaming + # @testset "Iterator with compressed file" begin + # # Create a compressed version + # compressed_file = test_file * ".zst" + # compress_dbn_file(test_file, compressed_file) + # + # # Test iterator works with compressed files + # stream = DBNStream(compressed_file) + # collected_records = collect(stream) + # @test length(collected_records) == 3 + # + # # Verify first record + # @test collected_records[1] isa TradeMsg + # @test collected_records[1].price == 1500000000000 + # + # # Clean up + # rm(compressed_file, force=true) + # end + + @testset "Iterator state management" begin + # Test manual iteration + stream = DBNStream(test_file) + state = iterate(stream) + @test state !== nothing + + record1, iter_state = state + @test record1 isa TradeMsg + + state = iterate(stream, iter_state) + @test state !== nothing + + record2, iter_state = state + @test record2 isa TradeMsg + @test record2.hd.ts_event != record1.hd.ts_event + + state = iterate(stream, iter_state) + @test state !== nothing + + record3, iter_state = state + @test record3 isa TradeMsg + + # Should be at end now + state = iterate(stream, iter_state) + @test state === nothing + end + + # Clean up + rm(test_file, force=true) + end + + @testset "Error/System Message Write Operations" begin + test_file = tempname() * ".dbn" + + metadata = Metadata( + UInt8(DBN_VERSION), + "TEST.MSGS", + Schema.TRADES, + 1609459200000000000, + 1609459260000000000, + UInt64(0), + SType.RAW_SYMBOL, + SType.RAW_SYMBOL, + false, + String[], + String[], + String[], + Tuple{String,String,Int64,Int64}[] + ) + + @testset "ErrorMsg Write/Read" begin + # Create an ErrorMsg. hd.length is in 4-byte units (LENGTH_MULTIPLIER). + err_text = "Connection timeout occurred" + payload_bytes = length(err_text) + 1 # message + null terminator + total_bytes = 16 + payload_bytes + padded = ((total_bytes + 3) ÷ 4) * 4 # round up to 4-byte boundary + err_length = UInt8(padded ÷ 4) + error_msg = ErrorMsg( + RecordHeader(err_length, RType.ERROR_MSG, UInt16(0), UInt32(0), 1609459200000000000), + err_text + ) + + # Write and read back + write_dbn(test_file, metadata, [error_msg]) + records = read_dbn(test_file) + + @test length(records) == 1 + @test records[1] isa ErrorMsg + @test records[1].hd.rtype == RType.ERROR_MSG + @test records[1].err == "Connection timeout occurred" + + rm(test_file, force=true) + end + + @testset "SymbolMappingMsg Write/Read" begin + # DBN v2/v3 SymbolMappingMsg layout (the encoder writes for DBN_VERSION=3): + # header(16) + stype_in(1) + sym[71] + stype_out(1) + sym[71] + start_ts(8) + end_ts(8) = 176 + # hd.length = 176 / 4 = 44 + in_symbol = "AAPL.NASDAQ" + out_symbol = "12345" + sym_length = UInt8(44) + symbol_mapping = SymbolMappingMsg( + RecordHeader(sym_length, RType.SYMBOL_MAPPING_MSG, UInt16(0), UInt32(0), 1609459200000000000), + SType.RAW_SYMBOL, + in_symbol, + SType.INSTRUMENT_ID, + out_symbol, + 1609459200000000000, + 1609459260000000000 + ) + + # Write and read back + write_dbn(test_file, metadata, [symbol_mapping]) + records = read_dbn(test_file) + + @test length(records) == 1 + @test records[1] isa SymbolMappingMsg + @test records[1].hd.rtype == RType.SYMBOL_MAPPING_MSG + @test records[1].stype_in == SType.RAW_SYMBOL + @test records[1].stype_in_symbol == "AAPL.NASDAQ" + @test records[1].stype_out == SType.INSTRUMENT_ID + @test records[1].stype_out_symbol == "12345" + + rm(test_file, force=true) + end + + @testset "SymbolMappingMsg cross-version (v1 wire → v3 file)" begin + # Live Databento gateway emits SymbolMappingMsg in v1 wire layout + # (hd.length=20, 80 bytes total) even when the consumer writes a + # v3 file. The encoder must re-derive the header length from the + # v2+ layout it is actually about to serialize (hd.length=44, + # 176 bytes total) so that the resulting record header matches + # the body and the file remains parseable. + v1_wire_length = UInt8(20) + mapping = SymbolMappingMsg( + RecordHeader(v1_wire_length, RType.SYMBOL_MAPPING_MSG, + UInt16(0), UInt32(1191182337), 1779203015494476543), + SType.INSTRUMENT_ID, + "SPX.OPT", + SType.INSTRUMENT_ID, + "SPX 271217C02800000", + Int64(-1), + Int64(-1), + ) + + # Pair with a follow-up record so we exercise the stream offset: + # if hd.length on the SymbolMappingMsg is wrong, the decoder + # mis-positions for the next read and the file is unparseable. + # TradeMsg on-wire: 16-byte header + 32-byte body = 48 bytes, hd.length = 12 + trade = TradeMsg( + RecordHeader(UInt8(12), RType.MBP_0_MSG, UInt16(0), UInt32(1191182337), + 1779203015494476600), + Int64(100_000_000_000), UInt32(10), + Action.TRADE, Side.BID, 0x00, 0x00, + Int64(1779203015494476600), Int32(0), UInt32(1), + ) + + write_dbn(test_file, metadata, [mapping, trade]) + records = read_dbn(test_file) + + @test length(records) == 2 + @test records[1] isa SymbolMappingMsg + @test records[1].stype_in_symbol == "SPX.OPT" + @test records[1].stype_out_symbol == "SPX 271217C02800000" + @test records[1].hd.instrument_id == UInt32(1191182337) + @test records[1].hd.length == UInt8(44) # rewritten to v3 layout + @test records[2] isa TradeMsg + @test records[2].hd.instrument_id == UInt32(1191182337) + + rm(test_file, force=true) + end + + @testset "SystemMsg Write/Read" begin + # Create a SystemMsg. hd.length is in 4-byte units. + msg_text = "Market open notification" + code_text = "OPEN" + payload_bytes = length(msg_text) + 1 + length(code_text) + 1 # msg + null + code + null + total_bytes = 16 + payload_bytes + padded = ((total_bytes + 3) ÷ 4) * 4 + msg_length = UInt8(padded ÷ 4) + system_msg = SystemMsg( + RecordHeader(msg_length, RType.SYSTEM_MSG, UInt16(0), UInt32(0), 1609459200000000000), + msg_text, + code_text + ) + + # Write and read back + write_dbn(test_file, metadata, [system_msg]) + records = read_dbn(test_file) + + @test length(records) == 1 + @test records[1] isa SystemMsg + @test records[1].hd.rtype == RType.SYSTEM_MSG + @test records[1].msg == "Market open notification" + @test records[1].code == "OPEN" + + rm(test_file, force=true) + end + + @testset "Mixed Message Types Write/Read" begin + # Test writing multiple message types together. hd.length is in 4-byte units. + err_payload = 10 + 1 # "Test error" + null + err_total = 16 + err_payload # 27 → pad to 28 + err_units = UInt8(((err_total + 3) ÷ 4)) # 7 + + sys_payload = 19 + 1 + 4 + 1 # msg + null + code + null + sys_total = 16 + sys_payload # 41 → pad to 44 + sys_units = UInt8(((sys_total + 3) ÷ 4)) # 11 + + messages = [ + ErrorMsg( + RecordHeader(err_units, RType.ERROR_MSG, UInt16(0), UInt32(0), 1609459200000000000), + "Test error" + ), + SystemMsg( + RecordHeader(sys_units, RType.SYSTEM_MSG, UInt16(0), UInt32(0), 1609459210000000000), + "Test system message", + "TEST" + ), + SymbolMappingMsg( + # v3 layout: 16 + 1 + 71 + 1 + 71 + 8 + 8 = 176, /4 = 44 + RecordHeader(UInt8(44), RType.SYMBOL_MAPPING_MSG, UInt16(0), UInt32(0), 1609459220000000000), + SType.RAW_SYMBOL, + "TEST", + SType.INSTRUMENT_ID, + "999", + 1609459200000000000, + 1609459260000000000 + ) + ] + + write_dbn(test_file, metadata, messages) + records = read_dbn(test_file) + + @test length(records) == 3 + @test records[1] isa ErrorMsg + @test records[2] isa SystemMsg + @test records[3] isa SymbolMappingMsg + + # Verify order is preserved + @test records[1].hd.ts_event < records[2].hd.ts_event + @test records[2].hd.ts_event < records[3].hd.ts_event + + rm(test_file, force=true) + end + end + + @testset "Batch Compression Tests" begin + # Create a temporary directory for test files + test_dir = mktempdir() + + try + # Create test files for a specific date + test_date = Date("2024-01-15") + date_str = "2024-01-15" + + # Create sample metadata + metadata = Metadata( + UInt8(DBN_VERSION), + "TEST.BATCH", + Schema.TRADES, + 1609459200000000000, + 1609459260000000000, + UInt64(0), + SType.RAW_SYMBOL, + SType.RAW_SYMBOL, + false, + ["AAPL"], + String[], + String[], + Tuple{String,String,Int64,Int64}[] + ) + + # Create test records + record = TradeMsg( + RecordHeader(UInt8(40), RType.MBP_0_MSG, UInt16(1), UInt32(1), 1609459200000000000), + 1500000000000, + 100, + Action.TRADE, + Side.BID, + 0x00, + 0, + 1609459200000000000, + 0, + 1 + ) + + # Create multiple test files with the date pattern + test_files = [ + joinpath(test_dir, "$(date_str)_trades.dbn"), + joinpath(test_dir, "$(date_str)_mbp1.dbn"), + joinpath(test_dir, "$(date_str)_ohlcv.dbn") + ] + + for file in test_files + write_dbn(file, metadata, [record]) + end + + # Also create a file that shouldn't match the pattern + other_file = joinpath(test_dir, "2024-01-16_trades.dbn") + write_dbn(other_file, metadata, [record]) + + @testset "compress_daily_files basic functionality" begin + # Test compressing files for the specific date + results = compress_daily_files(test_date, test_dir) + + # Should have compressed 3 files (not the 2024-01-16 file) + @test length(results) == 3 + + # Check that compressed files exist + for file in test_files + compressed_file = replace(file, ".dbn" => ".dbn.zst") + @test isfile(compressed_file) + @test !isfile(file) # Original should be deleted + end + + # Check that the other date file was not touched + @test isfile(other_file) + + # Verify compression results + for result in results + @test result !== nothing + @test haskey(result, :original_size) + @test haskey(result, :compressed_size) + @test haskey(result, :compression_ratio) + @test haskey(result, :space_saved) + @test result[:compression_ratio] > 0 + @test result[:space_saved] > 0 + end + end + + @testset "compress_daily_files with custom pattern" begin + # Create more test files + additional_files = [ + joinpath(test_dir, "custom_$(date_str)_data.dbn"), + joinpath(test_dir, "symbols_$(date_str).dbn") + ] + + for file in additional_files + write_dbn(file, metadata, [record]) + end + + # Test with custom pattern - need to create regex dynamically + custom_pattern = Regex(".*" * date_str * ".*\\.dbn\$") + results = compress_daily_files(test_date, test_dir, pattern=custom_pattern) + + # Should compress all files containing the date + @test length(results) == 2 + + # Check compressed files exist + for file in additional_files + compressed_file = replace(file, ".dbn" => ".dbn.zst") + @test isfile(compressed_file) + @test !isfile(file) + end + end + + @testset "compress_daily_files with no matching files" begin + # Test with a date that has no files + future_date = Date("2025-01-01") + results = compress_daily_files(future_date, test_dir) + + # Should return empty results + @test length(results) == 0 + end + + @testset "compress_daily_files error handling" begin + # Create a file with invalid content + bad_file = joinpath(test_dir, "$(date_str)_bad.dbn") + open(bad_file, "w") do f + write(f, "invalid content") + end + + # This should handle the error gracefully + results = compress_daily_files(test_date, test_dir) + + # Should have one failed result (nothing) + failed_count = count(r -> r === nothing, results) + @test failed_count == 1 + end + + finally + # Clean up the test directory + rm(test_dir, recursive=true, force=true) + end + end end \ No newline at end of file diff --git a/test/test_phase9_working.jl b/test/test_phase9_working.jl index 58079bc5..dc227cdd 100644 --- a/test/test_phase9_working.jl +++ b/test/test_phase9_working.jl @@ -1,419 +1,419 @@ -using Test +using Test using DatabentoBinaryEncoding import DatabentoBinaryEncoding as DBN -using Dates - -@testset "Phase 9: Edge Cases and Error Handling" begin - - @testset "Invalid/Corrupted Files" begin - @testset "Corrupted DBN header" begin - # Create a file with invalid magic bytes - corrupted_file = tempname() * ".dbn" - try - open(corrupted_file, "w") do io - write(io, b"INVALID_MAGIC") - write(io, zeros(UInt8, 100)) # Random data - end - - @test_throws ErrorException read_dbn(corrupted_file) - @test_throws ErrorException DBNDecoder(corrupted_file) - finally - safe_rm(corrupted_file) - end - end - - @testset "Truncated header" begin - # Create a file with incomplete header - truncated_file = tempname() * ".dbn" - try - open(truncated_file, "w") do io - write(io, b"DBN\x02") # Only write 4 bytes of header - end - - @test_throws Exception read_dbn(truncated_file) - finally - safe_rm(truncated_file) - end - end - - @testset "Invalid version" begin - # Create a file with unsupported version - invalid_version_file = tempname() * ".dbn" - try - open(invalid_version_file, "w") do io - # Write DBN header with invalid version (255) - write(io, b"DBN") - write(io, UInt8(255)) # Invalid version - write(io, zeros(UInt8, 100)) # Pad with zeros - end - - @test_throws Exception read_dbn(invalid_version_file) - finally - safe_rm(invalid_version_file) - end - end - end - - @testset "Empty Files" begin - @testset "Completely empty file" begin - empty_file = tempname() * ".dbn" - try - touch(empty_file) # Create empty file - - @test_throws Exception read_dbn(empty_file) - @test_throws Exception DBNDecoder(empty_file) - finally - safe_rm(empty_file) - end - end - - @testset "File with only header (no records)" begin - header_only_file = tempname() * ".dbn" - try - metadata = Metadata( - UInt8(3), # version - "TEST", # dataset - Schema.TRADES, # schema - 1000000000, # start_ts - 2000000000, # end_ts - UInt64(0), # limit - SType.RAW_SYMBOL, # stype_in - SType.INSTRUMENT_ID, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings - ) - - # Create a header-only file using write_dbn with empty records - write_dbn(header_only_file, metadata, TradeMsg[]) - - # Should be able to read header-only file - records = read_dbn(header_only_file) - @test isempty(records) - - # Streaming should also work - stream_records = collect(DBNStream(header_only_file)) - @test isempty(stream_records) - finally - safe_rm(header_only_file) - end - end - end - - @testset "Boundary Values" begin - @testset "Price boundaries" begin - price_boundary_file = tempname() * ".dbn" - try - metadata = Metadata( - UInt8(3), # version - "TEST", # dataset - Schema.TRADES, # schema - 1000000000, # start_ts - 2000000000, # end_ts - UInt64(0), # limit - SType.RAW_SYMBOL, # stype_in - SType.INSTRUMENT_ID, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings - ) - - # Test with various price boundaries - prices = [ - 0, # Zero price - 1, # Minimum non-zero - typemax(Int64), # Max price - UNDEF_PRICE, # Undefined price - -1000000000 # Negative price (valid in some markets) - ] - - trades = TradeMsg[] - for (i, price) in enumerate(prices) - trade = TradeMsg( - RecordHeader( - UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), - RType.MBP_0_MSG, - UInt16(1), - UInt32(i), - 1500000000 - ), - price, # price - UInt32(100), # size - Action.ADD, # action - Side.BID, # side - UInt8(0), # flags - UInt8(0), # depth - Int64(1500000000), # ts_recv - Int32(0), # ts_in_delta - UInt32(i) # sequence - ) - push!(trades, trade) - end - - # Write and read back - write_dbn(price_boundary_file, metadata, trades) - records = read_dbn(price_boundary_file) - @test length(records) == length(prices) - - for (i, record) in enumerate(records) - @test record.price == prices[i] - - # Test price conversion functions - if prices[i] == UNDEF_PRICE - @test isnan(price_to_float(record.price)) - else - float_price = price_to_float(record.price) - @test !isnan(float_price) - # Round-trip conversion should preserve value (within precision) - @test abs(float_to_price(float_price) - record.price) <= 1 - end - end - finally - safe_rm(price_boundary_file) - end - end - - @testset "Timestamp boundaries" begin - timestamp_file = tempname() * ".dbn" - try - metadata = Metadata( - UInt8(3), # version - "TEST", # dataset - Schema.TRADES, # schema - 0, # start_ts (min timestamp) - typemax(Int64), # end_ts (max timestamp) - UInt64(0), # limit - SType.RAW_SYMBOL, # stype_in - SType.INSTRUMENT_ID, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings - ) - - # Test with various timestamp boundaries - timestamps = [ - 0, # Unix epoch - typemax(Int64), # Max int64 - 1_000_000_000_000_000_000, # 1 second in nanoseconds - UNDEF_TIMESTAMP # Undefined timestamp - ] - - trades = TradeMsg[] - for (i, ts) in enumerate(timestamps) - trade = TradeMsg( - RecordHeader( - UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), - RType.MBP_0_MSG, - UInt16(1), - UInt32(i), - ts - ), - Int64(100000000), # price - UInt32(100), # size - Action.ADD, # action - Side.BID, # side - UInt8(0), # flags - UInt8(0), # depth - ts, # ts_recv - Int32(0), # ts_in_delta - UInt32(i) # sequence - ) - push!(trades, trade) - end - - # Write and read back - write_dbn(timestamp_file, metadata, trades) - records = read_dbn(timestamp_file) - @test length(records) == length(timestamps) - - for (i, record) in enumerate(records) - @test record.hd.ts_event == timestamps[i] - @test record.ts_recv == timestamps[i] - end - finally - safe_rm(timestamp_file) - end - end - end - - @testset "Mixed Record Types" begin - mixed_file = tempname() * ".dbn" - try - metadata = Metadata( - UInt8(3), # version - "TEST", # dataset - Schema.MBO, # schema - 1000000000, # start_ts - 2000000000, # end_ts - UInt64(0), # limit - SType.RAW_SYMBOL, # stype_in - SType.INSTRUMENT_ID, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings - ) - - # Create different record types as vectors - mbo = MBOMsg( - RecordHeader( - UInt8(sizeof(MBOMsg) ÷ DBN.LENGTH_MULTIPLIER), - RType.MBO_MSG, - UInt16(1), - UInt32(100), - 1100000000 - ), - UInt64(1001), # order_id - Int64(100000000), # price - UInt32(100), # size - UInt8(0), # flags - UInt8(0), # channel_id - Action.ADD, # action - Side.BID, # side - Int64(1100000000), # ts_recv - Int32(0), # ts_in_delta - UInt32(1) # sequence - ) - - trade = TradeMsg( - RecordHeader( - UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), - RType.MBP_0_MSG, - UInt16(1), - UInt32(100), - 1200000000 - ), - Int64(101000000), # price - UInt32(50), # size - Action.TRADE, # action - Side.ASK, # side - UInt8(0), # flags - UInt8(0), # depth - Int64(1200000000), # ts_recv - Int32(0), # ts_in_delta - UInt32(2) # sequence - ) - - status = StatusMsg( - RecordHeader( - UInt8(sizeof(StatusMsg) ÷ DBN.LENGTH_MULTIPLIER), - RType.STATUS_MSG, - UInt16(1), - UInt32(100), - 1300000000 - ), - UInt64(1300000000), # ts_recv - UInt16(3), # action (using raw value for HALT) - UInt16(1), # reason - UInt16(2), # trading_event - UInt8(0), # is_trading - UInt8(0), # is_quoting - UInt8(0) # is_short_sell_restricted - ) - - # For now, just test with compatible record types (MBO and Trade work well together) - records = [mbo, trade] # Mixed MBO and Trade messages - - # Write using the unified write function - write_dbn(mixed_file, metadata, records) - - # Read back and verify mixed types - read_records = read_dbn(mixed_file) - @test length(read_records) == 2 - - # Check record types - @test read_records[1] isa MBOMsg - @test read_records[2] isa TradeMsg - - # Verify timestamps are in order - @test read_records[1].hd.ts_event < read_records[2].hd.ts_event - - # Test streaming with mixed types - stream_records = collect(DBNStream(mixed_file)) - @test length(stream_records) == 2 - @test typeof(stream_records[1]) == typeof(read_records[1]) - @test typeof(stream_records[2]) == typeof(read_records[2]) - finally - safe_rm(mixed_file) - end - end - - @testset "Very Large Files" begin - @testset "File with many records" begin - large_file = tempname() * ".dbn" - try - metadata = Metadata( - UInt8(3), # version - "TEST", # dataset - Schema.TRADES, # schema - 1000000000, # start_ts - 2000000000, # end_ts - UInt64(0), # limit - SType.RAW_SYMBOL, # stype_in - SType.INSTRUMENT_ID, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings - ) - - # Write a large number of records - num_records = 1000 # Reduced for faster testing - trades = TradeMsg[] - for i in 1:num_records - trade = TradeMsg( - RecordHeader( - UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), - RType.MBP_0_MSG, - UInt16(1), - UInt32(i % 100 + 1), - 1000000000 + i * 1000 - ), - Int64(100000000 + i), # price - UInt32(i % 1000 + 1), # size - Action.TRADE, # action - i % 2 == 0 ? Side.BID : Side.ASK, # side - UInt8(0), # flags - UInt8(0), # depth - Int64(1000000000 + i * 1000), # ts_recv - Int32(0), # ts_in_delta - UInt32(i) # sequence - ) - push!(trades, trade) - end - - # Write all at once - write_dbn(large_file, metadata, trades) - - # Test streaming read (more memory efficient) - count = 0 - for record in DBNStream(large_file) - count += 1 - @test record isa TradeMsg - @test record.sequence == count - end - @test count == num_records - - # Test file size is reasonable - file_size = filesize(large_file) - expected_size = 300 + num_records * sizeof(TradeMsg) # Approximate header size + records - @test file_size > expected_size * 0.9 # Within 10% of expected - @test file_size < expected_size * 1.1 - finally - safe_rm(large_file) - end - end - end - - @testset "Write Permission Errors" begin +using Dates + +@testset "Phase 9: Edge Cases and Error Handling" begin + + @testset "Invalid/Corrupted Files" begin + @testset "Corrupted DBN header" begin + # Create a file with invalid magic bytes + corrupted_file = tempname() * ".dbn" + try + open(corrupted_file, "w") do io + write(io, b"INVALID_MAGIC") + write(io, zeros(UInt8, 100)) # Random data + end + + @test_throws ErrorException read_dbn(corrupted_file) + @test_throws ErrorException DBNDecoder(corrupted_file) + finally + safe_rm(corrupted_file) + end + end + + @testset "Truncated header" begin + # Create a file with incomplete header + truncated_file = tempname() * ".dbn" + try + open(truncated_file, "w") do io + write(io, b"DBN\x02") # Only write 4 bytes of header + end + + @test_throws Exception read_dbn(truncated_file) + finally + safe_rm(truncated_file) + end + end + + @testset "Invalid version" begin + # Create a file with unsupported version + invalid_version_file = tempname() * ".dbn" + try + open(invalid_version_file, "w") do io + # Write DBN header with invalid version (255) + write(io, b"DBN") + write(io, UInt8(255)) # Invalid version + write(io, zeros(UInt8, 100)) # Pad with zeros + end + + @test_throws Exception read_dbn(invalid_version_file) + finally + safe_rm(invalid_version_file) + end + end + end + + @testset "Empty Files" begin + @testset "Completely empty file" begin + empty_file = tempname() * ".dbn" + try + touch(empty_file) # Create empty file + + @test_throws Exception read_dbn(empty_file) + @test_throws Exception DBNDecoder(empty_file) + finally + safe_rm(empty_file) + end + end + + @testset "File with only header (no records)" begin + header_only_file = tempname() * ".dbn" + try + metadata = Metadata( + UInt8(3), # version + "TEST", # dataset + Schema.TRADES, # schema + 1000000000, # start_ts + 2000000000, # end_ts + UInt64(0), # limit + SType.RAW_SYMBOL, # stype_in + SType.INSTRUMENT_ID, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings + ) + + # Create a header-only file using write_dbn with empty records + write_dbn(header_only_file, metadata, TradeMsg[]) + + # Should be able to read header-only file + records = read_dbn(header_only_file) + @test isempty(records) + + # Streaming should also work + stream_records = collect(DBNStream(header_only_file)) + @test isempty(stream_records) + finally + safe_rm(header_only_file) + end + end + end + + @testset "Boundary Values" begin + @testset "Price boundaries" begin + price_boundary_file = tempname() * ".dbn" + try + metadata = Metadata( + UInt8(3), # version + "TEST", # dataset + Schema.TRADES, # schema + 1000000000, # start_ts + 2000000000, # end_ts + UInt64(0), # limit + SType.RAW_SYMBOL, # stype_in + SType.INSTRUMENT_ID, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings + ) + + # Test with various price boundaries + prices = [ + 0, # Zero price + 1, # Minimum non-zero + typemax(Int64), # Max price + UNDEF_PRICE, # Undefined price + -1000000000 # Negative price (valid in some markets) + ] + + trades = TradeMsg[] + for (i, price) in enumerate(prices) + trade = TradeMsg( + RecordHeader( + UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), + RType.MBP_0_MSG, + UInt16(1), + UInt32(i), + 1500000000 + ), + price, # price + UInt32(100), # size + Action.ADD, # action + Side.BID, # side + UInt8(0), # flags + UInt8(0), # depth + Int64(1500000000), # ts_recv + Int32(0), # ts_in_delta + UInt32(i) # sequence + ) + push!(trades, trade) + end + + # Write and read back + write_dbn(price_boundary_file, metadata, trades) + records = read_dbn(price_boundary_file) + @test length(records) == length(prices) + + for (i, record) in enumerate(records) + @test record.price == prices[i] + + # Test price conversion functions + if prices[i] == UNDEF_PRICE + @test isnan(price_to_float(record.price)) + else + float_price = price_to_float(record.price) + @test !isnan(float_price) + # Round-trip conversion should preserve value (within precision) + @test abs(float_to_price(float_price) - record.price) <= 1 + end + end + finally + safe_rm(price_boundary_file) + end + end + + @testset "Timestamp boundaries" begin + timestamp_file = tempname() * ".dbn" + try + metadata = Metadata( + UInt8(3), # version + "TEST", # dataset + Schema.TRADES, # schema + 0, # start_ts (min timestamp) + typemax(Int64), # end_ts (max timestamp) + UInt64(0), # limit + SType.RAW_SYMBOL, # stype_in + SType.INSTRUMENT_ID, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings + ) + + # Test with various timestamp boundaries + timestamps = [ + 0, # Unix epoch + typemax(Int64), # Max int64 + 1_000_000_000_000_000_000, # 1 second in nanoseconds + UNDEF_TIMESTAMP # Undefined timestamp + ] + + trades = TradeMsg[] + for (i, ts) in enumerate(timestamps) + trade = TradeMsg( + RecordHeader( + UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), + RType.MBP_0_MSG, + UInt16(1), + UInt32(i), + ts + ), + Int64(100000000), # price + UInt32(100), # size + Action.ADD, # action + Side.BID, # side + UInt8(0), # flags + UInt8(0), # depth + ts, # ts_recv + Int32(0), # ts_in_delta + UInt32(i) # sequence + ) + push!(trades, trade) + end + + # Write and read back + write_dbn(timestamp_file, metadata, trades) + records = read_dbn(timestamp_file) + @test length(records) == length(timestamps) + + for (i, record) in enumerate(records) + @test record.hd.ts_event == timestamps[i] + @test record.ts_recv == timestamps[i] + end + finally + safe_rm(timestamp_file) + end + end + end + + @testset "Mixed Record Types" begin + mixed_file = tempname() * ".dbn" + try + metadata = Metadata( + UInt8(3), # version + "TEST", # dataset + Schema.MBO, # schema + 1000000000, # start_ts + 2000000000, # end_ts + UInt64(0), # limit + SType.RAW_SYMBOL, # stype_in + SType.INSTRUMENT_ID, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings + ) + + # Create different record types as vectors + mbo = MBOMsg( + RecordHeader( + UInt8(sizeof(MBOMsg) ÷ DBN.LENGTH_MULTIPLIER), + RType.MBO_MSG, + UInt16(1), + UInt32(100), + 1100000000 + ), + UInt64(1001), # order_id + Int64(100000000), # price + UInt32(100), # size + UInt8(0), # flags + UInt8(0), # channel_id + Action.ADD, # action + Side.BID, # side + Int64(1100000000), # ts_recv + Int32(0), # ts_in_delta + UInt32(1) # sequence + ) + + trade = TradeMsg( + RecordHeader( + UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), + RType.MBP_0_MSG, + UInt16(1), + UInt32(100), + 1200000000 + ), + Int64(101000000), # price + UInt32(50), # size + Action.TRADE, # action + Side.ASK, # side + UInt8(0), # flags + UInt8(0), # depth + Int64(1200000000), # ts_recv + Int32(0), # ts_in_delta + UInt32(2) # sequence + ) + + status = StatusMsg( + RecordHeader( + UInt8(sizeof(StatusMsg) ÷ DBN.LENGTH_MULTIPLIER), + RType.STATUS_MSG, + UInt16(1), + UInt32(100), + 1300000000 + ), + UInt64(1300000000), # ts_recv + UInt16(3), # action (using raw value for HALT) + UInt16(1), # reason + UInt16(2), # trading_event + UInt8(0), # is_trading + UInt8(0), # is_quoting + UInt8(0) # is_short_sell_restricted + ) + + # For now, just test with compatible record types (MBO and Trade work well together) + records = [mbo, trade] # Mixed MBO and Trade messages + + # Write using the unified write function + write_dbn(mixed_file, metadata, records) + + # Read back and verify mixed types + read_records = read_dbn(mixed_file) + @test length(read_records) == 2 + + # Check record types + @test read_records[1] isa MBOMsg + @test read_records[2] isa TradeMsg + + # Verify timestamps are in order + @test read_records[1].hd.ts_event < read_records[2].hd.ts_event + + # Test streaming with mixed types + stream_records = collect(DBNStream(mixed_file)) + @test length(stream_records) == 2 + @test typeof(stream_records[1]) == typeof(read_records[1]) + @test typeof(stream_records[2]) == typeof(read_records[2]) + finally + safe_rm(mixed_file) + end + end + + @testset "Very Large Files" begin + @testset "File with many records" begin + large_file = tempname() * ".dbn" + try + metadata = Metadata( + UInt8(3), # version + "TEST", # dataset + Schema.TRADES, # schema + 1000000000, # start_ts + 2000000000, # end_ts + UInt64(0), # limit + SType.RAW_SYMBOL, # stype_in + SType.INSTRUMENT_ID, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings + ) + + # Write a large number of records + num_records = 1000 # Reduced for faster testing + trades = TradeMsg[] + for i in 1:num_records + trade = TradeMsg( + RecordHeader( + UInt8(sizeof(TradeMsg) ÷ DBN.LENGTH_MULTIPLIER), + RType.MBP_0_MSG, + UInt16(1), + UInt32(i % 100 + 1), + 1000000000 + i * 1000 + ), + Int64(100000000 + i), # price + UInt32(i % 1000 + 1), # size + Action.TRADE, # action + i % 2 == 0 ? Side.BID : Side.ASK, # side + UInt8(0), # flags + UInt8(0), # depth + Int64(1000000000 + i * 1000), # ts_recv + Int32(0), # ts_in_delta + UInt32(i) # sequence + ) + push!(trades, trade) + end + + # Write all at once + write_dbn(large_file, metadata, trades) + + # Test streaming read (more memory efficient) + count = 0 + for record in DBNStream(large_file) + count += 1 + @test record isa TradeMsg + @test record.sequence == count + end + @test count == num_records + + # Test file size is reasonable + file_size = filesize(large_file) + expected_size = 300 + num_records * sizeof(TradeMsg) # Approximate header size + records + @test file_size > expected_size * 0.9 # Within 10% of expected + @test file_size < expected_size * 1.1 + finally + safe_rm(large_file) + end + end + end + + @testset "Write Permission Errors" begin @testset "Read-only directory" begin # Root can write to these paths despite their permissions, so the # assertion is not meaningful in root-run containers. @@ -426,29 +426,29 @@ using Dates for path in readonly_paths if isdir(path) readonly_file = joinpath(path, "test_dbn_readonly.dbn") - - metadata = Metadata( - UInt8(3), # version - "TEST", # dataset - Schema.TRADES, # schema - 1000000000, # start_ts - 2000000000, # end_ts - UInt64(0), # limit - SType.RAW_SYMBOL, # stype_in - SType.INSTRUMENT_ID, # stype_out - false, # ts_out - String[], # symbols - String[], # partial - String[], # not_found - Tuple{String,String,Int64,Int64}[] # mappings - ) - + + metadata = Metadata( + UInt8(3), # version + "TEST", # dataset + Schema.TRADES, # schema + 1000000000, # start_ts + 2000000000, # end_ts + UInt64(0), # limit + SType.RAW_SYMBOL, # stype_in + SType.INSTRUMENT_ID, # stype_out + false, # ts_out + String[], # symbols + String[], # partial + String[], # not_found + Tuple{String,String,Int64,Int64}[] # mappings + ) + # Should throw an error when trying to write @test_throws Exception write_dbn(readonly_file, metadata, TradeMsg[]) break # Only need one successful test end end end - end - end + end + end end diff --git a/test/test_utils.jl b/test/test_utils.jl index b166747b..320a6b24 100644 --- a/test/test_utils.jl +++ b/test/test_utils.jl @@ -1,41 +1,41 @@ -# Test utility functions - -""" - safe_rm(path::String; max_attempts=5, delay=0.1) - -Safely delete a file with retry logic for Windows file locking issues. - -Windows may not immediately release file locks even after close(), -causing "resource busy or locked (EBUSY)" errors. This function: -1. Forces garbage collection to release file handles -2. Retries deletion with exponential backoff -3. Fails gracefully with a warning rather than throwing an error - -# Arguments -- `path::String`: Path to the file to delete -- `max_attempts::Int=5`: Maximum number of deletion attempts -- `delay::Float64=0.1`: Initial delay between attempts (in seconds) -""" -function safe_rm(path::String; max_attempts=5, delay=0.1) - if !isfile(path) - return - end - - # Force garbage collection to ensure file handles are released - GC.gc() - - for attempt in 1:max_attempts - try - rm(path; force=true) - return - catch e - if attempt == max_attempts - @warn "Failed to delete file after $max_attempts attempts: $path" exception=e - # Don't rethrow - we don't want cleanup failures to fail the test - return - end - # Wait a bit for Windows to release the file lock - sleep(delay) - end - end -end +# Test utility functions + +""" + safe_rm(path::String; max_attempts=5, delay=0.1) + +Safely delete a file with retry logic for Windows file locking issues. + +Windows may not immediately release file locks even after close(), +causing "resource busy or locked (EBUSY)" errors. This function: +1. Forces garbage collection to release file handles +2. Retries deletion with exponential backoff +3. Fails gracefully with a warning rather than throwing an error + +# Arguments +- `path::String`: Path to the file to delete +- `max_attempts::Int=5`: Maximum number of deletion attempts +- `delay::Float64=0.1`: Initial delay between attempts (in seconds) +""" +function safe_rm(path::String; max_attempts=5, delay=0.1) + if !isfile(path) + return + end + + # Force garbage collection to ensure file handles are released + GC.gc() + + for attempt in 1:max_attempts + try + rm(path; force=true) + return + catch e + if attempt == max_attempts + @warn "Failed to delete file after $max_attempts attempts: $path" exception=e + # Don't rethrow - we don't want cleanup failures to fail the test + return + end + # Wait a bit for Windows to release the file lock + sleep(delay) + end + end +end