diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8946952..21fb35e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,9 +11,11 @@ updates: prefix: build include: scope groups: - # arrow and parquet pin each other's exact version, so every update of them arrives together, majors too. + # arrow and parquet pin each other's exact version, so every update of them arrives together, majors too. The + # crates are named one by one: a glob left the arrow crates out of the group on the first run. arrow: - patterns: ['arrow*', 'parquet'] + patterns: ['arrow-array', 'arrow-buffer', 'arrow-data', 'arrow-schema', 'parquet'] + update-types: [major, minor, patch] production: dependency-type: production update-types: [minor, patch] diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d6fc02..e951537 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: # The oldest version is the rust-version of Cargo.toml. - rust: ['1.85', stable] + rust: ['1.88', stable] steps: # Pinned to a commit, because a tag can be moved to other code. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/Cargo.lock b/Cargo.lock index f1c36c7..bdc1c26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,15 +18,15 @@ dependencies = [ [[package]] name = "alloc-no-stdlib" -version = "2.0.4" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" [[package]] name = "alloc-stdlib" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" dependencies = [ "alloc-no-stdlib", ] @@ -98,9 +98,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arrow-array" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02ccba2e977a3aabb4384036109ca32f552399a2bc0588f925f91ed073ce70c" +checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" dependencies = [ "ahash", "arrow-buffer", @@ -109,57 +109,52 @@ dependencies = [ "chrono", "half", "hashbrown", - "num", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90f8bece6a9ee316a699fbbfde368a206676a1206ce89b50f07937648e76c3c" +checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] -name = "arrow-cast" -version = "56.2.1" +name = "arrow-cmp" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ffe645cfb4e80b1ca37a3a106ce7b4af66ccdd60c655a57e6b9aab096164a7" +checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687" dependencies = [ "arrow-array", "arrow-buffer", - "arrow-data", "arrow-schema", - "arrow-select", - "atoi", - "base64", - "chrono", - "half", - "lexical-core", - "num", - "ryu", ] [[package]] name = "arrow-data" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78468c813909465dd0f858950c8a0614eb63608134acf95c602ec21381258b28" +checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-ipc" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f88b0fbb33af28089ccd3e4dcd0ff09de46842168d00220b920f7231feddf5" +checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2" dependencies = [ "arrow-array", "arrow-buffer", @@ -171,30 +166,22 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a0d5eb3fe25337ff83e8333a08379bdd1540b0961b1c888f6e505d971c198e1" +checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" [[package]] name = "arrow-select" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2368a78bd32902dba39d52519d70f63799c8b5dc8a9477129a30c2fd3dc70c19" +checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", - "num", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ "num-traits", ] @@ -206,9 +193,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "base64" -version = "0.22.1" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "bitflags" @@ -218,9 +205,9 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "brotli" -version = "8.0.4" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -229,9 +216,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.3" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -243,12 +230,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" @@ -506,9 +487,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -540,12 +521,6 @@ dependencies = [ "cc", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -579,63 +554,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] - [[package]] name = "libc" version = "0.2.189" @@ -671,9 +589,9 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lz4_flex" -version = "0.11.6" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -693,25 +611,11 @@ dependencies = [ "libmimalloc-sys", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" -version = "0.4.8" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" dependencies = [ "num-integer", "num-traits", @@ -735,27 +639,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -778,15 +661,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "pairpack" version = "1.0.0" @@ -806,14 +680,13 @@ dependencies = [ [[package]] name = "parquet" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3abbfef8a25900f4925c86e4cb881ea24672ca3c31ee4fb50a8083c4c56d313" +checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", @@ -826,23 +699,16 @@ dependencies = [ "half", "hashbrown", "lz4_flex", - "num", "num-bigint", - "paste", + "num-integer", + "num-traits", "seq-macro", "simdutf8", "snap", - "thrift", "twox-hash", "zstd", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -933,12 +799,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - [[package]] name = "semver" version = "1.0.28" @@ -1058,17 +918,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "tiny-keccak" version = "2.0.2" @@ -1270,27 +1119,27 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 45fbe37..2b9736b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "pairpack" version = "1.0.0" edition = "2021" -rust-version = "1.85" +rust-version = "1.88" license = "MIT" description = "Store a string column of a parquet shard as character edits of another string column, and rebuild it byte for byte." @@ -16,13 +16,13 @@ path = "src/main.rs" [dependencies] anyhow = "1" -arrow-array = "56" -arrow-buffer = "56" -arrow-data = "56" -arrow-schema = "56" +arrow-array = "60" +arrow-buffer = "60" +arrow-data = "60" +arrow-schema = "60" clap = { version = "4", features = ["derive"] } mimalloc = { version = "0.1", default-features = false } -parquet = { version = "56", features = ["arrow"] } +parquet = { version = "60", features = ["arrow"] } rayon = "1.11" serde_json = "1" diff --git a/src/shard.rs b/src/shard.rs index 32407af..644a781 100644 --- a/src/shard.rs +++ b/src/shard.rs @@ -15,7 +15,7 @@ use arrow_array::{Array, ArrayRef, Int32Array, LargeStringArray, ListArray, Reco use arrow_buffer::{NullBuffer, OffsetBuffer}; use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef}; use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder}; -use parquet::arrow::arrow_writer::{compute_leaves, get_column_writers}; +use parquet::arrow::arrow_writer::{compute_leaves, ArrowRowGroupWriterFactory}; use parquet::arrow::{add_encoded_arrow_schema_to_metadata, ArrowSchemaConverter, ProjectionMask}; use parquet::basic::{Compression, ZstdLevel}; use parquet::column::writer::ColumnCloseResult; @@ -222,7 +222,7 @@ fn arrow_select_concat(schema: &SchemaRef, batches: &[RecordBatch]) -> Result = parts.iter().collect(); let mut mutable = arrow_data::transform::MutableArrayData::new(parts, false, 0); for (k, b) in batches.iter().enumerate() { - mutable.extend(k, 0, b.num_rows()); + mutable.try_extend(k, 0, b.num_rows())?; } columns.push(arrow_array::make_array(mutable.freeze())); } @@ -308,6 +308,7 @@ impl Plan { } let out = File::create(dst).with_context(|| format!("create {}", dst.display()))?; let mut writer = SerializedFileWriter::new(out, self.parquet.root_schema_ptr(), self.properties.clone())?; + let factory = ArrowRowGroupWriterFactory::new(&writer, self.schema.clone()); for group in 0..meta.metadata().num_row_groups() { let group_meta = meta.metadata().row_group(group); let batch = read_group(&file, &meta, group, roots)?; @@ -315,7 +316,7 @@ impl Plan { for (index, array) in make(&batch)? { arrays[index] = Some(array); } - let mut encoders = get_column_writers(&self.parquet, &self.properties, &self.schema)?; + let mut encoders = factory.create_column_writers(group)?; let mut chunks: Vec> = (0..encoders.len()).map(|_| None).collect(); let mut leaf = 0; let mut copies = Vec::new();