From 7f4f987d26f287a518cab857c98f3ffacffdbbfc Mon Sep 17 00:00:00 2001 From: Ciprian-Florin Ifrim <94687473+CiprianFlorin-Ifrim@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:53:25 +0100 Subject: [PATCH 1/2] build(deps): bump arrow and parquet to 60 together The arrow crates and parquet are released in lockstep, and each pins the exact version of the others, which means that a single crate cannot be upgraded on its own. Dependabot nonetheless proposed four separate pull requests, one for each crate, and every one of them failed to resolve, as each left the remaining crates at version 56. The relevant question is therefore not which of the four pull requests to merge, but how to move the whole family at once. The five crates now advance to 60 in a single change, which also replaces get_column_writers, removed in parquet 60, with the column writers that ArrowRowGroupWriterFactory provides. Version 60 raises the minimum supported Rust to 1.88, and the rust-version declared in Cargo.toml moves with the oldest entry of the test matrix, which keeps the two in agreement. A shard of 203,982 rows, repacked under the new version, stores the same 163,704 rows as edits and rebuilds every row against its source. The file is 59 KB smaller, a difference that cannot be attributed to the edits, as their count is unchanged. Rather, it reflects the newer zstd release that version 60 brings with it. --- .github/workflows/tests.yml | 2 +- Cargo.lock | 249 +++++++----------------------------- Cargo.toml | 12 +- src/shard.rs | 7 +- 4 files changed, 60 insertions(+), 210 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d6fc02..e951537 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,7 +61,7 @@ jobs: fail-fast: false matrix: # The oldest version is the rust-version of Cargo.toml. - rust: ['1.85', stable] + rust: ['1.88', stable] steps: # Pinned to a commit, because a tag can be moved to other code. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/Cargo.lock b/Cargo.lock index f1c36c7..bdc1c26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,15 +18,15 @@ dependencies = [ [[package]] name = "alloc-no-stdlib" -version = "2.0.4" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" [[package]] name = "alloc-stdlib" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" dependencies = [ "alloc-no-stdlib", ] @@ -98,9 +98,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arrow-array" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b02ccba2e977a3aabb4384036109ca32f552399a2bc0588f925f91ed073ce70c" +checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" dependencies = [ "ahash", "arrow-buffer", @@ -109,57 +109,52 @@ dependencies = [ "chrono", "half", "hashbrown", - "num", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a90f8bece6a9ee316a699fbbfde368a206676a1206ce89b50f07937648e76c3c" +checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] -name = "arrow-cast" -version = "56.2.1" +name = "arrow-cmp" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61ffe645cfb4e80b1ca37a3a106ce7b4af66ccdd60c655a57e6b9aab096164a7" +checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687" dependencies = [ "arrow-array", "arrow-buffer", - "arrow-data", "arrow-schema", - "arrow-select", - "atoi", - "base64", - "chrono", - "half", - "lexical-core", - "num", - "ryu", ] [[package]] name = "arrow-data" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78468c813909465dd0f858950c8a0614eb63608134acf95c602ec21381258b28" +checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-ipc" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f88b0fbb33af28089ccd3e4dcd0ff09de46842168d00220b920f7231feddf5" +checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2" dependencies = [ "arrow-array", "arrow-buffer", @@ -171,30 +166,22 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a0d5eb3fe25337ff83e8333a08379bdd1540b0961b1c888f6e505d971c198e1" +checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" [[package]] name = "arrow-select" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2368a78bd32902dba39d52519d70f63799c8b5dc8a9477129a30c2fd3dc70c19" +checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", - "num", -] - -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ "num-traits", ] @@ -206,9 +193,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "base64" -version = "0.22.1" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" [[package]] name = "bitflags" @@ -218,9 +205,9 @@ checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" [[package]] name = "brotli" -version = "8.0.4" +version = "9.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +checksum = "f8b851b75c23ca7873623d612fe49bd1989aeb03d08fb9432187eb253d3d4c6b" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -229,9 +216,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.3" +version = "6.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +checksum = "941cd9bd4ddab83cb46fa5a2d428f1c857b24ac78cb876cf7beb710840934bd7" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -243,12 +230,6 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.12.1" @@ -506,9 +487,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.16.1" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "heck" @@ -540,12 +521,6 @@ dependencies = [ "cc", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -579,63 +554,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lexical-core" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" -dependencies = [ - "lexical-parse-integer", - "lexical-util", -] - -[[package]] -name = "lexical-parse-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" -dependencies = [ - "lexical-util", -] - -[[package]] -name = "lexical-util" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" - -[[package]] -name = "lexical-write-float" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" -dependencies = [ - "lexical-util", - "lexical-write-integer", -] - -[[package]] -name = "lexical-write-integer" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" -dependencies = [ - "lexical-util", -] - [[package]] name = "libc" version = "0.2.189" @@ -671,9 +589,9 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lz4_flex" -version = "0.11.6" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -693,25 +611,11 @@ dependencies = [ "libmimalloc-sys", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" -version = "0.4.8" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" dependencies = [ "num-integer", "num-traits", @@ -735,27 +639,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -778,15 +661,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "pairpack" version = "1.0.0" @@ -806,14 +680,13 @@ dependencies = [ [[package]] name = "parquet" -version = "56.2.1" +version = "60.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3abbfef8a25900f4925c86e4cb881ea24672ca3c31ee4fb50a8083c4c56d313" +checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba" dependencies = [ "ahash", "arrow-array", "arrow-buffer", - "arrow-cast", "arrow-data", "arrow-ipc", "arrow-schema", @@ -826,23 +699,16 @@ dependencies = [ "half", "hashbrown", "lz4_flex", - "num", "num-bigint", - "paste", + "num-integer", + "num-traits", "seq-macro", "simdutf8", "snap", - "thrift", "twox-hash", "zstd", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -933,12 +799,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - [[package]] name = "semver" version = "1.0.28" @@ -1058,17 +918,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "tiny-keccak" version = "2.0.2" @@ -1270,27 +1119,27 @@ checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 45fbe37..2b9736b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "pairpack" version = "1.0.0" edition = "2021" -rust-version = "1.85" +rust-version = "1.88" license = "MIT" description = "Store a string column of a parquet shard as character edits of another string column, and rebuild it byte for byte." @@ -16,13 +16,13 @@ path = "src/main.rs" [dependencies] anyhow = "1" -arrow-array = "56" -arrow-buffer = "56" -arrow-data = "56" -arrow-schema = "56" +arrow-array = "60" +arrow-buffer = "60" +arrow-data = "60" +arrow-schema = "60" clap = { version = "4", features = ["derive"] } mimalloc = { version = "0.1", default-features = false } -parquet = { version = "56", features = ["arrow"] } +parquet = { version = "60", features = ["arrow"] } rayon = "1.11" serde_json = "1" diff --git a/src/shard.rs b/src/shard.rs index 32407af..644a781 100644 --- a/src/shard.rs +++ b/src/shard.rs @@ -15,7 +15,7 @@ use arrow_array::{Array, ArrayRef, Int32Array, LargeStringArray, ListArray, Reco use arrow_buffer::{NullBuffer, OffsetBuffer}; use arrow_schema::{DataType, Field, Fields, Schema, SchemaRef}; use parquet::arrow::arrow_reader::{ArrowReaderMetadata, ArrowReaderOptions, ParquetRecordBatchReaderBuilder}; -use parquet::arrow::arrow_writer::{compute_leaves, get_column_writers}; +use parquet::arrow::arrow_writer::{compute_leaves, ArrowRowGroupWriterFactory}; use parquet::arrow::{add_encoded_arrow_schema_to_metadata, ArrowSchemaConverter, ProjectionMask}; use parquet::basic::{Compression, ZstdLevel}; use parquet::column::writer::ColumnCloseResult; @@ -222,7 +222,7 @@ fn arrow_select_concat(schema: &SchemaRef, batches: &[RecordBatch]) -> Result = parts.iter().collect(); let mut mutable = arrow_data::transform::MutableArrayData::new(parts, false, 0); for (k, b) in batches.iter().enumerate() { - mutable.extend(k, 0, b.num_rows()); + mutable.try_extend(k, 0, b.num_rows())?; } columns.push(arrow_array::make_array(mutable.freeze())); } @@ -308,6 +308,7 @@ impl Plan { } let out = File::create(dst).with_context(|| format!("create {}", dst.display()))?; let mut writer = SerializedFileWriter::new(out, self.parquet.root_schema_ptr(), self.properties.clone())?; + let factory = ArrowRowGroupWriterFactory::new(&writer, self.schema.clone()); for group in 0..meta.metadata().num_row_groups() { let group_meta = meta.metadata().row_group(group); let batch = read_group(&file, &meta, group, roots)?; @@ -315,7 +316,7 @@ impl Plan { for (index, array) in make(&batch)? { arrays[index] = Some(array); } - let mut encoders = get_column_writers(&self.parquet, &self.properties, &self.schema)?; + let mut encoders = factory.create_column_writers(group)?; let mut chunks: Vec> = (0..encoders.len()).map(|_| None).collect(); let mut leaf = 0; let mut copies = Vec::new(); From 35b1f7413c955b7001094ec36013daf44ec450ba Mon Sep 17 00:00:00 2001 From: Ciprian-Florin Ifrim <94687473+CiprianFlorin-Ifrim@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:53:25 +0100 Subject: [PATCH 2/2] ci(pairpack): name the arrow crates in the Dependabot group The arrow group was intended to keep the arrow crates and parquet in one pull request, yet on its first run Dependabot placed parquet alone in the group and proposed each arrow crate separately. The failure cannot be attributed to the update types, as the same group admitted the major bump of parquet. Rather, the glob that was meant to match the arrow crates appears not to have selected them, and the group now names all five crates explicitly and admits every update type, which lets a future release of the family arrive as a single pull request that can resolve. --- .github/dependabot.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8946952..21fb35e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,9 +11,11 @@ updates: prefix: build include: scope groups: - # arrow and parquet pin each other's exact version, so every update of them arrives together, majors too. + # arrow and parquet pin each other's exact version, so every update of them arrives together, majors too. The + # crates are named one by one: a glob left the arrow crates out of the group on the first run. arrow: - patterns: ['arrow*', 'parquet'] + patterns: ['arrow-array', 'arrow-buffer', 'arrow-data', 'arrow-schema', 'parquet'] + update-types: [major, minor, patch] production: dependency-type: production update-types: [minor, patch]