From e87b5e2fdc565ffd6fd0f6b421fb4f58ff32db68 Mon Sep 17 00:00:00 2001 From: Anton Borisov <148864665+fresh-borzoni@users.noreply.github.com> Date: Mon, 2 Nov 2026 23:29:25 +0000 Subject: [PATCH] deps: bump arrow and parquet to 59.3.2 (#7419) * deps: bump arrow and parquet to 59.3.2 arrow-rs 59.3.2 includes apache/arrow-rs#11754, the 59.x backport of apache/arrow-rs#11021: DeltaBitPackDecoder::skip reads miniblocks wider than 64 values instead of failing with "cannot skip miniblock of size N". Snowflake writes 512-value miniblocks and Databricks Photon 256, and a native scan reaches skip when it skips a long run of rows, such as a run of Iceberg position deletes. The native and Delta contrib lockfiles move arrow-*, parquet and parquet-variant* from 59.3.0 to 59.3.2. See #7417. * deps: keep prost-build and prost-derive on itertools 0.14.0 cargo update re-picked itertools 0.13.0 for them when moving arrow and parquet. Restore the previous choice, so the lockfiles only change the arrow and parquet crates. * test: update the RSS frame size for arrow-rs 59.3.2 (cherry picked from commit 644a8708839270d382a78715fca7e2b724143588) --- contrib/delta/native/Cargo.lock | 60 ++++++++-------- native/Cargo.lock | 72 +++++++++---------- .../src/writers/rss/rss_partition_writer.rs | 10 +-- 3 files changed, 71 insertions(+), 71 deletions(-) diff --git a/contrib/delta/native/Cargo.lock b/contrib/delta/native/Cargo.lock index dbbbf9dfc41..4e8e8216174 100644 --- a/contrib/delta/native/Cargo.lock +++ b/contrib/delta/native/Cargo.lock @@ -63,9 +63,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arrow" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" +checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df" dependencies = [ "arrow-arith", "arrow-array", @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" +checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef" dependencies = [ "arrow-array", "arrow-buffer", @@ -98,9 +98,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43" dependencies = [ "ahash", "arrow-buffer", @@ -117,9 +117,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92" dependencies = [ "bytes", "half", @@ -129,9 +129,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" +checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85" dependencies = [ "arrow-array", "arrow-buffer", @@ -151,9 +151,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86" +checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698" dependencies = [ "arrow-array", "arrow-cast", @@ -166,9 +166,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba" dependencies = [ "arrow-buffer", "arrow-schema", @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" +checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d" dependencies = [ "arrow-array", "arrow-buffer", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1" +checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb" dependencies = [ "arrow-array", "arrow-buffer", @@ -220,9 +220,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" +checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834" dependencies = [ "arrow-array", "arrow-buffer", @@ -233,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" +checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e" dependencies = [ "arrow-array", "arrow-buffer", @@ -246,9 +246,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" dependencies = [ "serde_core", "serde_json", @@ -256,9 +256,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4" dependencies = [ "ahash", "arrow-array", @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" +checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470" dependencies = [ "arrow-array", "arrow-buffer", @@ -1777,9 +1777,9 @@ dependencies = [ [[package]] name = "parquet" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" +checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680" dependencies = [ "ahash", "arrow-array", diff --git a/native/Cargo.lock b/native/Cargo.lock index 83b63fe46c6..939de12ce4e 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -232,9 +232,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" +checksum = "3f04e0a89cc175eaf3740282537851e6fd9d7276f97730b9d0ea2859c11a85df" dependencies = [ "arrow-arith", "arrow-array", @@ -253,9 +253,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" +checksum = "157943c720919e8f5b6731407ce5cd4c9a5ef47a75bf10edaf24b7081b5763ef" dependencies = [ "arrow-array", "arrow-buffer", @@ -267,9 +267,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +checksum = "72b5b1682924f11ecf6929689c16cbc69efbb6dd141b06ad98ee1cb10d858c43" dependencies = [ "ahash", "arrow-buffer", @@ -287,9 +287,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +checksum = "b5d687afc6c927965a8a759f3b68de91933d9dfa2e1bb6e156bac84b85665e92" dependencies = [ "bytes", "half", @@ -299,9 +299,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" +checksum = "3f23c3379feede42135ef27ca60e67f6e643cd4343ae74e4c8b16c83f55afd85" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,9 +321,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86" +checksum = "85b1984628b091b8319b3c305fce21d04f9e9b2578bb733b0d0108462ae1d698" dependencies = [ "arrow-array", "arrow-cast", @@ -336,9 +336,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +checksum = "bb0a031c7ff5e2bdb443117054fb466af1d4df5b7f0ecbe197713dc8b113b1ba" dependencies = [ "arrow-buffer", "arrow-schema", @@ -349,9 +349,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" +checksum = "f5dd1afadda2eb821aaf851bb4c0adef60c30cc0f1a29bd02fb99dd07fb9f58d" dependencies = [ "arrow-array", "arrow-buffer", @@ -365,9 +365,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1" +checksum = "e2bc4d56a0dad84cc51c5812ccff48b642047a1f3be80349cd84318983e79edb" dependencies = [ "arrow-array", "arrow-buffer", @@ -390,9 +390,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" +checksum = "561b8209faa2cd489764b7fd07d7cb9e8d274d78048c38de59f77c41140cb834" dependencies = [ "arrow-array", "arrow-buffer", @@ -403,9 +403,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" +checksum = "0d8c68b87a569f92b90c46cb0a09d86bd9609ddc9b1bf1fc5a503c8c6907c14e" dependencies = [ "arrow-array", "arrow-buffer", @@ -416,9 +416,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +checksum = "e87acb0c6bf6237fca7a50d7b51e801c98a18a5fe6f3bb34f7d77b85ca165388" dependencies = [ "bitflags 2.13.2", "serde_core", @@ -427,9 +427,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +checksum = "c844141958d805b47d1208e0d12aac53fcf388ce8011639fe82aa492adde0bb4" dependencies = [ "ahash", "arrow-array", @@ -441,9 +441,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" +checksum = "bc130892b1a9552da77971924b390e06cf375d1f7e4344a7b7b40ca4426fa470" dependencies = [ "arrow-array", "arrow-buffer", @@ -4920,9 +4920,9 @@ dependencies = [ [[package]] name = "parquet" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" +checksum = "8eb260d566f2fbdabe356d26aaa5d13403aac29b5b4e44463b8f9bd7b2b12680" dependencies = [ "ahash", "arrow-array", @@ -4958,9 +4958,9 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3950502c5d383bc98ac83424c50ad06214cfb8a8554ef6f19413e71894079288" +checksum = "4d0085fde95ccfae0b8d703acd1245bda8855a0bf996eba848e8e2f9995f8606" dependencies = [ "arrow", "arrow-schema", @@ -4974,9 +4974,9 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c4fc004d60a76d727fe4a383b48e8c4d629384b9d9f0cc359164ec2cbd3326" +checksum = "e1f9036efdea8177c84895f409a442e3960a8ccc642792760d8d48415542bb3a" dependencies = [ "arrow", "arrow-schema", @@ -4991,9 +4991,9 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "59.3.0" +version = "59.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216b1aecab3ee2e8f429ade4dfef24e233d008ee91ce745fb273c39fc5936b85" +checksum = "50ae7a6fdff86cdd5075653039b62e2ba674b0aae9228022cd74dc9cf71d7353" dependencies = [ "arrow-schema", "base64 0.23.1", diff --git a/native/shuffle/src/writers/rss/rss_partition_writer.rs b/native/shuffle/src/writers/rss/rss_partition_writer.rs index eb5774cc5fc..6101a3ded1a 100644 --- a/native/shuffle/src/writers/rss/rss_partition_writer.rs +++ b/native/shuffle/src/writers/rss/rss_partition_writer.rs @@ -1229,7 +1229,7 @@ mod buffer_tests { let block_writer = ShuffleBlockWriter::try_new(batch.schema().as_ref(), crate::CompressionCodec::None) .unwrap(); - let mut output = BoundedBuffer::try_new(67_996).unwrap(); + let mut output = BoundedBuffer::try_new(65_948).unwrap(); block_writer .write_rss_batch( &batch, @@ -1238,11 +1238,11 @@ mod buffer_tests { &Time::default(), ) .unwrap(); - assert_eq!(output.inner.get_ref().len(), 67_996); - assert_eq!(output.inner.get_ref().capacity(), 67_996); + assert_eq!(output.inner.get_ref().len(), 65_948); + assert_eq!(output.inner.get_ref().capacity(), 65_948); assert!(output.write(&[1]).is_err()); assert!(output.exceeded); - assert_eq!(output.minimum_size, 67_997); - assert_eq!(output.inner.get_ref().capacity(), 67_996); + assert_eq!(output.minimum_size, 65_949); + assert_eq!(output.inner.get_ref().capacity(), 65_948); } }