From 78583a8ab4fe1dbd7a987e82073bd22578a224d8 Mon Sep 17 00:00:00 2001 From: baseRT sync Date: Wed, 19 Aug 2026 10:24:10 +1000 Subject: [PATCH 1/3] sync: open-core source for 0.2.3 Generated by tools/sync_public.py from the internal tree at efda25f9. Only allow-listed paths are included; tuned quant profiles and internal tooling are excluded by rule. --- base-convert/Cargo.lock | 2380 +++++++++++++++-- base-convert/Cargo.toml | 18 +- base-convert/crates/base-arch/src/gemma.rs | 10 +- base-convert/crates/base-convert/src/main.rs | 2 +- base-convert/crates/base-hub/Cargo.toml | 8 +- base-convert/crates/base-hub/src/fetch.rs | 412 ++- base-convert/crates/base-hub/src/lib.rs | 1 + bindings/node/package.json | 2 +- bindings/python/baseRT/__init__.py | 2 +- bindings/python/setup.py | 2 +- bindings/rust/baseRT-sys/Cargo.toml | 2 +- bindings/rust/baseRT/Cargo.toml | 2 +- .../swift/Sources/CBaseRT/include/baseRT.h | 2 +- include/baseRT/baseRT.h | 2 +- 14 files changed, 2550 insertions(+), 295 deletions(-) diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 5ab2233..e6c3d7d 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler2" @@ -8,6 +8,24 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "1.0.0" @@ -64,9 +82,76 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "base-arch" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "base-format", @@ -77,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "base-format", @@ -91,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "base-arch", @@ -104,17 +189,17 @@ dependencies = [ "clap", "ed25519-dalek", "half", - "indicatif 0.17.11", + "indicatif", "memmap2", "rand_core 0.6.4", "serde_json", - "sha2", + "sha2 0.10.9", "tempfile", ] [[package]] name = "base-format" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "bitflags", @@ -122,7 +207,7 @@ dependencies = [ "memmap2", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", "xxhash-rust", @@ -130,22 +215,24 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "base-format", "dirs 5.0.1", "hf-hub", + "indicatif", + "reqwest", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tempfile", "ureq", ] [[package]] name = "base-quant" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "base-format", @@ -157,7 +244,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "base-format", @@ -171,7 +258,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "base-format", @@ -179,7 +266,7 @@ dependencies = [ "rand_core 0.6.4", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "tempfile", "thiserror 1.0.69", ] @@ -205,6 +292,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "blake3" +version = "1.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ae7bad254120e9e4c63bafc385310756f90c484eac0e36b8317cf09cb92a77" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if 1.0.4", + "constant_time_eq", + "cpufeatures 0.3.0", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -214,6 +315,51 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bon" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a602c73c7b0148ec6d12af6fd5cc7a46e2eacc8878271a999abac56eed12f561" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.117", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + [[package]] name = "bumpalo" version = "3.20.2" @@ -221,10 +367,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" [[package]] -name = "byteorder" -version = "1.5.0" +name = "bytemuck" +version = "1.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" [[package]] name = "bytes" @@ -239,15 +385,53 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "rand_core 0.10.1", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clap" version = "4.6.1" @@ -279,7 +463,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -288,12 +472,40 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "console" version = "0.15.11" @@ -308,50 +520,71 @@ dependencies = [ ] [[package]] -name = "console" -version = "0.16.3" +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" + +[[package]] +name = "const_panic" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d64e8af5551369d19cf50138de61f1c42074ab970f74e99be916646777f8fc87" +checksum = "9603f79528ece8163c496f8932121cb36cfe46259e9c907bb3d8205139d7caa3" dependencies = [ - "encode_unicode", - "libc", - "unicode-width", - "windows-sys 0.61.2", + "typewit", ] [[package]] -name = "const-oid" -version = "0.9.6" +name = "constant_time_eq" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] -name = "cookie" -version = "0.18.1" +name = "core-foundation" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "percent-encoding", - "time", - "version_check", + "core-foundation-sys", + "libc", ] [[package]] -name = "cookie_store" -version = "0.22.1" +name = "core-foundation" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ - "cookie", - "document-features", - "idna", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "time", - "url", + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "countio" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9702aee5d1d744c01d82f6915644f950f898e014903385464c773b96fefdecb" +dependencies = [ + "futures-io", ] [[package]] @@ -363,13 +596,31 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", ] [[package]] @@ -413,16 +664,35 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctor" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a755b7c2d4af2bdcff7ce1739e2db9a1b81a9b07123d8015786ae03c0980d" +dependencies = [ + "link-section", + "linktime-proc-macro", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ - "cfg-if", - "cpufeatures", + "cfg-if 1.0.4", + "cpufeatures 0.2.17", "curve25519-dalek-derive", - "digest", + "digest 0.10.7", "fiat-crypto", "rustc_version", "subtle", @@ -437,7 +707,41 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.117", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.117", ] [[package]] @@ -446,7 +750,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "zeroize", ] @@ -462,8 +766,19 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid 0.10.2", + "crypto-common 0.2.2", ] [[package]] @@ -516,17 +831,14 @@ checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] -name = "document-features" -version = "0.2.12" +name = "dunce" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" -dependencies = [ - "litrs", -] +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" [[package]] name = "ed25519" @@ -548,7 +860,7 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] @@ -565,6 +877,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if 1.0.4", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -609,6 +930,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -625,36 +952,129 @@ dependencies = [ ] [[package]] -name = "generic-array" -version = "0.14.7" +name = "fs_extra" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ - "typenum", - "version_check", + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", ] [[package]] -name = "getrandom" -version = "0.2.17" +name = "futures-channel" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ - "cfg-if", - "libc", - "wasi", + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "gearhash" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8cf82cf76cd16485e56295a1377c775ce708c9f1a0be6b029076d60a245d213" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", ] [[package]] name = "getrandom" -version = "0.3.4" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", + "js-sys", "libc", - "r-efi 5.3.0", - "wasip2", + "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -663,11 +1083,66 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", + "js-sys", "libc", - "r-efi 6.0.0", + "r-efi", + "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", +] + +[[package]] +name = "git-version" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad568aa3db0fcbc81f2f116137f263d7304f512a1209b35b85150d3ef88ad19" +dependencies = [ + "git-version-macro", +] + +[[package]] +name = "git-version-macro" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "globset" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c34a9410465b45bd9787443bc7370f37735bad04b0f0cd57ff1a3186c98988" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "h2" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", ] [[package]] @@ -676,7 +1151,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crunchy", "serde", "zerocopy", @@ -697,6 +1172,12 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +[[package]] +name = "heapify" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" + [[package]] name = "heck" version = "0.5.0" @@ -705,21 +1186,56 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hf-hub" -version = "0.5.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aef3982638978efa195ff11b305f51f1f22f4f0a6cabee7af79b383ebee6a213" +checksum = "e7ccb6bcc85dec15413ef5949879f9a5497ca4568ed702547eb83fac23376e5c" dependencies = [ - "dirs 6.0.0", + "base64", + "bon", + "bytes", + "futures", + "getrandom 0.2.17", + "globset", + "hf-xet", + "hyper", + "pathdiff", + "percent-encoding", + "reqwest", + "serde", + "serde_json", + "sha2 0.11.0", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tokio-util", + "tracing", + "url", + "wasm-bindgen-futures", +] + +[[package]] +name = "hf-xet" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c237ef4fb0ce1962a5117f8bd8c74454b41629826a9df17d14a1840ca18f0754" +dependencies = [ + "anyhow", + "async-trait", + "bytes", "http", - "indicatif 0.18.4", - "libc", - "log", - "rand", + "more-asserts", "serde", "serde_json", "thiserror 2.0.18", - "ureq", - "windows-sys 0.61.2", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "uuid", + "xet-client", + "xet-core-structures", + "xet-data", + "xet-runtime", ] [[package]] @@ -732,12 +1248,135 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "hybrid-array" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707114b52a152fa7bdb290cd7cd5912d9467273b6d74e21b8d81aca1f8533f6b" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.2.0" @@ -826,6 +1465,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -865,7 +1510,7 @@ version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" dependencies = [ - "console 0.15.11", + "console", "number_prefix", "portable-atomic", "unicode-width", @@ -873,17 +1518,10 @@ dependencies = [ ] [[package]] -name = "indicatif" -version = "0.18.4" +name = "ipnet" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb" -dependencies = [ - "console 0.16.3", - "portable-atomic", - "unicode-width", - "unit-prefix", - "web-time", -] +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" [[package]] name = "is_terminal_polyfill" @@ -891,22 +1529,115 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if 1.0.4", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.2", + "libc", +] + [[package]] name = "js-sys" version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ + "cfg-if 1.0.4", + "futures-util", "once_cell", "wasm-bindgen", ] +[[package]] +name = "konst" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f660d5f887e3562f9ab6f4a14988795b694099d66b4f5dedc02d197ba9becb1d" +dependencies = [ + "const_panic", + "konst_proc_macros", + "typewit", +] + +[[package]] +name = "konst_proc_macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "leb128fmt" version = "0.1.0" @@ -928,6 +1659,18 @@ dependencies = [ "libc", ] +[[package]] +name = "link-section" +version = "0.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c29a617ce3df32c08497bdc1ab6e2376e0b17948ac166a2fbe5977c5954cd9" + +[[package]] +name = "linktime-proc-macro" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e57c38c1e860fd37c604281cdfb1dd2216977fd76a50f85ba2f388ef3219616" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -940,12 +1683,6 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" -[[package]] -name = "litrs" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" - [[package]] name = "log" version = "0.4.29" @@ -953,18 +1690,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] -name = "memchr" -version = "2.8.0" +name = "lru-slab" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] -name = "memmap2" -version = "0.9.10" +name = "lz4_flex" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ - "libc", + "twox-hash", +] + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", ] [[package]] @@ -977,18 +1754,90 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.61.2", +] + +[[package]] +name = "more-asserts" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fafa6961cabd9c63bcd77a45d7e3b7f3b552b70417831fb0f56db717e72407e" + +[[package]] +name = "ntapi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b335231dfd352ffb0f8017f3b6027a4917f7df785ea2143d8af2adc66980ae" +dependencies = [ + "winapi", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "number_prefix" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" +dependencies = [ + "libc", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-system-configuration" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" +dependencies = [ + "objc2-core-foundation", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1001,18 +1850,71 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" +dependencies = [ + "memchr", +] + +[[package]] +name = "pathdiff" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkcs8" version = "0.10.2" @@ -1023,6 +1925,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "portable-atomic" version = "1.13.1" @@ -1044,15 +1952,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -1060,7 +1959,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -1073,44 +1972,86 @@ dependencies = [ ] [[package]] -name = "quote" -version = "1.0.45" +name = "quinn" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" dependencies = [ - "proc-macro2", + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", ] [[package]] -name = "r-efi" -version = "5.3.0" +name = "quinn-proto" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +checksum = "04759210543be93709136e28212294a659ef5001836ff4eab4d663e4529bba83" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.2", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] [[package]] -name = "r-efi" -version = "6.0.0" +name = "quinn-udp" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] [[package]] -name = "rand" -version = "0.9.4" +name = "quote" +version = "1.0.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" dependencies = [ - "rand_chacha", - "rand_core 0.9.5", + "proc-macro2", ] [[package]] -name = "rand_chacha" -version = "0.9.0" +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ - "ppv-lite86", - "rand_core 0.9.5", + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", ] [[package]] @@ -1124,11 +2065,17 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.5" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" dependencies = [ - "getrandom 0.3.4", + "rand_core 0.10.1", ] [[package]] @@ -1151,6 +2098,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "redb" +version = "3.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ba239c1c1693315d3cc0e601db3b3965543afbf48c41730fdca2f069f510f4a" +dependencies = [ + "libc", +] + [[package]] name = "redox_users" version = "0.4.6" @@ -1173,6 +2129,94 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "reqwest-middleware" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3f1384cffa4f274dad2d4ddd73aed32fed8f786d96c6be8aa4e5fd3c3b58" +dependencies = [ + "anyhow", + "async-trait", + "http", + "reqwest", + "thiserror 2.0.18", + "tower-service", +] + [[package]] name = "ring" version = "0.17.14" @@ -1180,13 +2224,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", ] +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1215,6 +2265,7 @@ version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -1224,21 +2275,62 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -1251,39 +2343,92 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] -name = "semver" -version = "1.0.28" +name = "ryu" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] -name = "serde" -version = "1.0.228" +name = "safe-transmute" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] +checksum = "3944826ff8fa8093089aba3acb4ef44b9446a99a16f3bf4e74af3f77d340ab7d" [[package]] -name = "serde_core" -version = "1.0.228" +name = "same-file" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" dependencies = [ - "serde_derive", + "winapi-util", ] [[package]] -name = "serde_derive" -version = "1.0.228" +name = "schannel" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ - "proc-macro2", - "quote", - "syn", + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] @@ -1300,15 +2445,69 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_repr" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3b1629de253c70a0508c3899572da79ca359fdab27c7920ff00406df418906" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "sha2" version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", - "cpufeatures", - "digest", + "cfg-if 1.0.4", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shellexpand" +version = "3.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32824fab5e16e6c4d86dc1ba84489390419a39f97699852b66480bb87d297ed8" +dependencies = [ + "bstr", + "dirs 6.0.0", + "os_str_bytes", ] [[package]] @@ -1332,6 +2531,28 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.2" @@ -1339,14 +2560,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" [[package]] -name = "socks" -version = "0.3.4" +name = "socket2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ - "byteorder", "libc", - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -1365,6 +2585,22 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1377,6 +2613,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "2.0.117" @@ -1388,6 +2630,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1396,7 +2658,42 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "sysinfo" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" +dependencies = [ + "libc", + "memchr", + "ntapi", + "objc2-core-foundation", + "objc2-io-kit", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", ] [[package]] @@ -1438,7 +2735,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1449,48 +2746,301 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if 1.0.4", +] + +[[package]] +name = "time" +version = "0.3.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "time-macros" +version = "0.2.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-retry" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a129d95275ebf4c493ec53bf0f8cd95f5ac161bc4f381700809a54f595d4470" +dependencies = [ + "pin-project-lite", + "rand", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio_with_wasm" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6" +dependencies = [ + "js-sys", + "tokio", + "tokio_with_wasm_proc", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "tokio_with_wasm_proc" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992" +dependencies = [ + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.18", + "time", + "tracing-subscriber", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", ] [[package]] -name = "time" -version = "0.3.51" +name = "tracing-serde" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde_core", - "time-core", - "time-macros", + "serde", + "tracing-core", ] [[package]] -name = "time-core" -version = "0.1.9" +name = "tracing-subscriber" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] [[package]] -name = "time-macros" -version = "0.2.30" +name = "try-lock" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" -dependencies = [ - "num-conv", - "time-core", -] +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] -name = "tinystr" -version = "0.8.3" +name = "twox-hash" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] +checksum = "8464ec13c3691491391d9fce00f6416c9a48e46972f72d7865688be2080192c9" [[package]] name = "typenum" @@ -1498,6 +3048,18 @@ version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ce102ab67701b8526c123c1bab5cbe42d7040ccfd0f64af1a385808d2f43de" +[[package]] +name = "typewit" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "214ca0b2191785cbc06209b9ca1861e048e39b5ba33574b3cedd58363d5bb5f6" + +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1516,12 +3078,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unit-prefix" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" - [[package]] name = "untrusted" version = "0.9.0" @@ -1535,15 +3091,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" dependencies = [ "base64", - "cookie_store", "flate2", "log", "percent-encoding", "rustls", "rustls-pki-types", - "serde", - "serde_json", - "socks", "ureq-proto", "utf8-zero", "webpki-roots", @@ -1573,6 +3125,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8-zero" version = "0.8.1" @@ -1591,18 +3149,63 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cefc03fd367c0c6d4305de1b312cf00248c4114f4a0418ce6a6af769e3b0bd9" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.3+wasi-0.2.9" @@ -1621,19 +3224,38 @@ dependencies = [ "wit-bindgen 0.51.0", ] +[[package]] +name = "wasite" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66fe902b4a6b8028a753d5424909b764ccf79b7a209eac9bf97e59cda9f71a42" +dependencies = [ + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "wasm-bindgen" version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.118" @@ -1653,7 +3275,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -1688,6 +3310,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -1700,6 +3335,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" @@ -1710,6 +3355,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "1.0.8" @@ -1719,6 +3373,19 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "whoami" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626c4bac6755d76ffc12cb01b2eac751db1996b9e0041de9aa02c8c211ddc82c" +dependencies = [ + "libc", + "libredox", + "objc2-system-configuration", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1735,18 +3402,133 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1814,6 +3596,15 @@ dependencies = [ "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -1940,7 +3731,7 @@ dependencies = [ "heck", "indexmap", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -1956,7 +3747,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -2004,6 +3795,147 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xet-client" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b8da8cc70aa2e3c500c0400e012df82c656ab9fca47f9f939fffc5afd89aca" +dependencies = [ + "anyhow", + "async-trait", + "base64", + "bytes", + "crc32fast", + "futures", + "http", + "hyper", + "more-asserts", + "rand", + "redb", + "reqwest", + "reqwest-middleware", + "serde", + "serde_json", + "serde_repr", + "statrs", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-retry", + "tokio_with_wasm", + "tracing", + "url", + "urlencoding", + "web-time", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-core-structures" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73503c223783dccc864abde22115e09d12f190448a0baf58ab2c54bc709e2f99" +dependencies = [ + "async-trait", + "base64", + "blake3", + "bytemuck", + "bytes", + "countio", + "futures", + "futures-util", + "getrandom 0.4.2", + "heapify", + "itertools", + "lz4_flex", + "more-asserts", + "rand", + "regex", + "safe-transmute", + "serde", + "static_assertions", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", + "uuid", + "web-time", + "xet-runtime", +] + +[[package]] +name = "xet-data" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89052ec5dec2187cad30b86af92cc24fd61c4a57a795f1ff7ff5f38d49184eb" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "gearhash", + "http", + "itertools", + "more-asserts", + "rand", + "serde", + "serde_json", + "sha2 0.11.0", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "url", + "uuid", + "web-time", + "xet-client", + "xet-core-structures", + "xet-runtime", +] + +[[package]] +name = "xet-runtime" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af5c60d5eed38ab4c576f4421bae835e7bd07631fb381705605529d2015c106b" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "chrono", + "colored", + "const-str", + "ctor", + "dirs 6.0.0", + "futures", + "git-version", + "humantime", + "konst", + "libc", + "more-asserts", + "oneshot", + "pin-project", + "rand", + "reqwest", + "serde", + "serde_json", + "shellexpand", + "sysinfo", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tokio_with_wasm", + "tracing", + "tracing-appender", + "tracing-subscriber", + "web-time", + "whoami", + "winapi", +] + [[package]] name = "xxhash-rust" version = "0.8.15" @@ -2029,7 +3961,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2050,7 +3982,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2070,7 +4002,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -2110,7 +4042,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index e69167a..95ccd30 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,11 +12,11 @@ members = [ ] [workspace.package] -version = "0.2.2" +version = "0.2.3" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" -rust-version = "1.80" +rust-version = "1.85" [workspace.dependencies] # Internal crates @@ -43,8 +43,14 @@ xxhash-rust = { version = "0.8", features = ["xxh64"] } indicatif = "0.17" rayon = "1" tempfile = "3" -# Sync-only `api::sync` client (ureq 3 → rustls; no tokio/reqwest/OpenSSL). -# 0.5 is required: it resolves HF's relative `Location` redirects, which 0.3.2 -# could not (HF now 307s `resolve/...` to a relative `/api/resolve-cache/...`). -hf-hub = { version = "0.5", default-features = false, features = ["ureq"] } +# 1.0 is required for Xet: `hf-xet` is a hard dependency there, so Xet-backed +# repos (the default for new uploads on the Hub) transfer over the dedup CAS +# path instead of falling back to plain HTTPS. 1.0 dropped the `ureq` feature — +# it is reqwest/tokio underneath — so the sync surface is now the `blocking` +# feature, which parks one current-thread runtime on a background thread. +# `rustls-tls` keeps the build off OpenSSL, as the ureq stack did. +hf-hub = { version = "1.0", default-features = false, features = [ + "blocking", + "rustls-tls", +] } dirs = "5" diff --git a/base-convert/crates/base-arch/src/gemma.rs b/base-convert/crates/base-arch/src/gemma.rs index 61947f8..b33f2e6 100644 --- a/base-convert/crates/base-arch/src/gemma.rs +++ b/base-convert/crates/base-arch/src/gemma.rs @@ -846,15 +846,14 @@ mod tests { g.insert( "gemma4.attention.head_count_kv".into(), KvValue::Array( - std::iter::repeat([ + std::iter::repeat_n([ KvValue::U32(8), KvValue::U32(8), KvValue::U32(8), KvValue::U32(8), KvValue::U32(8), KvValue::U32(2), - ]) - .take(5) + ], 5) .flatten() .collect(), ), @@ -888,15 +887,14 @@ mod tests { g.insert( "gemma4.attention.sliding_window_pattern".into(), KvValue::Array( - std::iter::repeat([ + std::iter::repeat_n([ KvValue::Bool(true), KvValue::Bool(true), KvValue::Bool(true), KvValue::Bool(true), KvValue::Bool(true), KvValue::Bool(false), - ]) - .take(5) + ], 5) .flatten() .collect(), ), diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index 5f935c7..d2c6bd8 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -4943,7 +4943,7 @@ mod gguf_passthrough_tests { let row_bytes = 2 * 144; // 512 elements / 256 per block x 144 B // Give every row a distinct byte pattern so a mis-shuffle shows up. let src: Vec = (0..rows) - .flat_map(|r| std::iter::repeat((r as u8) + 1).take(row_bytes)) + .flat_map(|r| std::iter::repeat_n((r as u8) + 1, row_bytes)) .collect(); let out = unpermute_rope_rows(&info(512, rows as u64, GgmlType::Q4K), &src, n_heads) .unwrap(); diff --git a/base-convert/crates/base-hub/Cargo.toml b/base-convert/crates/base-hub/Cargo.toml index 03a3747..0dd506c 100644 --- a/base-convert/crates/base-hub/Cargo.toml +++ b/base-convert/crates/base-hub/Cargo.toml @@ -13,8 +13,14 @@ sha2.workspace = true base-format.workspace = true hf-hub.workspace = true dirs.workspace = true -# Generic HTTPS GET for the hosted catalog (same ureq 3 / rustls stack as hf-hub). +# Download progress bars: hf-hub 1.0 reports progress as events and draws +# nothing itself (0.5 owned the bar behind `with_progress`). +indicatif.workspace = true +# Generic HTTPS GET for the hosted catalog. ureq = "3" +# The HTTP client handed to hf-hub, so downloads carry our own timeouts. +# Version must track hf-hub's reqwest, or the `Client` types do not unify. +reqwest = { version = "0.13", default-features = false } [dev-dependencies] tempfile.workspace = true diff --git a/base-convert/crates/base-hub/src/fetch.rs b/base-convert/crates/base-hub/src/fetch.rs index e5bdebc..f78ead8 100644 --- a/base-convert/crates/base-hub/src/fetch.rs +++ b/base-convert/crates/base-hub/src/fetch.rs @@ -1,20 +1,52 @@ //! File fetching, abstracted so tests can run without network. //! -//! [`HfFetcher`] is the real implementation over hf-hub's blocking API; its -//! built-in progress bars cover downloads. [`MockFetcher`] copies from a +//! [`HfFetcher`] is the real implementation over hf-hub's blocking API. Which +//! transport a file takes is the Hub's choice, not ours: Xet-backed files go +//! through hf-xet's chunk-deduplicated CAS path, and everything else through +//! the parallel, resumable range downloader in [`crate::download`]. Neither +//! the trait nor its callers see the difference. [`MockFetcher`] copies from a //! local fixture directory so the pull/convert pipeline can be exercised in //! CI with no HuggingFace access. use anyhow::{Context, Result}; +use hf_hub::progress::{DownloadEvent, FileStatus, Progress, ProgressEvent, ProgressHandler}; +use hf_hub::repository::RepoTreeEntry; +use hf_hub::{HFClient, HFClientSync, HFRepositorySync, RepoTypeModel}; +use indicatif::{ProgressBar, ProgressStyle}; +use std::collections::HashMap; use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::time::Duration; /// Download retries hf-hub performs on transient failures (peer disconnects, -/// truncated chunks — routine on multi-GB model pulls). hf-hub's own default is -/// `0`, which turns the very first network hiccup into a hard failure; its retry -/// loop resumes mid-file via Range headers, so opting in makes large pulls -/// resilient. Override with `$BASERT_HF_MAX_RETRIES` (`0` disables retries). +/// truncated chunks — routine on multi-GB model pulls). hf-hub's own default +/// is `3`; ours is higher because a pull that dies at 90% of a 30GB artifact +/// costs far more than a few extra backoffs. The same count bounds per-chunk +/// retries in [`crate::download`]. Override with `$BASERT_HF_MAX_RETRIES` +/// (`0` disables retries). const DEFAULT_HF_MAX_RETRIES: usize = 5; +/// Files at or above this size go through the parallel, resumable path. Below +/// it the whole transfer is shorter than one retry's backoff, so hf-hub's own +/// single-stream download (and its snapshot/symlink cache bookkeeping) is the +/// better trade — this threshold separates `config.json` from a weight shard. +const RANGED_MIN_BYTES: u64 = 32 * 1024 * 1024; + +/// Force every large file through the range downloader, even one the Hub +/// serves over Xet (`$BASERT_HF_FORCE_RANGED`). +/// +/// Two uses. It is how the range path gets exercised against huggingface.co +/// at all — the Hub has migrated essentially every repository to Xet, so the +/// fallback would otherwise only ever run against a non-migrated endpoint or +/// an `$HF_ENDPOINT` mirror. And it is the escape hatch when hf-xet is the +/// thing going wrong on a given link: the same bytes are still reachable over +/// plain ranged HTTPS from the CDN. +fn force_ranged() -> bool { + std::env::var("BASERT_HF_FORCE_RANGED") + .map(|v| !v.is_empty() && v != "0") + .unwrap_or(false) +} + /// Resolve the retry count from `$BASERT_HF_MAX_RETRIES`, falling back to /// [`DEFAULT_HF_MAX_RETRIES`] when the var is unset or unparseable. fn resolve_max_retries() -> usize { @@ -42,9 +74,144 @@ pub trait Fetcher { } } -/// Real fetcher backed by hf-hub's synchronous API. Reads the HF token from -/// `$HF_TOKEN` / `$HUGGING_FACE_HUB_TOKEN`, falling back to the cached login -/// token (`~/.cache/huggingface/token`). +/// Terminal progress for one hf-hub-driven download. +/// +/// Two event families have to land on the same bar. Plain HTTPS transfers +/// report per-file deltas ([`DownloadEvent::Progress`], only the files whose +/// counters moved), so the running total is a sum over remembered per-file +/// positions. Xet transfers report one aggregate for the in-flight batch with +/// no per-file breakdown ([`DownloadEvent::AggregateProgress`]), which sets +/// the position directly. A download uses one family or the other, never both. +/// +/// hf-hub calls handlers from its transfer threads and forbids blocking, so +/// the state is a plain mutex held only long enough to update counters. +struct BarProgress { + label: String, + state: Mutex, +} + +#[derive(Default)] +struct BarState { + bar: Option, + /// Latest `bytes_completed` per file, for summing HTTPS deltas. + per_file: HashMap, +} + +impl BarProgress { + fn new(label: impl Into) -> Self { + Self { + label: label.into(), + state: Mutex::new(BarState::default()), + } + } +} + +/// Shared bar rendering, so the hf-hub-driven and range-driven paths look +/// identical to whoever is watching the pull. +pub(crate) fn bar_style() -> ProgressStyle { + ProgressStyle::with_template( + "{msg:.bold} [{bar:30}] {bytes}/{total_bytes} ({bytes_per_sec}, {eta})", + ) + .unwrap_or_else(|_| ProgressStyle::default_bar()) + .progress_chars("=> ") +} + +impl ProgressHandler for BarProgress { + fn on_progress(&self, event: &ProgressEvent) { + let ProgressEvent::Download(event) = event else { + return; + }; + // A poisoned mutex here means a previous callback panicked; progress + // display is not worth propagating that into the transfer. + let Ok(mut state) = self.state.lock() else { + return; + }; + match event { + DownloadEvent::Start { total_bytes, .. } => { + let bar = ProgressBar::new(*total_bytes); + bar.set_style(bar_style()); + bar.set_message(self.label.clone()); + state.bar = Some(bar); + } + DownloadEvent::Progress { files } => { + for f in files { + // A `Complete` file with an unknown size (a cache hit) must + // not rewind the sum to zero. + let done = match f.status { + FileStatus::Complete if f.total_bytes == 0 => { + *state.per_file.get(&f.filename).unwrap_or(&0) + } + FileStatus::Complete => f.total_bytes, + _ => f.bytes_completed, + }; + state.per_file.insert(f.filename.clone(), done); + } + let total: u64 = state.per_file.values().sum(); + if let Some(bar) = &state.bar { + bar.set_position(total); + } + } + DownloadEvent::AggregateProgress { + bytes_completed, + total_bytes, + .. + } => { + if let Some(bar) = &state.bar { + // The xet batch total is authoritative once it is known: + // dedup means fewer bytes cross the wire than HEAD implied. + if *total_bytes > 0 { + bar.set_length(*total_bytes); + } + bar.set_position(*bytes_completed); + } + } + DownloadEvent::Complete => { + if let Some(bar) = state.bar.take() { + bar.finish_and_clear(); + } + } + } + } +} + +/// Build one hf-hub client pointed at `staging_root`. +/// +/// The reqwest client is ours rather than hf-hub's default so it can carry +/// timeouts: `read_timeout` is what turns a connection that stops delivering +/// bytes — without ever closing — into an error the retry loop can act on, +/// instead of a pull that hangs until someone notices. +fn build_client(staging_root: &Path) -> Result { + let http = reqwest::Client::builder() + .read_timeout(crate::download::read_timeout()) + .connect_timeout(Duration::from_secs(30)) + .user_agent(concat!("baseRT/", env!("CARGO_PKG_VERSION"))) + .build() + .context("building the HTTP client")?; + let mut builder = HFClient::builder() + // Points the whole cache — blobs, snapshots, refs — at our staging + // tree instead of `~/.cache/huggingface/hub`. + .cache_dir(staging_root.to_path_buf()) + .retry_max_attempts(resolve_max_retries()) + .client(http); + // hf-hub resolves `$HF_TOKEN` itself but not the legacy + // `$HUGGING_FACE_HUB_TOKEN`, so both are read here and passed explicitly; + // an unset pair leaves hf-hub's own resolution in place. + if let Some(tok) = std::env::var("HF_TOKEN") + .ok() + .or_else(|| std::env::var("HUGGING_FACE_HUB_TOKEN").ok()) + .filter(|s| !s.is_empty()) + { + builder = builder.token(tok); + } + builder + .build_sync() + .context("initializing HuggingFace API client") +} + +/// Real fetcher backed by hf-hub's blocking API. Reads the HF token from +/// `$HF_TOKEN` / `$HUGGING_FACE_HUB_TOKEN`, falling back to hf-hub's own +/// resolution (`$HF_TOKEN_PATH`, then the cached login token under +/// `$HF_HOME`). /// /// Downloads land in a private staging directory (normally /// `/.src/hf` — see [`crate::cache::hf_staging_dir`]), NOT the @@ -53,61 +220,169 @@ pub trait Fetcher { /// same filesystem as the models root also lets installs move (rename) the /// downloaded bytes instead of copying them. pub struct HfFetcher { - api: hf_hub::api::sync::Api, + client: HFClientSync, staging_root: PathBuf, + /// Memoized tree listings, keyed by `(repo, revision)`. A pull asks about + /// several files in the same repo — the artifact, then its sidecars — and + /// the listing that answers "how big, and is it Xet?" is the same one + /// `list_files` needs. + trees: Mutex>>, +} + +/// What routing needs to know about one remote file. +struct FileFacts { + size: u64, + xet: bool, + /// Content id to key the staged blob on — the LFS oid where there is one + /// (a sha256 of the content), else the git object id. + key: String, } impl HfFetcher { pub fn new(staging_root: impl Into) -> Result { let staging_root = staging_root.into(); - // `ApiBuilder::new()` snapshots the login token from the default HF - // cache location before `with_cache_dir` re-points downloads at our - // staging dir, so `~/.cache/huggingface/token` keeps working. - let mut builder = hf_hub::api::sync::ApiBuilder::new() - .with_progress(true) - .with_retries(resolve_max_retries()) - .with_cache_dir(staging_root.clone()); - if let Some(tok) = std::env::var("HF_TOKEN") - .ok() - .or_else(|| std::env::var("HUGGING_FACE_HUB_TOKEN").ok()) - .filter(|s| !s.is_empty()) - { - builder = builder.with_token(Some(tok)); + let client = build_client(&staging_root)?; + Ok(Self { + client, + staging_root, + trees: Mutex::new(HashMap::new()), + }) + } + + /// The repo's file tree at `revision`, fetched once and remembered. + fn tree(&self, repo: &str, revision: &str) -> Result> { + let key = (repo.to_string(), revision.to_string()); + if let Some(hit) = self.trees.lock().unwrap().get(&key) { + return Ok(hit.clone()); } - let api = builder - .build() - .context("initializing HuggingFace API client")?; - Ok(Self { api, staging_root }) + let entries = self + .repo(repo) + .list_tree() + .revision(revision) + .recursive(true) + .send() + .with_context(|| format!("listing files in {repo}@{revision}"))?; + self.trees.lock().unwrap().insert(key, entries.clone()); + Ok(entries) + } + + /// Size, transport and content id for one file, from the tree listing. + /// + /// The listing is the right source for this, not a HEAD on the resolve + /// URL: `resolve/...` 302s to a CDN, and hf-hub's public + /// `get_file_metadata` follows that redirect with its normal client, so + /// the headers routing depends on — `X-Repo-Commit`, `X-Xet-Hash` — are + /// gone from the response it reads. (It fails outright on Xet-backed + /// files for exactly that reason.) The tree endpoint answers the same + /// questions with no redirect in the way. + fn facts(&self, repo: &str, revision: &str, filename: &str) -> Result> { + Ok(self + .tree(repo, revision)? + .into_iter() + .find_map(|e| match e { + RepoTreeEntry::File { + path, + size, + oid, + lfs, + xet_hash, + .. + } if path == filename => Some(FileFacts { + size: lfs.as_ref().and_then(|l| l.size).unwrap_or(size), + xet: xet_hash.is_some(), + key: lfs.and_then(|l| l.sha256).unwrap_or(oid), + }), + _ => None, + })) } - fn repo(&self, repo: &str, revision: &str) -> hf_hub::api::sync::ApiRepo { - self.api.repo(hf_hub::Repo::with_revision( - repo.to_string(), - hf_hub::RepoType::Model, - revision.to_string(), - )) + fn repo(&self, repo: &str) -> HFRepositorySync { + let (owner, name) = hf_hub::split_id(repo); + self.client.model(owner, name) + } + + /// Where the parallel path parks a finished blob: the same `blobs/` + /// directory hf-hub uses, so one `staging_dir` still covers every byte we + /// fetched however it arrived. + fn blob_path(&self, repo: &str, etag: &str) -> PathBuf { + let safe: String = etag + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' || c == '_' { + c + } else { + '_' + } + }) + .collect(); + self.staging_dir(repo) + .unwrap_or_else(|| self.staging_root.clone()) + .join("blobs") + .join(safe) } } impl Fetcher for HfFetcher { fn get_file(&self, repo: &str, revision: &str, filename: &str) -> Result { - self.repo(repo, revision) - .get(filename) - .with_context(|| format!("downloading {filename} from {repo}@{revision}")) + let handle = self.repo(repo); + let facts = self.facts(repo, revision, filename)?; + + // Xet-backed: hand it to hf-hub, which routes to hf-xet — already + // chunk-parallel, deduplicated against the local chunk cache, and + // resumable. Re-implementing that on top of range requests would be + // strictly worse. Small files take the same path: not worth a chunk + // plan, and a retry costs less there than planning one. A file the + // listing does not describe goes there too, rather than guessing. + let facts = facts.filter(|f| (!f.xet || force_ranged()) && f.size >= RANGED_MIN_BYTES); + let Some(facts) = facts else { + return handle + .download_file() + .filename(filename) + .revision(revision) + .progress(Progress::new(BarProgress::new(filename.to_string()))) + .send() + .with_context(|| format!("downloading {filename} from {repo}@{revision}")); + }; + + // Plain HTTPS blob, large: parallel and resumable (see `download`). + let dst = self.blob_path(repo, &facts.key); + if std::fs::metadata(&dst).map(|m| m.len()).unwrap_or(0) == facts.size { + return Ok(dst); + } + let staging_root = self.staging_root.clone(); + let mint = move || build_client(&staging_root); + crate::download::download_ranged( + &mint, + repo, + revision, + filename, + facts.size, + &dst, + resolve_max_retries(), + ) + .with_context(|| format!("downloading {filename} from {repo}@{revision}"))?; + Ok(dst) } fn list_files(&self, repo: &str, revision: &str) -> Result> { - let info = self - .repo(repo, revision) - .info() - .with_context(|| format!("fetching repo info for {repo}@{revision}"))?; - Ok(info.siblings.into_iter().map(|s| s.rfilename).collect()) + Ok(self + .tree(repo, revision)? + .into_iter() + .filter_map(|e| match e { + RepoTreeEntry::File { path, .. } => Some(path), + _ => None, + }) + .collect()) } fn staging_dir(&self, repo: &str) -> Option { // hf-hub keeps everything for a repo under `models----`. - let folder = hf_hub::Repo::model(repo.to_string()).folder_name(); - Some(self.staging_root.join(folder)) + // The helper that spells this is crate-private, so the one-line + // mapping is mirrored here. + Some( + self.staging_root + .join(format!("models--{}", repo.replace('/', "--"))), + ) } } @@ -127,8 +402,8 @@ pub fn install_file(fetcher: &dyn Fetcher, repo: &str, src: &Path, dst: &Path) - if owned { // Resolve the snapshot symlink to the actual blob before renaming; // renaming the symlink itself would strand the payload in staging. - let real = std::fs::canonicalize(src) - .with_context(|| format!("resolving {}", src.display()))?; + let real = + std::fs::canonicalize(src).with_context(|| format!("resolving {}", src.display()))?; if std::fs::rename(&real, dst).is_ok() { return Ok(()); } @@ -212,7 +487,8 @@ mod tests { impl StagedFetcher { fn repo_dir(&self, repo: &str) -> PathBuf { - self.staging.join(format!("models--{}", repo.replace('/', "--"))) + self.staging + .join(format!("models--{}", repo.replace('/', "--"))) } /// Materialize a staged download of `filename` with `bytes`. @@ -258,7 +534,9 @@ mod tests { #[test] fn install_moves_owned_blob_then_cleanup_leaves_one_copy() { let tmp = tempfile::tempdir().unwrap(); - let f = StagedFetcher { staging: tmp.path().join("staging") }; + let f = StagedFetcher { + staging: tmp.path().join("staging"), + }; let src = f.stage("org/m", "main", "m-Q4.base", b"payload"); let dst = tmp.path().join("model.base"); @@ -270,16 +548,45 @@ mod tests { assert!(!blob.exists(), "blob must be moved out of staging"); cleanup_staging(&f, "org/m"); - assert!(!f.repo_dir("org/m").exists(), "staging tree must be removed"); + assert!( + !f.repo_dir("org/m").exists(), + "staging tree must be removed" + ); // Exactly one copy survives. assert_eq!(std::fs::read(&dst).unwrap(), b"payload"); } + #[test] + fn install_moves_a_plain_staged_blob() { + // The range downloader parks a plain file in `blobs/`, not a + // snapshot symlink into it — the shape hf-hub's own path produces and + // the one the other install tests cover. Install must move it just + // the same, leaving nothing behind in staging. + let tmp = tempfile::tempdir().unwrap(); + let f = StagedFetcher { + staging: tmp.path().join("staging"), + }; + let blobs = f.repo_dir("org/m").join("blobs"); + std::fs::create_dir_all(&blobs).unwrap(); + let src = blobs.join("sha256-of-the-content"); + std::fs::write(&src, b"ranged-payload").unwrap(); + let dst = tmp.path().join("model.base"); + + install_file(&f, "org/m", &src, &dst).unwrap(); + assert_eq!(std::fs::read(&dst).unwrap(), b"ranged-payload"); + assert!(!src.exists(), "the staged blob must be moved, not copied"); + + cleanup_staging(&f, "org/m"); + assert!(!f.repo_dir("org/m").exists()); + } + #[test] fn install_overwrites_existing_artifact() { // `--force` re-pulls install over an existing model.base. let tmp = tempfile::tempdir().unwrap(); - let f = StagedFetcher { staging: tmp.path().join("staging") }; + let f = StagedFetcher { + staging: tmp.path().join("staging"), + }; let src = f.stage("org/m", "main", "m-Q4.base", b"new-bytes"); let dst = tmp.path().join("model.base"); std::fs::write(&dst, b"old-bytes").unwrap(); @@ -318,7 +625,12 @@ mod tests { // Unset -> the opted-in default (must be > 0, else the retry loop that // makes multi-GB pulls resilient stays disabled — the bug this fixes). std::env::remove_var("BASERT_HF_MAX_RETRIES"); - const { assert!(DEFAULT_HF_MAX_RETRIES > 0, "retries must be opted in by default") }; + const { + assert!( + DEFAULT_HF_MAX_RETRIES > 0, + "retries must be opted in by default" + ) + }; assert_eq!(resolve_max_retries(), DEFAULT_HF_MAX_RETRIES); // A valid override is honored. diff --git a/base-convert/crates/base-hub/src/lib.rs b/base-convert/crates/base-hub/src/lib.rs index 1cb59e0..9f3269b 100644 --- a/base-convert/crates/base-hub/src/lib.rs +++ b/base-convert/crates/base-hub/src/lib.rs @@ -8,6 +8,7 @@ pub mod cache; pub mod catalog; +pub mod download; pub mod fetch; pub mod gen; pub mod registry; diff --git a/bindings/node/package.json b/bindings/node/package.json index c0b78f1..9f78847 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.2.2", + "version": "0.2.3", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index ecb8a8c..1e902bd 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.2.2" +__version__ = "0.2.3" # --------------------------------------------------------------------------- # Library loading diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 4212907..6ec7381 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.2.2", + version="0.2.3", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index 6450a69..f997fcf 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.2.2" +version = "0.2.3" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 97ea571..70f89fb 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.2.2" +version = "0.2.3" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 6456ede..0decfaa 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 2 -#define BASERT_VERSION_PATCH 2 +#define BASERT_VERSION_PATCH 3 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 6456ede..0decfaa 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 2 -#define BASERT_VERSION_PATCH 2 +#define BASERT_VERSION_PATCH 3 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. From acb955ee48d0112ffb0852deacd6c8fec00b4c7b Mon Sep 17 00:00:00 2001 From: prabod Date: Wed, 19 Aug 2026 10:28:24 +1000 Subject: [PATCH 2/3] docs: the 0.2.3 download behavior, and the Rust floor that goes with it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hand-written to accompany the generated sync; none of these paths are in the allow-list, so `tools/sync_public.py` cannot carry them. - Rust 1.85+ in both places that state the floor. hf-hub 1.0 is edition 2024, so 1.80 is now a hard build failure and these two pages are what a new user follows. - The five `BASERT_HF_*` download knobs, in the table that already documents the rest of the environment. - A "How downloads behave" section in the models guide, because the transport is now user-visible. The resume paragraph says exactly what was measured, which is not the tidier claim: a ranged download resumes from its chunk map, a Xet download restarts from the beginning. Killed at 2246 MiB, a Xet pull came back 12s into its next attempt with 328 MiB — a restart, and hf-hub 1.0 exposes no way to hand hf-xet a chunk cache that would change it. Readers planning an overnight pull on a bad link need that difference, and `FORCE_RANGED` is the lever it implies. --- base-convert/crates/base-hub/src/download.rs | 378 +++++++++++++++++++ docs/getting-started/installation.md | 7 +- docs/guides/models.md | 32 ++ docs/index.md | 2 +- 4 files changed, 417 insertions(+), 2 deletions(-) create mode 100644 base-convert/crates/base-hub/src/download.rs diff --git a/base-convert/crates/base-hub/src/download.rs b/base-convert/crates/base-hub/src/download.rs new file mode 100644 index 0000000..ca9727a --- /dev/null +++ b/base-convert/crates/base-hub/src/download.rs @@ -0,0 +1,378 @@ +//! Resumable, multi-connection download of a single Hub file. +//! +//! This exists because neither transport hf-hub offers is safe for the files +//! we pull. A `.base` bundle or a safetensors shard is tens of gigabytes over +//! a link that will not stay up for the whole transfer, and hf-hub 1.0's plain +//! HTTPS path issues one un-ranged `GET` into `File::create`: its retry only +//! wraps the *initial* request, so a peer that disconnects mid-body fails the +//! pull outright, and the next attempt restarts from byte zero. That is the +//! "download stopped halfway" failure. +//! +//! Xet-backed files do not come through here — hf-hub hands those to hf-xet, +//! which already transfers chunk-parallel and resumes from its local chunk +//! cache. This module is the fallback for everything the Hub still serves as +//! plain LFS/HTTPS blobs, and it gives that path the same two properties: +//! +//! * **Parallel.** The file is cut into fixed chunks fetched concurrently over +//! `Range` requests, one connection per worker. A single TCP stream tops out +//! well below the link on a long fat pipe to a CDN edge. +//! * **Resumable.** Chunk completion is recorded in a sidecar bitmap next to +//! the partial file, fsynced as each chunk lands. A pull killed at 90% — +//! crash, `^C`, or a dead link — resumes at 90%, across process restarts. +//! +//! Reads also carry a timeout (`$BASERT_HF_READ_TIMEOUT_SECS`), so a +//! connection that goes quiet without closing is failed and retried rather +//! than hanging the pull forever. +//! +//! Not locked against a second process pulling the same file at the same +//! time. Both would write identical bytes at identical offsets, so the result +//! is not corrupt, but the loser's final rename can fail once the winner has +//! moved the partial away. hf-hub 0.5 did not lock here either, so this is +//! not a regression; it is worth a lock file if concurrent pulls ever become +//! a normal thing to do. + +use anyhow::{bail, Context, Result}; +use hf_hub::{HFClientSync, RepoTypeModel}; +use indicatif::ProgressBar; +use std::fs::{File, OpenOptions}; +use std::os::unix::fs::FileExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; +use std::time::Duration; + +/// Concurrent range requests. Eight saturates a fast link without looking like +/// an attack to the CDN; the Hub's own Python client uses the same order. +const DEFAULT_CONNECTIONS: usize = 8; +/// Bytes per range request. Large enough that per-request overhead vanishes +/// against a multi-GB file, small enough that a failed chunk is cheap to redo +/// and that `connections * chunk` stays a sane peak memory footprint. +const DEFAULT_CHUNK_MB: u64 = 32; +/// Seconds a read may stall before the chunk is failed and retried. +const DEFAULT_READ_TIMEOUT_SECS: u64 = 60; + +/// Read a positive `usize` knob from the environment. +fn env_usize(var: &str, default: usize) -> usize { + std::env::var(var) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(default) +} + +/// Read a positive `u64` knob from the environment. +fn env_u64(var: &str, default: u64) -> u64 { + std::env::var(var) + .ok() + .and_then(|s| s.trim().parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(default) +} + +/// How many range requests to run concurrently (`$BASERT_HF_CONNECTIONS`). +pub fn connections() -> usize { + env_usize("BASERT_HF_CONNECTIONS", DEFAULT_CONNECTIONS) +} + +/// Bytes per range request (`$BASERT_HF_CHUNK_MB`). +pub fn chunk_size() -> u64 { + env_u64("BASERT_HF_CHUNK_MB", DEFAULT_CHUNK_MB) * 1024 * 1024 +} + +/// Per-read stall timeout (`$BASERT_HF_READ_TIMEOUT_SECS`). +pub fn read_timeout() -> Duration { + Duration::from_secs(env_u64( + "BASERT_HF_READ_TIMEOUT_SECS", + DEFAULT_READ_TIMEOUT_SECS, + )) +} + +/// Records which chunks of a partial download are already on disk. +/// +/// One byte per chunk in a sidecar file, synced as each chunk completes. A +/// byte-per-chunk (rather than a packed bitmap) keeps the update a single +/// positioned write with no read-modify-write race between workers. +struct ChunkMap { + file: File, + path: PathBuf, + done: Vec, +} + +impl ChunkMap { + /// Open (or create) the sidecar for `chunks` chunks. A sidecar whose + /// length disagrees with the chunk count is from a different plan — chunk + /// size changed, or the file did — and is discarded rather than trusted. + fn open(path: PathBuf, chunks: usize) -> Result { + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&path) + .with_context(|| format!("opening chunk map {}", path.display()))?; + let len = file.metadata()?.len() as usize; + let done = if len == chunks { + let mut buf = vec![0u8; chunks]; + file.read_exact_at(&mut buf, 0)?; + buf.into_iter().map(|b| b == 1).collect() + } else { + file.set_len(chunks as u64)?; + file.write_all_at(&vec![0u8; chunks], 0)?; + vec![false; chunks] + }; + Ok(Self { file, path, done }) + } + + fn mark(&self, index: usize) -> Result<()> { + self.file.write_all_at(&[1u8], index as u64)?; + // Durability matters more than speed here: the whole point is that a + // kill -9 mid-pull does not cost the bytes already on disk. + self.file.sync_data()?; + Ok(()) + } + + fn remove(self) { + let _ = std::fs::remove_file(&self.path); + } +} + +/// Byte range covered by chunk `index`. +fn chunk_range(index: usize, chunk: u64, size: u64) -> std::ops::Range { + let start = index as u64 * chunk; + let end = (start + chunk).min(size); + start..end +} + +/// Download `filename` at `revision` into `dst`, in parallel and resumably. +/// +/// `size` is the expected length from the Hub's file metadata; the result is +/// verified against it, so a short transfer is an error rather than a +/// truncated model that fails much later at load time with a confusing +/// message. `mint_client` produces one client per worker: `HFClientSync` +/// parks a single-threaded runtime on one background thread, so sharing a +/// client across workers would funnel every range request through one +/// reactor. +pub fn download_ranged( + mint_client: &(dyn Fn() -> Result + Sync), + repo: &str, + revision: &str, + filename: &str, + size: u64, + dst: &Path, + max_retries: usize, +) -> Result<()> { + let chunk = chunk_size(); + let chunks = size.div_ceil(chunk) as usize; + if let Some(parent) = dst.parent() { + std::fs::create_dir_all(parent)?; + } + + let partial = PathBuf::from(format!("{}.partial", dst.display())); + let file = OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(&partial) + .with_context(|| format!("opening {}", partial.display()))?; + // Preallocate so workers can write at their own offsets in any order. + if file.metadata()?.len() != size { + file.set_len(size)?; + } + + let map = ChunkMap::open(PathBuf::from(format!("{}.chunks", dst.display())), chunks)?; + let pending: Vec = (0..chunks).filter(|i| !map.done[*i]).collect(); + let resumed: u64 = (0..chunks) + .filter(|i| map.done[*i]) + .map(|i| { + let r = chunk_range(i, chunk, size); + r.end - r.start + }) + .sum(); + + let bar = ProgressBar::new(size); + bar.set_style(crate::fetch::bar_style()); + bar.set_message(filename.to_string()); + bar.set_position(resumed); + if resumed > 0 { + eprintln!( + "{filename}: resuming at {:.1}% ({}/{} chunks already on disk)", + resumed as f64 / size as f64 * 100.0, + chunks - pending.len(), + chunks + ); + } + + let workers = connections().min(pending.len().max(1)); + let queue = Mutex::new(pending); + let completed = AtomicU64::new(resumed); + let failure: Mutex> = Mutex::new(None); + + std::thread::scope(|scope| { + for _ in 0..workers { + scope.spawn(|| { + // One client — and so one connection pool and one runtime — + // per worker. Built inside the worker so a client that fails + // to build only fails its own share of the chunks. + let client = match mint_client() { + Ok(c) => c, + Err(e) => { + *failure.lock().unwrap() = Some(e); + return; + }, + }; + let (owner, name) = hf_hub::split_id(repo); + let handle: hf_hub::HFRepositorySync = client.model(owner, name); + loop { + if failure.lock().unwrap().is_some() { + return; + } + let Some(index) = queue.lock().unwrap().pop() else { + return; + }; + let range = chunk_range(index, chunk, size); + match fetch_chunk(&handle, revision, filename, range.clone(), max_retries) { + Ok(bytes) => { + let expected = (range.end - range.start) as usize; + if bytes.len() != expected { + *failure.lock().unwrap() = Some(anyhow::anyhow!( + "{filename}: range {}..{} returned {} bytes, expected {expected}", + range.start, + range.end, + bytes.len() + )); + return; + } + if let Err(e) = file + .write_all_at(&bytes, range.start) + .with_context(|| format!("writing {filename} at {}", range.start)) + .and_then(|()| map.mark(index)) + { + *failure.lock().unwrap() = Some(e); + return; + } + let total = completed.fetch_add(expected as u64, Ordering::Relaxed) + + expected as u64; + bar.set_position(total); + }, + Err(e) => { + *failure.lock().unwrap() = Some(e); + return; + }, + } + } + }); + } + }); + + if let Some(e) = failure.into_inner().unwrap() { + // The partial file and its chunk map are deliberately left behind: + // they are what makes the next attempt resume instead of restart. + bar.abandon(); + return Err(e); + } + + file.sync_all()?; + let written = file.metadata()?.len(); + if written != size { + bail!("{filename}: downloaded {written} bytes, expected {size}"); + } + drop(file); + map.remove(); + std::fs::rename(&partial, dst).with_context(|| format!("finalizing {}", dst.display()))?; + bar.finish_and_clear(); + Ok(()) +} + +/// Fetch one range, retrying transient failures with exponential backoff. +fn fetch_chunk( + handle: &hf_hub::HFRepositorySync, + revision: &str, + filename: &str, + range: std::ops::Range, + max_retries: usize, +) -> Result> { + let mut attempt = 0; + loop { + let result = handle + .download_file_to_bytes() + .filename(filename) + .revision(revision) + .range(range.clone()) + .send(); + match result { + Ok(bytes) => return Ok(bytes.to_vec()), + Err(e) => { + if attempt >= max_retries { + return Err(anyhow::Error::new(e).context(format!( + "{filename}: range {}..{} failed after {} attempts", + range.start, + range.end, + attempt + 1 + ))); + } + // 100ms, 200ms, 400ms, … capped: a CDN edge that just dropped + // us needs a moment, not a tight loop. + let backoff = Duration::from_millis(100u64 << attempt.min(6)); + std::thread::sleep(backoff); + attempt += 1; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn chunk_ranges_tile_the_file_exactly() { + let size = 100u64; + let chunk = 30u64; + let chunks = size.div_ceil(chunk) as usize; + assert_eq!(chunks, 4); + let ranges: Vec<_> = (0..chunks).map(|i| chunk_range(i, chunk, size)).collect(); + assert_eq!(ranges[0], 0..30); + assert_eq!(ranges[3], 90..100, "last chunk must be clamped to the size"); + // No gaps, no overlap, full coverage. + let mut next = 0; + for r in &ranges { + assert_eq!(r.start, next); + next = r.end; + } + assert_eq!(next, size); + } + + #[test] + fn chunk_map_persists_completion_across_reopen() { + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("blob.chunks"); + let map = ChunkMap::open(path.clone(), 4).unwrap(); + assert_eq!(map.done, vec![false; 4]); + map.mark(1).unwrap(); + map.mark(2).unwrap(); + drop(map); + + // Reopening is what a resumed pull does: the two finished chunks must + // not be fetched again. + let reopened = ChunkMap::open(path.clone(), 4).unwrap(); + assert_eq!(reopened.done, vec![false, true, true, false]); + reopened.remove(); + assert!(!path.exists()); + } + + #[test] + fn chunk_map_from_a_different_plan_is_discarded() { + // A sidecar sized for another chunk plan must not be reinterpreted: + // its bytes mean different ranges, and trusting them would leave holes + // in the file. + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("blob.chunks"); + let map = ChunkMap::open(path.clone(), 8).unwrap(); + map.mark(7).unwrap(); + drop(map); + + let reopened = ChunkMap::open(path, 4).unwrap(); + assert_eq!(reopened.done, vec![false; 4], "stale plan must reset"); + } +} diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 2c4f69c..6806c5d 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -36,7 +36,7 @@ BaseRT has two parts you put on your `PATH`: ### Requirements - Apple Silicon (M1 or later), macOS 14+. -- Rust 1.80+ (to build the CLI). +- Rust 1.85+ (to build the CLI). - For bindings: Python 3.9+, Node 18+, or Swift 5.9+ as needed. - [`gh`](https://cli.github.com) (optional, to download releases). @@ -88,6 +88,11 @@ basert serve --help | `BASERT_MODELS_DIR` | Where pulled/converted models are cached | `~/.cache/baseRT/models` | | `BASERT_LIB_PATH` / `BASERT_LIB_DIR` | Where bindings look for `libbaseRT.dylib` | `build/` at repo root | | `HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN` | HuggingFace auth for gated/private repos | unset | +| `BASERT_HF_CONNECTIONS` | Parallel range requests when a file is fetched over plain HTTPS | `8` | +| `BASERT_HF_CHUNK_MB` | Bytes per range request | `32` | +| `BASERT_HF_READ_TIMEOUT_SECS` | How long a download may stall before it is retried | `60` | +| `BASERT_HF_MAX_RETRIES` | Retries per chunk (`0` fails on the first error) | `5` | +| `BASERT_HF_FORCE_RANGED` | Use ranged HTTPS even for a Xet-backed file | unset | ## Next steps diff --git a/docs/guides/models.md b/docs/guides/models.md index 4734f3d..16009c7 100644 --- a/docs/guides/models.md +++ b/docs/guides/models.md @@ -35,6 +35,38 @@ When no profile is given, convert-on-pull uses a generic default profile (`default-q4`). Tuned, model-specific quality is delivered through the catalog as pre-converted artifacts. +## How downloads behave + +Model bundles are large — a 30B-class Q4 artifact is around 20 GB — so which +transport a file takes matters. The Hub decides per file, and `basert` follows: + +- **Xet-backed repositories** (the default for new uploads, including the + pre-converted `basecompute/` bundles) transfer over the deduplicated CAS path. + This is the fast one, and there is nothing to enable. +- **Everything else** is fetched over several connections at once, in fixed + chunks, rather than through a single stream. + +Either way, a stalled transfer is cut off and retried rather than left hanging, +and a transfer shorter than the size the Hub advertised is reported as an error +instead of being installed as a truncated model. + +### If a pull is interrupted + +The two transports differ, and the difference is worth knowing before an +overnight pull: + +- A **ranged** download resumes. Chunk completion is recorded beside the partial + file as each chunk lands, so re-running the same `basert pull` continues from + what is already on disk, including after the process has exited. +- A **Xet** download restarts from the beginning. It is fast enough that the + window is much smaller — but it is a restart, not a resume. + +On a link that drops often, `BASERT_HF_FORCE_RANGED=1` takes the ranged path +even for a Xet-backed file, trading throughput for the ability to pick up where +it stopped. The other `BASERT_HF_*` variables in +[Installation](../getting-started/installation.md#environment-variables) tune +connection count, chunk size, and the stall timeout. + ## Listing ```sh diff --git a/docs/index.md b/docs/index.md index ad458d7..3c76128 100644 --- a/docs/index.md +++ b/docs/index.md @@ -61,5 +61,5 @@ basert serve --model Qwen/Qwen3-4B --api-key "$(uuidgen)" # OpenAI server ## Requirements - Apple Silicon (M1 or later), macOS 14+. -- Rust 1.80+ to build the `basert` CLI. +- Rust 1.85+ to build the `basert` CLI. - A binding toolchain as needed (Python 3.9+, Node 18+, Swift 5.9+). From 7c2bb32061775de4f0d541badaca42a840ef9f5a Mon Sep 17 00:00:00 2001 From: baseRT sync Date: Wed, 19 Aug 2026 13:30:13 +1000 Subject: [PATCH 3/3] sync: refresh for the final 0.2.3 tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regenerated from the internal tree at e8a1fa11, which carries two changes that landed after this branch was first written: * downloads are resumable by DEFAULT — the parallel ranged path is now what every large file takes, and Xet is opt-in behind BASERT_HF_XET, because Xet cannot resume; * the catalog is generated by scanning the published org rather than hand-written: 69 rows -> 105, adding every Whisper bundle and Qwen3.8-27B, and correcting eight rows whose size/sha256 no longer matched the Hub (those four models could not be pulled at all). New file: base-hub/src/scan.rs, plus the `catalog-scan` command. The hand-written docs follow the same change: the environment table now documents 24 x 16MB defaults and BASERT_HF_XET in place of the removed BASERT_HF_FORCE_RANGED, and the models guide states plainly that a pull resumes, with Xet described as the deduplicating alternative that does not. Verified in this tree: cargo build --locked --workspace clean, cargo test --workspace 262 passed, clippy clean under -D warnings. --- base-convert/crates/base-convert/src/hub.rs | 73 ++ base-convert/crates/base-convert/src/main.rs | 16 + base-convert/crates/base-hub/catalog.json | 1193 ++++++++++++------ base-convert/crates/base-hub/src/download.rs | 27 +- base-convert/crates/base-hub/src/fetch.rs | 75 +- base-convert/crates/base-hub/src/gen.rs | 36 +- base-convert/crates/base-hub/src/lib.rs | 1 + base-convert/crates/base-hub/src/registry.rs | 30 +- base-convert/crates/base-hub/src/scan.rs | 610 +++++++++ docs/getting-started/installation.md | 6 +- docs/guides/models.md | 52 +- 11 files changed, 1686 insertions(+), 433 deletions(-) create mode 100644 base-convert/crates/base-hub/src/scan.rs diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index 3f45395..c14f829 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -888,6 +888,79 @@ fn human_size(bytes: u64) -> String { } } +/// `basert catalog-scan` — regenerate the catalog from what an organization +/// has actually published. +/// +/// The catalog used to be hand-edited, one JSON row per (model, quant, +/// backend), and drifted from the Hub the moment anyone published without +/// remembering to add a row. This derives it instead: the org listing names +/// the repos, each repo's tree gives size and sha256, and two ranged reads of +/// each bundle give the header that knows its arch, backend and quant. A row +/// whose sha256 is unchanged is carried over untouched, so a rescan is cheap +/// and hand-curated fields (`source_repo`) survive. +pub fn cmd_catalog_scan(org: String, out: Option, dry_run: bool) -> Result<()> { + let root = cache::models_dir()?; + let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; + + let known = base_hub::catalog::Catalog::bundled() + .context("reading the bundled catalog to reuse unchanged rows")?; + eprintln!("scanning {org} …"); + let repos = base_hub::scan::list_org_repos(&org)?; + eprintln!(" {} repositories", repos.len()); + + let report = base_hub::scan::scan_org(&fetcher, &base_hub::scan::HubApi, &org, &repos, &known)?; + + // What the scan decided, before anything is written. Silence here would + // make a dropped bundle indistinguishable from one that does not exist. + eprintln!( + " {} rows ({} reused unchanged), {} repos with no bundle", + report.entries.len(), + report.reused, + report.empty_repos.len() + ); + for (what, why) in &report.skipped { + eprintln!(" SKIPPED {what}: {why}"); + } + + let before: std::collections::BTreeSet<_> = known + .models + .iter() + .map(|m| (m.hf_repo.clone(), m.file.clone())) + .collect(); + let after: std::collections::BTreeSet<_> = report + .entries + .iter() + .map(|m| (m.hf_repo.clone(), m.file.clone())) + .collect(); + for (repo, file) in after.difference(&before) { + eprintln!(" + {repo}/{file}"); + } + for (repo, file) in before.difference(&after) { + eprintln!(" - {repo}/{file} (no longer published)"); + } + + let path = out.unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .parent() + .map(|p| p.join("base-hub/catalog.json")) + .unwrap_or_else(|| PathBuf::from("catalog.json")) + }); + if dry_run { + eprintln!("dry run — {} not written", path.display()); + return Ok(()); + } + + let doc = base_hub::catalog::Catalog { + schema: known.schema, + updated: report.updated_stamp(), + models: report.entries, + }; + let json = serde_json::to_string_pretty(&doc)? + "\n"; + std::fs::write(&path, json).with_context(|| format!("writing {}", path.display()))?; + eprintln!("wrote {}", path.display()); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index d2c6bd8..03dbd30 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -65,6 +65,8 @@ enum Cmd { Pull(PullArgs), /// List models in the local hub cache (and, with `--remote`, the catalog). List(ListArgs), + /// Regenerate the model catalog by scanning a published HF organization. + CatalogScan(CatalogScanArgs), /// Runtime commands — `serve`, `chat`, `complete`, `bench`, … — handled /// by the BaseRT runtime (dispatched in `hub::dispatch_external`). #[command(external_subcommand)] @@ -266,6 +268,19 @@ struct PullArgs { dry_run: bool, } +#[derive(Parser, Debug)] +struct CatalogScanArgs { + /// Organization to scan. + #[arg(long, default_value = "basecompute")] + org: String, + /// Where to write the catalog. Defaults to the bundled one, in place. + #[arg(long)] + out: Option, + /// Report what would change without writing anything. + #[arg(long)] + dry_run: bool, +} + #[derive(Parser, Debug)] struct ListArgs { /// Also list catalog models that aren't installed yet. @@ -339,6 +354,7 @@ fn main() -> Result<()> { Cmd::Keygen(a) => cmd_keygen(a), Cmd::Pull(a) => hub::cmd_pull(a), Cmd::List(a) => hub::cmd_list(a), + Cmd::CatalogScan(a) => hub::cmd_catalog_scan(a.org, a.out, a.dry_run), Cmd::External(argv) => hub::dispatch_external(argv), } } diff --git a/base-convert/crates/base-hub/catalog.json b/base-convert/crates/base-hub/catalog.json index 837cce3..274274d 100644 --- a/base-convert/crates/base-hub/catalog.json +++ b/base-convert/crates/base-hub/catalog.json @@ -1,205 +1,216 @@ { "schema": 1, - "updated": "2026-07-20", + "updated": "2026-08-19", "models": [ { - "id": "basecompute/Qwen3-0.6B", - "hf_repo": "basecompute/Qwen3-0.6B", - "file": "Qwen3-0.6B-Q4.base", - "source_repo": "Qwen/Qwen3-0.6B", - "arch": "qwen", - "quant": "default-q4", - "size": 430114816, - "sha256": "aeb9a2b16c1a5519895f99eec67e3d7c3347851d778a3663cfe80242ae87817e" - }, - { - "id": "basecompute/Qwen3-0.6B", - "hf_repo": "basecompute/Qwen3-0.6B", - "file": "Qwen3-0.6B-Q8.base", - "source_repo": "Qwen/Qwen3-0.6B", - "arch": "qwen", - "quant": "default-q8", - "size": 782403584, - "sha256": "a562fd2b682206a208fb938df6951849a70829b4228b604ccdf381327a716370" - }, - { - "id": "basecompute/Qwen3-4B", - "hf_repo": "basecompute/Qwen3-4B", - "file": "Qwen3-4B-Q4.base", - "source_repo": "Qwen/Qwen3-4B", - "arch": "qwen", - "quant": "default-q4", - "size": 2270401536, - "sha256": "df660a56b61bb68b940da100f9ad345a93d396df1955387e2f41f317e130e696" - }, - { - "id": "basecompute/Qwen3-4B", - "hf_repo": "basecompute/Qwen3-4B", - "file": "Qwen3-4B-Q8.base", - "source_repo": "Qwen/Qwen3-4B", - "arch": "qwen", - "quant": "default-q8", - "size": 4155544576, - "sha256": "7898a470ac96c67b97e6330eb0f2b5fc00475247880a66e12b7640d9cc169b9a" - }, - { - "id": "basecompute/Qwen3-30B-A3B-Thinking-2507", - "hf_repo": "basecompute/Qwen3-30B-A3B-Thinking-2507", - "file": "Qwen3-30B-A3B-Thinking-2507-Q4.base", - "source_repo": "Qwen/Qwen3-30B-A3B-Thinking-2507", - "arch": "qwen3_moe", + "id": "basecompute/Llama-3.1-8B-Instruct", + "hf_repo": "basecompute/Llama-3.1-8B-Instruct", + "file": "Llama-3.1-8B-Instruct-Q4.base", + "revision": "main", + "source_repo": "meta-llama/Llama-3.1-8B-Instruct", + "arch": "llama", "quant": "default-q4", - "size": 17182330880, - "sha256": "52b5666ee1026b12d1d7c234ec5eedeb4deed1949caeb0c0f1e53e6d0259042e" + "size": 5279326208, + "sha256": "9b397e21e9665f68552b5edb7709a90073369d1fc205d49e603e6cbf821d18da" }, { - "id": "basecompute/Qwen3-30B-A3B-Thinking-2507", - "hf_repo": "basecompute/Qwen3-30B-A3B-Thinking-2507", - "file": "Qwen3-30B-A3B-Thinking-2507-Q8.base", - "source_repo": "Qwen/Qwen3-30B-A3B-Thinking-2507", - "arch": "qwen3_moe", + "id": "basecompute/Llama-3.1-8B-Instruct", + "hf_repo": "basecompute/Llama-3.1-8B-Instruct", + "file": "Llama-3.1-8B-Instruct-Q8.base", + "revision": "main", + "source_repo": "meta-llama/Llama-3.1-8B-Instruct", + "arch": "llama", "quant": "default-q8", - "size": 31493787648, - "sha256": "5df2136010dcfedc36be6229770ad066b60fe5ec58825795bac9a6738b5567e8" + "size": 8288215040, + "sha256": "833e8c57398d0878c7e2684bcb570dc9c3d0b5a9eb6107fa337e631b50af6f87" }, { - "id": "basecompute/gemma-3-1b-it", - "hf_repo": "basecompute/gemma-3-1b-it", - "file": "gemma-3-1b-it-Q4.base", - "source_repo": "google/gemma-3-1b-it", - "arch": "gemma3", + "id": "basecompute/Llama-3.2-1B-Instruct", + "hf_repo": "basecompute/Llama-3.2-1B-Instruct", + "file": "Llama-3.2-1B-Instruct-Q4.base", + "revision": "main", + "source_repo": "meta-llama/Llama-3.2-1B-Instruct", + "arch": "llama", "quant": "default-q4", - "size": 579602688, - "sha256": "149ec28b4df637adda54064af2d7df753a3d55e500fcb710aef4c169a12ef387" + "size": 1079447552, + "sha256": "9558edb638113e8c702d530d29e677177b365859a8977df1a487b78ba5c9ba8f" }, { - "id": "basecompute/gemma-3-1b-it", - "hf_repo": "basecompute/gemma-3-1b-it", - "file": "gemma-3-1b-it-Q8.base", - "source_repo": "google/gemma-3-1b-it", - "arch": "gemma3", + "id": "basecompute/Llama-3.2-1B-Instruct", + "hf_repo": "basecompute/Llama-3.2-1B-Instruct", + "file": "Llama-3.2-1B-Instruct-Q8.base", + "revision": "main", + "source_repo": "meta-llama/Llama-3.2-1B-Instruct", + "arch": "llama", "quant": "default-q8", - "size": 1047890176, - "sha256": "d8e31808182c9e232660484dd5baa47d9fb711bc6e3ff48484dbaecf62a4238c" + "size": 1281118208, + "sha256": "0c85be62e9a04de84a5f3b8dc51966c6d669007915413fb86bed10ed60ef3924" }, { "id": "basecompute/Llama-3.2-1B-Instruct", "hf_repo": "basecompute/Llama-3.2-1B-Instruct", - "file": "Llama-3.2-1B-Instruct-Q4.base", + "file": "Llama-3.2-1B-Instruct-cuda-q4mix.base", + "revision": "main", "source_repo": "meta-llama/Llama-3.2-1B-Instruct", "arch": "llama", - "quant": "default-q4", - "size": 701861888, - "sha256": "292fe9db91c3a0da29e4d7e049eecc6fc30b0b9e009df3775b0f4894bac1a65e" + "quant": "cuda-q4mix", + "size": 767528960, + "sha256": "fbb077c79d7a1eb3594f8b4c8aa8ef636a54284eb90975d09a73010f0e046f6c", + "backend": "cuda" }, { "id": "basecompute/Llama-3.2-1B-Instruct", "hf_repo": "basecompute/Llama-3.2-1B-Instruct", - "file": "Llama-3.2-1B-Instruct-Q8.base", + "file": "Llama-3.2-1B-Instruct-cuda-q8.base", + "revision": "main", "source_repo": "meta-llama/Llama-3.2-1B-Instruct", "arch": "llama", - "quant": "default-q8", + "quant": "cuda-q8", "size": 1281118208, - "sha256": "67765adbe195378cda62ecea46c3cb2a6ab82d13709f4643f1abb0a70561c799" + "sha256": "cf3be554b160c49b1a4d896950ba1d07f6fbce2f5392071dd8a29b10884f2c24", + "backend": "cuda" }, { "id": "basecompute/Llama-3.2-3B-Instruct", "hf_repo": "basecompute/Llama-3.2-3B-Instruct", "file": "Llama-3.2-3B-Instruct-Q4.base", + "revision": "main", "source_repo": "meta-llama/Llama-3.2-3B-Instruct", "arch": "llama", "quant": "default-q4", - "size": 1814222848, - "sha256": "a0d5be46b5e13ce488e351cb8d769d3402250ee8adf3ff1fa41e70328f1fac8a" + "size": 2380601344, + "sha256": "20c803b2eeee70a7dada67ca0ee2d75081d95d0e35bac27d1857f9e3fdd84615" }, { "id": "basecompute/Llama-3.2-3B-Instruct", "hf_repo": "basecompute/Llama-3.2-3B-Instruct", "file": "Llama-3.2-3B-Instruct-Q8.base", + "revision": "main", "source_repo": "meta-llama/Llama-3.2-3B-Instruct", "arch": "llama", "quant": "default-q8", "size": 3320125440, - "sha256": "fe94e68856346a8016f83d142b5fac16555e180a6409f357b749e7f708ec7dea" + "sha256": "b26f0dd86648cec9d1f7f30dbf986c59077395dffe8607a28640a2e8d251177a" }, { - "id": "basecompute/gemma-4-E2B-it", - "hf_repo": "basecompute/gemma-4-E2B-it", - "file": "gemma-4-E2B-it-Q4.base", - "arch": "gemma4", - "quant": "default-q4", - "size": 2941140992, - "sha256": "785c3527ab6d33e4d2a1976e63ecc2bf13ceb0cab5b00ce375fd9cec2a971bc0" + "id": "basecompute/Llama-3.2-3B-Instruct", + "hf_repo": "basecompute/Llama-3.2-3B-Instruct", + "file": "Llama-3.2-3B-Instruct-cuda-q4mix.base", + "revision": "main", + "source_repo": "meta-llama/Llama-3.2-3B-Instruct", + "arch": "llama", + "quant": "cuda-q4mix", + "size": 1912723456, + "sha256": "356df54aa2a2731cc048312e5082ed89549f3a106a9a7583bcbac2d455e2c933", + "backend": "cuda" }, { - "id": "basecompute/gemma-4-E2B-it", - "hf_repo": "basecompute/gemma-4-E2B-it", - "file": "gemma-4-E2B-it-Q8.base", - "arch": "gemma4", - "quant": "default-q8", - "size": 5334106112, - "sha256": "3e9a9c8629756b2513cf2de35e6a3c18604374085b999b7841c128d542af52d9" + "id": "basecompute/Llama-3.2-3B-Instruct", + "hf_repo": "basecompute/Llama-3.2-3B-Instruct", + "file": "Llama-3.2-3B-Instruct-cuda-q8.base", + "revision": "main", + "source_repo": "meta-llama/Llama-3.2-3B-Instruct", + "arch": "llama", + "quant": "cuda-q8", + "size": 3320125440, + "sha256": "69c8eb644cb9dd022af2d574d7c1ecd62abbc51d1abb36db3a3e642b86ae4738", + "backend": "cuda" }, { - "id": "basecompute/gemma-4-E4B-it", - "hf_repo": "basecompute/gemma-4-E4B-it", - "file": "gemma-4-E4B-it-Q4.base", - "arch": "gemma4", + "id": "basecompute/Mistral-7B-Instruct-v0.3", + "hf_repo": "basecompute/Mistral-7B-Instruct-v0.3", + "file": "Mistral-7B-Instruct-v0.3-Q4.base", + "revision": "main", + "source_repo": "mistralai/Mistral-7B-Instruct-v0.3", + "arch": "llama", "quant": "default-q4", - "size": 4558422016, - "sha256": "68348e8a9d4d0a00f9fe9e3ea80835f9d45d2165e1fc407f68366bf72b64bb8f" + "size": 4272300032, + "sha256": "8fd0020cc9de60a0074522956475a6d2564babf47b08027a99661b7f889aff8d" }, { - "id": "basecompute/gemma-4-E4B-it", - "hf_repo": "basecompute/gemma-4-E4B-it", - "file": "gemma-4-E4B-it-Q8.base", - "arch": "gemma4", + "id": "basecompute/Mistral-7B-Instruct-v0.3", + "hf_repo": "basecompute/Mistral-7B-Instruct-v0.3", + "file": "Mistral-7B-Instruct-v0.3-Q8.base", + "revision": "main", + "source_repo": "mistralai/Mistral-7B-Instruct-v0.3", + "arch": "llama", "quant": "default-q8", - "size": 8297676800, - "sha256": "c293bd1e3479b293fab353783af6d96dc285d3b07b688034eb544e36c99c9ef7" + "size": 7476748288, + "sha256": "34a0f7e8b6fda223a39b6b632009ec19503db794bfa48ab7ec531dad9a37ef2f" }, { - "id": "basecompute/gemma-4-26B-A4B-it", - "hf_repo": "basecompute/gemma-4-26B-A4B-it", - "file": "gemma-4-26B-A4B-it-Q4.base", - "arch": "gemma4", - "quant": "default-q4", - "size": 16108079360, - "sha256": "efabbac27832f43d17eaac05c9a5c32545c9a02bea975836f038d32de8368782" + "id": "basecompute/Muse-Glimmer-30B", + "hf_repo": "basecompute/Muse-Glimmer-30B", + "file": "muse-glimmer-30B-kquant-dynamic.base", + "revision": "main", + "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", + "arch": "muse_glimmer", + "quant": "default-q4k-dynamic", + "size": 20952383488, + "sha256": "1eb639041f638099f6b1f7415c808cd6762b52be672d4ff5ef97829cc7669876" }, { - "id": "basecompute/gemma-4-26B-A4B-it", - "hf_repo": "basecompute/gemma-4-26B-A4B-it", - "file": "gemma-4-26B-A4B-it-Q8.base", - "arch": "gemma4", - "quant": "default-q8", - "size": 26656278784, - "sha256": "c2f13d6d299cb8a18e5c41929d974c348846214d37e2acbdf4865e8a71a4e411" + "id": "basecompute/Muse-Glimmer-30B", + "hf_repo": "basecompute/Muse-Glimmer-30B", + "file": "muse-glimmer-30B-kquant-17gb.base", + "revision": "main", + "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", + "arch": "muse_glimmer", + "quant": "q4k-17gb", + "size": 18054873088, + "sha256": "9c323d37c192e45375a936828427f50c7a303d5772465f672df478b1f8cfc1d2" }, { - "id": "basecompute/nomic-embed-text-v1.5", - "hf_repo": "basecompute/nomic-embed-text-v1.5", - "file": "nomic-embed-text-v1.5-Q4.base", - "source_repo": "nomic-ai/nomic-embed-text-v1.5", - "arch": "nomic-bert", + "id": "basecompute/Qwen3-0.6B", + "hf_repo": "basecompute/Qwen3-0.6B", + "file": "Qwen3-0.6B-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-0.6B", + "arch": "qwen", "quant": "default-q4", - "size": 78218752, - "sha256": "23b7ca9d4026bdf5c8b6d53faa761be66e6c475acdd7c59ebdc0447f2b231175" + "size": 430114816, + "sha256": "aeb9a2b16c1a5519895f99eec67e3d7c3347851d778a3663cfe80242ae87817e" }, { - "id": "basecompute/nomic-embed-text-v1.5", - "hf_repo": "basecompute/nomic-embed-text-v1.5", - "file": "nomic-embed-text-v1.5-Q8.base", - "source_repo": "nomic-ai/nomic-embed-text-v1.5", - "arch": "nomic-bert", + "id": "basecompute/Qwen3-0.6B", + "hf_repo": "basecompute/Qwen3-0.6B", + "file": "Qwen3-0.6B-Q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-0.6B", + "arch": "qwen", "quant": "default-q8", - "size": 142296576, - "sha256": "6239447345780bcf8af6bd410eaa4468bec55059899ccde63f86c060425fe422" + "size": 782403584, + "sha256": "a562fd2b682206a208fb938df6951849a70829b4228b604ccdf381327a716370" + }, + { + "id": "basecompute/Qwen3-0.6B", + "hf_repo": "basecompute/Qwen3-0.6B", + "file": "Qwen3-0.6B-cuda-q4mix.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-0.6B", + "arch": "qwen", + "quant": "cuda-q4mix", + "size": 541935616, + "sha256": "e63b22721c17a352b605b7e58a2157192069c1e1576748ecefbe715c66ac0e07", + "backend": "cuda" + }, + { + "id": "basecompute/Qwen3-0.6B", + "hf_repo": "basecompute/Qwen3-0.6B", + "file": "Qwen3-0.6B-cuda-q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-0.6B", + "arch": "qwen", + "quant": "cuda-q8", + "size": 782403584, + "sha256": "6a99cc4edaf52c60da74edad11d45aea50fa105da05aed3215dc3544cab77161", + "backend": "cuda" }, { "id": "basecompute/Qwen3-1.7B", "hf_repo": "basecompute/Qwen3-1.7B", "file": "Qwen3-1.7B-Q4.base", + "revision": "main", "source_repo": "Qwen/Qwen3-1.7B", "arch": "qwen", "quant": "default-q4", @@ -210,6 +221,7 @@ "id": "basecompute/Qwen3-1.7B", "hf_repo": "basecompute/Qwen3-1.7B", "file": "Qwen3-1.7B-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3-1.7B", "arch": "qwen", "quant": "default-q8", @@ -217,69 +229,89 @@ "sha256": "f54c1ed6a8cf757a70d9006df3a95c05f7b68de11157bf8e5a7dd671e0fb5bcc" }, { - "id": "basecompute/Qwen3-8B", - "hf_repo": "basecompute/Qwen3-8B", - "file": "Qwen3-8B-Q4.base", - "source_repo": "Qwen/Qwen3-8B", - "arch": "qwen", + "id": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "hf_repo": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "file": "Qwen3-30B-A3B-Instruct-2507-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "arch": "qwen3_moe", "quant": "default-q4", - "size": 4614987776, - "sha256": "da84d54a5c55d3c1c4c52064cac1d0fd2104584bf1a12cdb8e4179d91a797144" + "size": 17182920704, + "sha256": "7b294daeff8ba6e097f6cebe9f56c7b71f32476765a3f614eb256ef9d8c6ffce" }, { - "id": "basecompute/Qwen3-8B", - "hf_repo": "basecompute/Qwen3-8B", - "file": "Qwen3-8B-Q8.base", - "source_repo": "Qwen/Qwen3-8B", - "arch": "qwen", - "quant": "default-q8", - "size": 8454250496, - "sha256": "37785e5d3bcaacb4a3c8caf298d972d5812cfdbf4c6b800db28cfa5db5477ad6" + "id": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "hf_repo": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "file": "Qwen3-30B-A3B-Instruct-2507-cuda-q4mix.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "arch": "qwen3_moe", + "quant": "cuda-q4mix", + "size": 17794174976, + "sha256": "97bd85655b771a3f36e72784616ed1a31e91439e82e388d74668ff3fff551aea", + "backend": "cuda" }, { - "id": "basecompute/Llama-3.1-8B-Instruct", - "hf_repo": "basecompute/Llama-3.1-8B-Instruct", - "file": "Llama-3.1-8B-Instruct-Q4.base", - "source_repo": "meta-llama/Llama-3.1-8B-Instruct", - "arch": "llama", + "id": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "hf_repo": "basecompute/Qwen3-30B-A3B-Instruct-2507", + "file": "Qwen3-30B-A3B-Instruct-2507-cuda-q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-30B-A3B-Instruct-2507", + "arch": "qwen3_moe", + "quant": "cuda-q8", + "size": 31494377472, + "sha256": "36c5ed1a61e59449e23b5a4ec582dac844ec011efe91e6fb10e55a00380c9f53", + "backend": "cuda" + }, + { + "id": "basecompute/Qwen3-30B-A3B-Thinking-2507", + "hf_repo": "basecompute/Qwen3-30B-A3B-Thinking-2507", + "file": "Qwen3-30B-A3B-Thinking-2507-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-30B-A3B-Thinking-2507", + "arch": "qwen3_moe", "quant": "default-q4", - "size": 4524154880, - "sha256": "5d46c1a7e9247f095ee2e9955b1cae8bcea383d4417ecc3b26c5ce6fdd3babdc" + "size": 17182330880, + "sha256": "52b5666ee1026b12d1d7c234ec5eedeb4deed1949caeb0c0f1e53e6d0259042e" }, { - "id": "basecompute/Llama-3.1-8B-Instruct", - "hf_repo": "basecompute/Llama-3.1-8B-Instruct", - "file": "Llama-3.1-8B-Instruct-Q8.base", - "source_repo": "meta-llama/Llama-3.1-8B-Instruct", - "arch": "llama", + "id": "basecompute/Qwen3-30B-A3B-Thinking-2507", + "hf_repo": "basecompute/Qwen3-30B-A3B-Thinking-2507", + "file": "Qwen3-30B-A3B-Thinking-2507-Q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-30B-A3B-Thinking-2507", + "arch": "qwen3_moe", "quant": "default-q8", - "size": 8288215040, - "sha256": "682586936a9f29bb37055e2ff2ec724d1328435dec4b94a566793eb20bbd3d1c" + "size": 31493787648, + "sha256": "5df2136010dcfedc36be6229770ad066b60fe5ec58825795bac9a6738b5567e8" }, { - "id": "basecompute/Mistral-7B-Instruct-v0.3", - "hf_repo": "basecompute/Mistral-7B-Instruct-v0.3", - "file": "Mistral-7B-Instruct-v0.3-Q4.base", - "source_repo": "mistralai/Mistral-7B-Instruct-v0.3", - "arch": "llama", + "id": "basecompute/Qwen3-4B", + "hf_repo": "basecompute/Qwen3-4B", + "file": "Qwen3-4B-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-4B", + "arch": "qwen", "quant": "default-q4", - "size": 4079362048, - "sha256": "dbece3c8846afc89e282a98796b7651227557e7eb1645de139e12917b04d175d" + "size": 2270401536, + "sha256": "df660a56b61bb68b940da100f9ad345a93d396df1955387e2f41f317e130e696" }, { - "id": "basecompute/Mistral-7B-Instruct-v0.3", - "hf_repo": "basecompute/Mistral-7B-Instruct-v0.3", - "file": "Mistral-7B-Instruct-v0.3-Q8.base", - "source_repo": "mistralai/Mistral-7B-Instruct-v0.3", - "arch": "llama", + "id": "basecompute/Qwen3-4B", + "hf_repo": "basecompute/Qwen3-4B", + "file": "Qwen3-4B-Q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-4B", + "arch": "qwen", "quant": "default-q8", - "size": 7476748288, - "sha256": "3f2b5589475dd5da34b3433abec085b16d56818dba22748b08d39bd169f5243b" + "size": 4155544576, + "sha256": "7898a470ac96c67b97e6330eb0f2b5fc00475247880a66e12b7640d9cc169b9a" }, { "id": "basecompute/Qwen3-4B-Instruct-2507", "hf_repo": "basecompute/Qwen3-4B-Instruct-2507", "file": "Qwen3-4B-Instruct-2507-Q4.base", + "revision": "main", "source_repo": "Qwen/Qwen3-4B-Instruct-2507", "arch": "qwen", "quant": "default-q4", @@ -290,6 +322,7 @@ "id": "basecompute/Qwen3-4B-Instruct-2507", "hf_repo": "basecompute/Qwen3-4B-Instruct-2507", "file": "Qwen3-4B-Instruct-2507-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3-4B-Instruct-2507", "arch": "qwen", "quant": "default-q8", @@ -300,6 +333,7 @@ "id": "basecompute/Qwen3-4B-Thinking-2507", "hf_repo": "basecompute/Qwen3-4B-Thinking-2507", "file": "Qwen3-4B-Thinking-2507-Q4.base", + "revision": "main", "source_repo": "Qwen/Qwen3-4B-Thinking-2507", "arch": "qwen", "quant": "default-q4", @@ -310,6 +344,7 @@ "id": "basecompute/Qwen3-4B-Thinking-2507", "hf_repo": "basecompute/Qwen3-4B-Thinking-2507", "file": "Qwen3-4B-Thinking-2507-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3-4B-Thinking-2507", "arch": "qwen", "quant": "default-q8", @@ -317,21 +352,34 @@ "sha256": "930d78ec853410b68a433bfb8f608fe389566936899bf0a9dbc867618d80bce4" }, { - "id": "basecompute/Qwen3-30B-A3B-Instruct-2507", - "hf_repo": "basecompute/Qwen3-30B-A3B-Instruct-2507", - "file": "Qwen3-30B-A3B-Instruct-2507-Q4.base", - "source_repo": "Qwen/Qwen3-30B-A3B-Instruct-2507", - "arch": "qwen3_moe", + "id": "basecompute/Qwen3-8B", + "hf_repo": "basecompute/Qwen3-8B", + "file": "Qwen3-8B-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-8B", + "arch": "qwen", "quant": "default-q4", - "size": 17182920704, - "sha256": "7b294daeff8ba6e097f6cebe9f56c7b71f32476765a3f614eb256ef9d8c6ffce" + "size": 4614987776, + "sha256": "da84d54a5c55d3c1c4c52064cac1d0fd2104584bf1a12cdb8e4179d91a797144" + }, + { + "id": "basecompute/Qwen3-8B", + "hf_repo": "basecompute/Qwen3-8B", + "file": "Qwen3-8B-Q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3-8B", + "arch": "qwen", + "quant": "default-q8", + "size": 8454250496, + "sha256": "37785e5d3bcaacb4a3c8caf298d972d5812cfdbf4c6b800db28cfa5db5477ad6" }, { "id": "basecompute/Qwen3.5-2B", "hf_repo": "basecompute/Qwen3.5-2B", + "file": "Qwen3.5-2B-Q4.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-2B", "arch": "qwen35", - "file": "Qwen3.5-2B-Q4.base", "quant": "default-q4", "size": 1266810880, "sha256": "1c95f41730d5713fcb886a71d2481a29ff1a6e2fc9cea8a4d9ddbe29082fa262" @@ -339,19 +387,45 @@ { "id": "basecompute/Qwen3.5-2B", "hf_repo": "basecompute/Qwen3.5-2B", + "file": "Qwen3.5-2B-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-2B", "arch": "qwen35", - "file": "Qwen3.5-2B-Q8.base", "quant": "default-q8", "size": 2301460480, "sha256": "d4cf325278e873b3e0aefc18c5eff0a863f12fee5a82f0ac2f2e5ebce4067f3c" }, + { + "id": "basecompute/Qwen3.5-2B", + "hf_repo": "basecompute/Qwen3.5-2B", + "file": "Qwen3.5-2B-cuda-q4mix.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.5-2B", + "arch": "qwen35", + "quant": "cuda-q4mix", + "size": 1592131584, + "sha256": "db457f5dd499be915e51e757d0e999e279def9f861f0ecc206708deaa5ec49ca", + "backend": "cuda" + }, + { + "id": "basecompute/Qwen3.5-2B", + "hf_repo": "basecompute/Qwen3.5-2B", + "file": "Qwen3.5-2B-cuda-q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.5-2B", + "arch": "qwen35", + "quant": "cuda-q8", + "size": 2301460480, + "sha256": "027907a8ce268e1a7ffc339cd2746db87c2e5f01867910284b19cbb52ec62d78", + "backend": "cuda" + }, { "id": "basecompute/Qwen3.5-2B-Base", "hf_repo": "basecompute/Qwen3.5-2B-Base", + "file": "Qwen3.5-2B-Base-Q4.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-2B-Base", "arch": "qwen35", - "file": "Qwen3.5-2B-Base-Q4.base", "quant": "default-q4", "size": 1266810880, "sha256": "ef10005ccd4c6bd3e2d9fbcac31b7f4491eff2639584bdc8006e67226f63d986" @@ -359,9 +433,10 @@ { "id": "basecompute/Qwen3.5-2B-Base", "hf_repo": "basecompute/Qwen3.5-2B-Base", + "file": "Qwen3.5-2B-Base-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-2B-Base", "arch": "qwen35", - "file": "Qwen3.5-2B-Base-Q8.base", "quant": "default-q8", "size": 2301460480, "sha256": "f546b5074d92f104453a782a44143f3a44ed538569befdc7e2ff22675c719a30" @@ -369,9 +444,10 @@ { "id": "basecompute/Qwen3.5-35B-A3B", "hf_repo": "basecompute/Qwen3.5-35B-A3B", + "file": "Qwen3.5-35B-A3B-Q4.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-35B-A3B", "arch": "qwen35moe", - "file": "Qwen3.5-35B-A3B-Q4.base", "quant": "default-q4", "size": 19776225280, "sha256": "e62635e523e26c7a31df2d0d26cec485630b3b2172a9fd710b06e90c45174ff5" @@ -379,81 +455,21 @@ { "id": "basecompute/Qwen3.5-35B-A3B", "hf_repo": "basecompute/Qwen3.5-35B-A3B", + "file": "Qwen3.5-35B-A3B-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-35B-A3B", "arch": "qwen35moe", - "file": "Qwen3.5-35B-A3B-Q8.base", "quant": "default-q8", "size": 36228218880, "sha256": "6f8878ff2332451efbf2258887bad0535a97aeb657cf0b7ce8492219881184ff" }, - { - "id": "basecompute/Qwen3.6-27B", - "hf_repo": "basecompute/Qwen3.6-27B", - "source_repo": "Qwen/Qwen3.6-27B", - "arch": "qwen35", - "file": "Qwen3.6-27B-Q4.base", - "quant": "default-q4", - "size": 15447605248, - "sha256": "f24b31c552994f82e18f2a3b7a7352ff0c41cf070351e941816365bd02bca79c" - }, - { - "id": "basecompute/Qwen3.6-27B", - "hf_repo": "basecompute/Qwen3.6-27B", - "source_repo": "Qwen/Qwen3.6-27B", - "arch": "qwen35", - "file": "Qwen3.6-27B-Q8.base", - "quant": "default-q8", - "size": 28256698368, - "sha256": "f65023ea2a5e1cc2a6091cc926a4156935e4ada58133d4bc84bc6d759f738039" - }, - { - "id": "basecompute/Qwen3.6-35B-A3B", - "hf_repo": "basecompute/Qwen3.6-35B-A3B", - "source_repo": "Qwen/Qwen3.6-35B-A3B", - "arch": "qwen35moe", - "file": "Qwen3.6-35B-A3B-Q4.base", - "quant": "default-q4", - "size": 19776225280, - "sha256": "6819d2152fe025af5b1eb12d02e862c5277258a016e6b901c6c91791ba24c48b" - }, - { - "id": "basecompute/Qwen3.6-35B-A3B", - "hf_repo": "basecompute/Qwen3.6-35B-A3B", - "source_repo": "Qwen/Qwen3.6-35B-A3B", - "arch": "qwen35moe", - "file": "Qwen3.6-35B-A3B-Q8.base", - "quant": "default-q8", - "size": 36228218880, - "sha256": "11d093a918e2a0b917430547dafcdcab2fe1c2d689ec86b5343382f72d2b2c3b" - }, - { - "id": "basecompute/Qwen3.5-2B", - "hf_repo": "basecompute/Qwen3.5-2B", - "source_repo": "Qwen/Qwen3.5-2B", - "arch": "qwen35", - "file": "Qwen3.5-2B-cuda-q4mix.base", - "quant": "cuda-q4mix", - "size": 1592131584, - "sha256": "db457f5dd499be915e51e757d0e999e279def9f861f0ecc206708deaa5ec49ca", - "backend": "cuda" - }, - { - "id": "basecompute/Qwen3.5-2B", - "hf_repo": "basecompute/Qwen3.5-2B", - "source_repo": "Qwen/Qwen3.5-2B", - "arch": "qwen35", - "file": "Qwen3.5-2B-cuda-q8.base", - "quant": "cuda-q8", - "size": 2301460480, - "sha256": "027907a8ce268e1a7ffc339cd2746db87c2e5f01867910284b19cbb52ec62d78", - "backend": "cuda" - }, { "id": "basecompute/Qwen3.5-35B-A3B", "hf_repo": "basecompute/Qwen3.5-35B-A3B", + "file": "Qwen3.5-35B-A3B-cuda-q4mix.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-35B-A3B", "arch": "qwen35moe", - "file": "Qwen3.5-35B-A3B-cuda-q4mix.base", "quant": "cuda-q4mix", "size": 21123661824, "sha256": "1f836346dfe62bddb1c9b2a4f514c7b2fcea47b63d6d5dd1d8f4cfee95b50363", @@ -462,9 +478,10 @@ { "id": "basecompute/Qwen3.5-35B-A3B", "hf_repo": "basecompute/Qwen3.5-35B-A3B", + "file": "Qwen3.5-35B-A3B-cuda-q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.5-35B-A3B", "arch": "qwen35moe", - "file": "Qwen3.5-35B-A3B-cuda-q8.base", "quant": "cuda-q8", "size": 36357767168, "sha256": "7dbbe60e58c96ab845eedf14ad9684abaa5175ba04d137a3da704b2a16afacbd", @@ -473,9 +490,32 @@ { "id": "basecompute/Qwen3.6-27B", "hf_repo": "basecompute/Qwen3.6-27B", + "file": "Qwen3.6-27B-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.6-27B", + "arch": "qwen35", + "quant": "default-q4", + "size": 15447605248, + "sha256": "f24b31c552994f82e18f2a3b7a7352ff0c41cf070351e941816365bd02bca79c" + }, + { + "id": "basecompute/Qwen3.6-27B", + "hf_repo": "basecompute/Qwen3.6-27B", + "file": "Qwen3.6-27B-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.6-27B", "arch": "qwen35", + "quant": "default-q8", + "size": 28256698368, + "sha256": "f65023ea2a5e1cc2a6091cc926a4156935e4ada58133d4bc84bc6d759f738039" + }, + { + "id": "basecompute/Qwen3.6-27B", + "hf_repo": "basecompute/Qwen3.6-27B", "file": "Qwen3.6-27B-cuda-q4mix.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.6-27B", + "arch": "qwen35", "quant": "cuda-q4mix", "size": 18159501312, "sha256": "7aebf3944f2d0ba10b5b2d9ddbb9fd0bfe3f52f2eeea011da362416209709921", @@ -484,9 +524,10 @@ { "id": "basecompute/Qwen3.6-27B", "hf_repo": "basecompute/Qwen3.6-27B", + "file": "Qwen3.6-27B-cuda-q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.6-27B", "arch": "qwen35", - "file": "Qwen3.6-27B-cuda-q8.base", "quant": "cuda-q8", "size": 28386312192, "sha256": "d0be0cad29ccafde360cb92fa2298527abc35749e8ba390b2a5b7ce596ba938f", @@ -495,9 +536,32 @@ { "id": "basecompute/Qwen3.6-35B-A3B", "hf_repo": "basecompute/Qwen3.6-35B-A3B", + "file": "Qwen3.6-35B-A3B-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.6-35B-A3B", + "arch": "qwen35moe", + "quant": "default-q4", + "size": 19776225280, + "sha256": "6819d2152fe025af5b1eb12d02e862c5277258a016e6b901c6c91791ba24c48b" + }, + { + "id": "basecompute/Qwen3.6-35B-A3B", + "hf_repo": "basecompute/Qwen3.6-35B-A3B", + "file": "Qwen3.6-35B-A3B-Q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.6-35B-A3B", "arch": "qwen35moe", + "quant": "default-q8", + "size": 36228218880, + "sha256": "11d093a918e2a0b917430547dafcdcab2fe1c2d689ec86b5343382f72d2b2c3b" + }, + { + "id": "basecompute/Qwen3.6-35B-A3B", + "hf_repo": "basecompute/Qwen3.6-35B-A3B", "file": "Qwen3.6-35B-A3B-cuda-q4mix.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.6-35B-A3B", + "arch": "qwen35moe", "quant": "cuda-q4mix", "size": 21123661824, "sha256": "75951215950ba8feb94dfb86351505b6f6fc818c6dbedd64d946c6cd9214fe4a", @@ -506,108 +570,66 @@ { "id": "basecompute/Qwen3.6-35B-A3B", "hf_repo": "basecompute/Qwen3.6-35B-A3B", + "file": "Qwen3.6-35B-A3B-cuda-q8.base", + "revision": "main", "source_repo": "Qwen/Qwen3.6-35B-A3B", "arch": "qwen35moe", - "file": "Qwen3.6-35B-A3B-cuda-q8.base", "quant": "cuda-q8", "size": 36357767168, "sha256": "350f09147443f1ed5e7e8709e41d43fe7d886138ad93129c69c8881208951a04", "backend": "cuda" }, { - "id": "basecompute/gemma-4-26B-A4B-it", - "hf_repo": "basecompute/gemma-4-26B-A4B-it", - "source_repo": null, - "arch": "gemma4", - "file": "gemma-4-26B-A4B-it-cuda-q4mix.base", - "quant": "cuda-q4mix", - "size": 16108079360, - "sha256": "42175b1a0eb73da799e8a3745ec7d29d03803d5b3b61767fd7c77825ad9e7fef", - "backend": "cuda" + "id": "basecompute/Qwen3.8-27B", + "hf_repo": "basecompute/Qwen3.8-27B", + "file": "Qwen3.8-27B-Q4.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.8-27B", + "arch": "qwen35", + "quant": "default-q4", + "size": 17467801600, + "sha256": "069f4ceca264d9e3a69ee9af725249a924024a27d46680d37da6b5851c49a294" }, { - "id": "basecompute/gemma-4-26B-A4B-it", - "hf_repo": "basecompute/gemma-4-26B-A4B-it", - "source_repo": null, - "arch": "gemma4", - "file": "gemma-4-26B-A4B-it-cuda-q8.base", - "quant": "cuda-q8", - "size": 27029195008, - "sha256": "6f8c9ebeab8106c4b4489ae19c41cec9bffcd9ed1aaf8489833bd365a86200d5", - "backend": "cuda" - }, - { - "id": "basecompute/Qwen3-30B-A3B-Instruct-2507", - "hf_repo": "basecompute/Qwen3-30B-A3B-Instruct-2507", - "source_repo": "Qwen/Qwen3-30B-A3B-Instruct-2507", - "arch": "qwen3_moe", - "file": "Qwen3-30B-A3B-Instruct-2507-cuda-q4mix.base", - "quant": "cuda-q4mix", - "size": 17794174976, - "sha256": "97bd85655b771a3f36e72784616ed1a31e91439e82e388d74668ff3fff551aea", - "backend": "cuda" - }, - { - "id": "basecompute/Qwen3-30B-A3B-Instruct-2507", - "hf_repo": "basecompute/Qwen3-30B-A3B-Instruct-2507", - "source_repo": "Qwen/Qwen3-30B-A3B-Instruct-2507", - "arch": "qwen3_moe", - "file": "Qwen3-30B-A3B-Instruct-2507-cuda-q8.base", - "quant": "cuda-q8", - "size": 31494377472, - "sha256": "36c5ed1a61e59449e23b5a4ec582dac844ec011efe91e6fb10e55a00380c9f53", - "backend": "cuda" - }, - { - "id": "basecompute/Llama-3.2-1B-Instruct", - "hf_repo": "basecompute/Llama-3.2-1B-Instruct", - "source_repo": "meta-llama/Llama-3.2-1B-Instruct", - "arch": "llama", - "file": "Llama-3.2-1B-Instruct-cuda-q4mix.base", - "quant": "cuda-q4mix", - "size": 767528960, - "sha256": "fbb077c79d7a1eb3594f8b4c8aa8ef636a54284eb90975d09a73010f0e046f6c", - "backend": "cuda" - }, - { - "id": "basecompute/Llama-3.2-1B-Instruct", - "hf_repo": "basecompute/Llama-3.2-1B-Instruct", - "source_repo": "meta-llama/Llama-3.2-1B-Instruct", - "arch": "llama", - "file": "Llama-3.2-1B-Instruct-cuda-q8.base", - "quant": "cuda-q8", - "size": 1281118208, - "sha256": "cf3be554b160c49b1a4d896950ba1d07f6fbce2f5392071dd8a29b10884f2c24", - "backend": "cuda" + "id": "basecompute/Qwen3.8-27B", + "hf_repo": "basecompute/Qwen3.8-27B", + "file": "Qwen3.8-27B-Q8.base", + "revision": "main", + "source_repo": "Qwen/Qwen3.8-27B", + "arch": "qwen35", + "quant": "default-q8", + "size": 28386443264, + "sha256": "6d4d362bf410c55507e894c132e03ecc0b5fbdb038b09f999ed99f59569a79d0" }, { - "id": "basecompute/Llama-3.2-3B-Instruct", - "hf_repo": "basecompute/Llama-3.2-3B-Instruct", - "source_repo": "meta-llama/Llama-3.2-3B-Instruct", - "arch": "llama", - "file": "Llama-3.2-3B-Instruct-cuda-q4mix.base", - "quant": "cuda-q4mix", - "size": 1912723456, - "sha256": "356df54aa2a2731cc048312e5082ed89549f3a106a9a7583bcbac2d455e2c933", - "backend": "cuda" + "id": "basecompute/gemma-3-1b-it", + "hf_repo": "basecompute/gemma-3-1b-it", + "file": "gemma-3-1b-it-Q4.base", + "revision": "main", + "source_repo": "google/gemma-3-1b-it", + "arch": "gemma3", + "quant": "default-q4", + "size": 579602688, + "sha256": "149ec28b4df637adda54064af2d7df753a3d55e500fcb710aef4c169a12ef387" }, { - "id": "basecompute/Llama-3.2-3B-Instruct", - "hf_repo": "basecompute/Llama-3.2-3B-Instruct", - "source_repo": "meta-llama/Llama-3.2-3B-Instruct", - "arch": "llama", - "file": "Llama-3.2-3B-Instruct-cuda-q8.base", - "quant": "cuda-q8", - "size": 3320125440, - "sha256": "69c8eb644cb9dd022af2d574d7c1ecd62abbc51d1abb36db3a3e642b86ae4738", - "backend": "cuda" + "id": "basecompute/gemma-3-1b-it", + "hf_repo": "basecompute/gemma-3-1b-it", + "file": "gemma-3-1b-it-Q8.base", + "revision": "main", + "source_repo": "google/gemma-3-1b-it", + "arch": "gemma3", + "quant": "default-q8", + "size": 1047890176, + "sha256": "d8e31808182c9e232660484dd5baa47d9fb711bc6e3ff48484dbaecf62a4238c" }, { "id": "basecompute/gemma-3-1b-it", "hf_repo": "basecompute/gemma-3-1b-it", + "file": "gemma-3-1b-it-cuda-q4mix.base", + "revision": "main", "source_repo": "google/gemma-3-1b-it", "arch": "gemma3", - "file": "gemma-3-1b-it-cuda-q4mix.base", "quant": "cuda-q4mix", "size": 655100160, "sha256": "bf12127dd688be09db683c050cbe14f637382aab41d30e52752cdfd6910b2f34", @@ -616,20 +638,83 @@ { "id": "basecompute/gemma-3-1b-it", "hf_repo": "basecompute/gemma-3-1b-it", + "file": "gemma-3-1b-it-cuda-q8.base", + "revision": "main", "source_repo": "google/gemma-3-1b-it", "arch": "gemma3", - "file": "gemma-3-1b-it-cuda-q8.base", "quant": "cuda-q8", "size": 1047890176, "sha256": "bd6dd1f20e55bef8364f0c1ed76b953f406c1500ea768532a78c6f000ad82ecc", "backend": "cuda" }, + { + "id": "basecompute/gemma-4-26B-A4B-it", + "hf_repo": "basecompute/gemma-4-26B-A4B-it", + "file": "gemma-4-26B-A4B-it-Q4.base", + "revision": "main", + "arch": "gemma4", + "quant": "default-q4", + "size": 16108079360, + "sha256": "efabbac27832f43d17eaac05c9a5c32545c9a02bea975836f038d32de8368782" + }, + { + "id": "basecompute/gemma-4-26B-A4B-it", + "hf_repo": "basecompute/gemma-4-26B-A4B-it", + "file": "gemma-4-26B-A4B-it-Q8.base", + "revision": "main", + "arch": "gemma4", + "quant": "default-q8", + "size": 26656278784, + "sha256": "c2f13d6d299cb8a18e5c41929d974c348846214d37e2acbdf4865e8a71a4e411" + }, + { + "id": "basecompute/gemma-4-26B-A4B-it", + "hf_repo": "basecompute/gemma-4-26B-A4B-it", + "file": "gemma-4-26B-A4B-it-cuda-q4mix.base", + "revision": "main", + "arch": "gemma4", + "quant": "cuda-q4mix", + "size": 16108079360, + "sha256": "42175b1a0eb73da799e8a3745ec7d29d03803d5b3b61767fd7c77825ad9e7fef", + "backend": "cuda" + }, + { + "id": "basecompute/gemma-4-26B-A4B-it", + "hf_repo": "basecompute/gemma-4-26B-A4B-it", + "file": "gemma-4-26B-A4B-it-cuda-q8.base", + "revision": "main", + "arch": "gemma4", + "quant": "cuda-q8", + "size": 27029195008, + "sha256": "6f8c9ebeab8106c4b4489ae19c41cec9bffcd9ed1aaf8489833bd365a86200d5", + "backend": "cuda" + }, + { + "id": "basecompute/gemma-4-E2B-it", + "hf_repo": "basecompute/gemma-4-E2B-it", + "file": "gemma-4-E2B-it-Q4.base", + "revision": "main", + "arch": "gemma4", + "quant": "default-q4", + "size": 2941140992, + "sha256": "785c3527ab6d33e4d2a1976e63ecc2bf13ceb0cab5b00ce375fd9cec2a971bc0" + }, { "id": "basecompute/gemma-4-E2B-it", "hf_repo": "basecompute/gemma-4-E2B-it", - "source_repo": null, + "file": "gemma-4-E2B-it-Q8.base", + "revision": "main", "arch": "gemma4", + "quant": "default-q8", + "size": 5334106112, + "sha256": "3e9a9c8629756b2513cf2de35e6a3c18604374085b999b7841c128d542af52d9" + }, + { + "id": "basecompute/gemma-4-E2B-it", + "hf_repo": "basecompute/gemma-4-E2B-it", "file": "gemma-4-E2B-it-cuda-q4mix.base", + "revision": "main", + "arch": "gemma4", "quant": "cuda-q4mix", "size": 4298522624, "sha256": "47e5da7e8d4d339bacdaf82b0550b5de53506fcbf1effd52e0fe8b680b68e214", @@ -638,55 +723,415 @@ { "id": "basecompute/gemma-4-E2B-it", "hf_repo": "basecompute/gemma-4-E2B-it", - "source_repo": null, - "arch": "gemma4", "file": "gemma-4-E2B-it-cuda-q8.base", + "revision": "main", + "arch": "gemma4", "quant": "cuda-q8", "size": 5350326272, "sha256": "115c4ae330c2b4acd9dfc586a3ebe6a2080efb9de10f893f8d42eaf960a8e5d4", "backend": "cuda" }, { - "id": "basecompute/Qwen3-0.6B", - "hf_repo": "basecompute/Qwen3-0.6B", - "source_repo": "Qwen/Qwen3-0.6B", - "arch": "qwen", - "file": "Qwen3-0.6B-cuda-q4mix.base", - "quant": "cuda-q4mix", - "size": 541935616, - "sha256": "e63b22721c17a352b605b7e58a2157192069c1e1576748ecefbe715c66ac0e07", - "backend": "cuda" + "id": "basecompute/gemma-4-E4B-it", + "hf_repo": "basecompute/gemma-4-E4B-it", + "file": "gemma-4-E4B-it-Q4.base", + "revision": "main", + "arch": "gemma4", + "quant": "default-q4", + "size": 4558422016, + "sha256": "68348e8a9d4d0a00f9fe9e3ea80835f9d45d2165e1fc407f68366bf72b64bb8f" }, { - "id": "basecompute/Qwen3-0.6B", - "hf_repo": "basecompute/Qwen3-0.6B", - "source_repo": "Qwen/Qwen3-0.6B", - "arch": "qwen", - "file": "Qwen3-0.6B-cuda-q8.base", - "quant": "cuda-q8", - "size": 782403584, - "sha256": "6a99cc4edaf52c60da74edad11d45aea50fa105da05aed3215dc3544cab77161", - "backend": "cuda" + "id": "basecompute/gemma-4-E4B-it", + "hf_repo": "basecompute/gemma-4-E4B-it", + "file": "gemma-4-E4B-it-Q8.base", + "revision": "main", + "arch": "gemma4", + "quant": "default-q8", + "size": 8297676800, + "sha256": "c293bd1e3479b293fab353783af6d96dc285d3b07b688034eb544e36c99c9ef7" }, { - "id": "basecompute/Muse-Glimmer-30B", - "hf_repo": "basecompute/Muse-Glimmer-30B", - "file": "muse-glimmer-30B-kquant-dynamic.base", - "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", - "arch": "muse_glimmer", - "quant": "default-q4k-dynamic", - "size": 20952383488, - "sha256": "1eb639041f638099f6b1f7415c808cd6762b52be672d4ff5ef97829cc7669876" + "id": "basecompute/nomic-embed-text-v1.5", + "hf_repo": "basecompute/nomic-embed-text-v1.5", + "file": "nomic-embed-text-v1.5-Q4.base", + "revision": "main", + "source_repo": "nomic-ai/nomic-embed-text-v1.5", + "arch": "nomic-bert", + "quant": "default-q4", + "size": 78218752, + "sha256": "23b7ca9d4026bdf5c8b6d53faa761be66e6c475acdd7c59ebdc0447f2b231175" }, { - "id": "basecompute/Muse-Glimmer-30B", - "hf_repo": "basecompute/Muse-Glimmer-30B", - "file": "muse-glimmer-30B-kquant-17gb.base", - "source_repo": "meta-models/Muse-Glimmer-30B-GGUF", - "arch": "muse_glimmer", - "quant": "q4k-17gb", - "size": 18054873088, - "sha256": "9c323d37c192e45375a936828427f50c7a303d5772465f672df478b1f8cfc1d2" + "id": "basecompute/nomic-embed-text-v1.5", + "hf_repo": "basecompute/nomic-embed-text-v1.5", + "file": "nomic-embed-text-v1.5-Q8.base", + "revision": "main", + "source_repo": "nomic-ai/nomic-embed-text-v1.5", + "arch": "nomic-bert", + "quant": "default-q8", + "size": 142296576, + "sha256": "6239447345780bcf8af6bd410eaa4468bec55059899ccde63f86c060425fe422" + }, + { + "id": "basecompute/whisper-base", + "hf_repo": "basecompute/whisper-base", + "file": "whisper-base-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 149259264, + "sha256": "c34258af20c4f2416103bfc3386e024557180aad8685365c5f20ea8d44ffaf21" + }, + { + "id": "basecompute/whisper-base", + "hf_repo": "basecompute/whisper-base", + "file": "whisper-base-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 85951488, + "sha256": "aef13d7e068d5b71ad045ad0883a7bcfcb77a11c7eadbda09afdf085a703eaf1" + }, + { + "id": "basecompute/whisper-base", + "hf_repo": "basecompute/whisper-base", + "file": "whisper-base-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 106005504, + "sha256": "cf8a7d4673c908a408bcd18f2509fdcd28941b04001841a79a8836609ab18a8c" + }, + { + "id": "basecompute/whisper-base.en", + "hf_repo": "basecompute/whisper-base.en", + "file": "whisper-base.en-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 148997120, + "sha256": "dd98a93fa3ffe494c7ad40a0c62f612e84daa4a71abd0300f38d6f1b302079da" + }, + { + "id": "basecompute/whisper-base.en", + "hf_repo": "basecompute/whisper-base.en", + "file": "whisper-base.en-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 85689344, + "sha256": "8b3e419aa847e9ca1fa8bf496217458ed7271e4845a98242abcc269a9027be8a" + }, + { + "id": "basecompute/whisper-base.en", + "hf_repo": "basecompute/whisper-base.en", + "file": "whisper-base.en-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 105743360, + "sha256": "ee0dc4a69fe1eb867dbeed2fbb1f92345511d9c1cfa8b9d7dd1d2a5143f27480" + }, + { + "id": "basecompute/whisper-large", + "hf_repo": "basecompute/whisper-large", + "file": "whisper-large-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 3098462720, + "sha256": "4ae79eb52338320a0ac6275d88d4b86892990b6f16e3d30cf0c118001dfcf5b9" + }, + { + "id": "basecompute/whisper-large", + "hf_repo": "basecompute/whisper-large", + "file": "whisper-large-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 986696192, + "sha256": "c0c096b8f0f012d346a0aca3e6150c8f78ba215bf1a4b066d2a9df760fa3689c" + }, + { + "id": "basecompute/whisper-large", + "hf_repo": "basecompute/whisper-large", + "file": "whisper-large-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 1675610624, + "sha256": "5f15a8ec00a0e62bd86ac318bc262a0f188120f2d4294ef7baf91d27085f4303" + }, + { + "id": "basecompute/whisper-large-v2", + "hf_repo": "basecompute/whisper-large-v2", + "file": "whisper-large-v2-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 3098462720, + "sha256": "d4e16ca86425ff8531639be8c8ee80adf9cad8c142aa2d5aada751dd2483c745" + }, + { + "id": "basecompute/whisper-large-v2", + "hf_repo": "basecompute/whisper-large-v2", + "file": "whisper-large-v2-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 986696192, + "sha256": "7b43091ec053eda37a44789acba67174ceb3af3e27f2f92da73cbf81358d2747" + }, + { + "id": "basecompute/whisper-large-v2", + "hf_repo": "basecompute/whisper-large-v2", + "file": "whisper-large-v2-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 1675610624, + "sha256": "5de4b5dbe3127ba4a4758af9d89034feb8edaecc1f0b96d6723ae1558ea6f3a2" + }, + { + "id": "basecompute/whisper-large-v3", + "hf_repo": "basecompute/whisper-large-v3", + "file": "whisper-large-v3-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 3098855936, + "sha256": "61cc2c85ab30559b1637b4b9b7e53d552ff36959aac3c0b5d802cb9093b36b77" + }, + { + "id": "basecompute/whisper-large-v3", + "hf_repo": "basecompute/whisper-large-v3", + "file": "whisper-large-v3-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 987089408, + "sha256": "5a2c5ffdfbecefd2458217b3f6b18e0de757cc21c12ff2e7c9e9e7ec388ca420" + }, + { + "id": "basecompute/whisper-large-v3", + "hf_repo": "basecompute/whisper-large-v3", + "file": "whisper-large-v3-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 1676003840, + "sha256": "ad8578bc069bf0f202b2c8b07c3a09ea97430645c6ea0de9d32689eab61aa86b" + }, + { + "id": "basecompute/whisper-large-v3-turbo", + "hf_repo": "basecompute/whisper-large-v3-turbo", + "file": "whisper-large-v3-turbo-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 1624295936, + "sha256": "892a39ea1fd47f9444c54f366dde5b3c1fb79b5fcb41182ff44faac96d204e5a" + }, + { + "id": "basecompute/whisper-large-v3-turbo", + "hf_repo": "basecompute/whisper-large-v3-turbo", + "file": "whisper-large-v3-turbo-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 568510976, + "sha256": "c923ad0945a79112527b0c800faee2e594bcee5d994cb27cec2cf0c52454c0e6" + }, + { + "id": "basecompute/whisper-large-v3-turbo", + "hf_repo": "basecompute/whisper-large-v3-turbo", + "file": "whisper-large-v3-turbo-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 912902656, + "sha256": "1233dc80209bd536770fb73cee8e9949e15ca5f15d2914a0d926d7fe9db51c1b" + }, + { + "id": "basecompute/whisper-medium", + "hf_repo": "basecompute/whisper-medium", + "file": "whisper-medium-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 1537460224, + "sha256": "02e37bc41675649edfcde0b1817a256bf057d3d45bfeb2dc702e4cf113969616" + }, + { + "id": "basecompute/whisper-medium", + "hf_repo": "basecompute/whisper-medium", + "file": "whisper-medium-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 524535808, + "sha256": "c15c2f8094c356ffc392eb80bdb1bb0b97ac5f88935d01702d287c3a569076c2" + }, + { + "id": "basecompute/whisper-medium", + "hf_repo": "basecompute/whisper-medium", + "file": "whisper-medium-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 854837248, + "sha256": "af4b2041bc9b10e469b16a9c35ee09fc7c354d73ade7432d1c9881de8be5e53d" + }, + { + "id": "basecompute/whisper-medium.en", + "hf_repo": "basecompute/whisper-medium.en", + "file": "whisper-medium.en-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 1537181696, + "sha256": "3ab5098efe679b79b367592f17e18f726f5d64bf1cefed70e7d87fa7115416ce" + }, + { + "id": "basecompute/whisper-medium.en", + "hf_repo": "basecompute/whisper-medium.en", + "file": "whisper-medium.en-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 524257280, + "sha256": "5307e62e8b0e67d7ccb97b38eca3798fffc2f71a12d77bbb4d99322727fa64b7" + }, + { + "id": "basecompute/whisper-medium.en", + "hf_repo": "basecompute/whisper-medium.en", + "file": "whisper-medium.en-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 854558720, + "sha256": "ff6454b3b05c3093fe60a9cd52f3be2d7e0428e5556edd530edff0ce5bca59eb" + }, + { + "id": "basecompute/whisper-small", + "hf_repo": "basecompute/whisper-small", + "file": "whisper-small-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 489440768, + "sha256": "ae5ce9ab9cbddd8bb2e1448b66df8c6ca73042db89e1db67f333484cd344b5ae" + }, + { + "id": "basecompute/whisper-small", + "hf_repo": "basecompute/whisper-small", + "file": "whisper-small-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 204031488, + "sha256": "fcca86e7b1d349b18eb452b422f581488b69df999113edaa999ef9b2006de1f3" + }, + { + "id": "basecompute/whisper-small", + "hf_repo": "basecompute/whisper-small", + "file": "whisper-small-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 296830464, + "sha256": "ef9bd77afb49f18f21a0b4174db4ab5877d955ba1e8f664d22bddf37159c5cdd" + }, + { + "id": "basecompute/whisper-small.en", + "hf_repo": "basecompute/whisper-small.en", + "file": "whisper-small.en-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 489178624, + "sha256": "ac00231e3687f6d28dfcf4af8b2b77b051079c0a3dd6fe56189b2dcdfd7ba6b9" + }, + { + "id": "basecompute/whisper-small.en", + "hf_repo": "basecompute/whisper-small.en", + "file": "whisper-small.en-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 203769344, + "sha256": "69eb8b3b08e73ec5152869d830569d02d8477880cbc5b8e510a434c9d893cb3b" + }, + { + "id": "basecompute/whisper-small.en", + "hf_repo": "basecompute/whisper-small.en", + "file": "whisper-small.en-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 296568320, + "sha256": "d0263336cc21634016febcb2e98ad5b551a869717d319947018af3f038dba4f4" + }, + { + "id": "basecompute/whisper-tiny", + "hf_repo": "basecompute/whisper-tiny", + "file": "whisper-tiny-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 78938880, + "sha256": "0cb81c8694fa19917a64624a982efcc3487c8db794419b499d358fd81235901a" + }, + { + "id": "basecompute/whisper-tiny", + "hf_repo": "basecompute/whisper-tiny", + "file": "whisper-tiny-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 55083776, + "sha256": "b9afe4be71a15061d1241effa82279c3d082586ddeab9f77bef99cc00854d466" + }, + { + "id": "basecompute/whisper-tiny", + "hf_repo": "basecompute/whisper-tiny", + "file": "whisper-tiny-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 62685952, + "sha256": "2cfc1becc04e944bd8a425c4cdc8f5bf16261becf1011855464b116e1c1c10f6" + }, + { + "id": "basecompute/whisper-tiny.en", + "hf_repo": "basecompute/whisper-tiny.en", + "file": "whisper-tiny.en-F16.base", + "revision": "main", + "arch": "whisper", + "quant": "default-f16", + "size": 78676736, + "sha256": "75728f0bd59e25ad18c0e9d10a2d6d49d7255528717326e26bb62dc99fd050cc" + }, + { + "id": "basecompute/whisper-tiny.en", + "hf_repo": "basecompute/whisper-tiny.en", + "file": "whisper-tiny.en-Q4.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q4", + "size": 54821632, + "sha256": "55859049a1132bd02695ab555e8250331b548474712c599c34a0aaa8f82f6b38" + }, + { + "id": "basecompute/whisper-tiny.en", + "hf_repo": "basecompute/whisper-tiny.en", + "file": "whisper-tiny.en-Q8.base", + "revision": "main", + "arch": "whisper", + "quant": "default-q8", + "size": 62423808, + "sha256": "3dbaaad2a23d0c68f1bac54963aadeb151c4bd38cf92d70cc2d27bf626feeb07" } ] } diff --git a/base-convert/crates/base-hub/src/download.rs b/base-convert/crates/base-hub/src/download.rs index ca9727a..8994d49 100644 --- a/base-convert/crates/base-hub/src/download.rs +++ b/base-convert/crates/base-hub/src/download.rs @@ -41,13 +41,28 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; use std::time::Duration; -/// Concurrent range requests. Eight saturates a fast link without looking like -/// an attack to the CDN; the Hub's own Python client uses the same order. -const DEFAULT_CONNECTIONS: usize = 8; +/// Concurrent range requests. +/// +/// Measured against the 19.8GB Qwen3.5-35B-A3B bundle, three rounds on the +/// same link (MiB/s, 15s warmup + 45s window, cold cache each run): +/// +/// connections x chunk r1 r2 r3 mean +/// 8 x 32MB (old) 39.8 - - +/// 16 x 16MB 49.4 53.3 - +/// 24 x 16MB 54.4 81.7 83.5 73.2 +/// 32 x 8MB 58.3 77.1 77.8 71.1 +/// +/// 24 and 32 are indistinguishable; 24 is chosen for the smaller connection +/// count at equal throughput. Eight — the previous default — is measurably +/// short of the link. +const DEFAULT_CONNECTIONS: usize = 24; /// Bytes per range request. Large enough that per-request overhead vanishes -/// against a multi-GB file, small enough that a failed chunk is cheap to redo -/// and that `connections * chunk` stays a sane peak memory footprint. -const DEFAULT_CHUNK_MB: u64 = 32; +/// against a multi-GB file, small enough that a failed chunk is cheap to redo. +/// +/// Peak memory is `connections * chunk` — 384MB at the defaults, since each +/// worker holds one chunk in flight. Lower either knob on a constrained box; +/// 16MB earned its place over 8MB in round 3 (83.5 vs 62.4 at 24 connections). +const DEFAULT_CHUNK_MB: u64 = 16; /// Seconds a read may stall before the chunk is failed and retried. const DEFAULT_READ_TIMEOUT_SECS: u64 = 60; diff --git a/base-convert/crates/base-hub/src/fetch.rs b/base-convert/crates/base-hub/src/fetch.rs index f78ead8..9d072e5 100644 --- a/base-convert/crates/base-hub/src/fetch.rs +++ b/base-convert/crates/base-hub/src/fetch.rs @@ -32,17 +32,26 @@ const DEFAULT_HF_MAX_RETRIES: usize = 5; /// better trade — this threshold separates `config.json` from a weight shard. const RANGED_MIN_BYTES: u64 = 32 * 1024 * 1024; -/// Force every large file through the range downloader, even one the Hub -/// serves over Xet (`$BASERT_HF_FORCE_RANGED`). +/// Send large files over Xet's CAS path instead of ranged HTTPS +/// (`$BASERT_HF_XET`). /// -/// Two uses. It is how the range path gets exercised against huggingface.co -/// at all — the Hub has migrated essentially every repository to Xet, so the -/// fallback would otherwise only ever run against a non-migrated endpoint or -/// an `$HF_ENDPOINT` mirror. And it is the escape hatch when hf-xet is the -/// thing going wrong on a given link: the same bytes are still reachable over -/// plain ranged HTTPS from the CDN. -fn force_ranged() -> bool { - std::env::var("BASERT_HF_FORCE_RANGED") +/// Off by default, which is the whole point of this knob's existence rather +/// than its opposite. Xet cannot resume: hf-hub 1.0 builds its `XetSession` +/// from a default `XetConfig` and exposes no way to pass one, so there is no +/// chunk cache to pick up from, and an interrupted 20GB pull starts over. +/// Measured on the 19.8GB Qwen3.5-35B-A3B bundle: killed at 2246MiB, the next +/// attempt was 12s in with 328MiB. +/// +/// It is not a speed trade either, which is what made this an easy call. Over +/// three rounds on the same link the ranged path at its defaults matched or +/// beat Xet every time (Xet 47.3 / 28.8 MiB/s; ranged 24x16MB 54.4 / 81.7). +/// +/// What Xet still has is cross-model deduplication: chunks shared with +/// something already fetched do not cross the wire at all. That does not show +/// up in a cold single-model pull like the one measured, so this stays +/// available for anyone pulling a family of related bundles. +fn prefer_xet() -> bool { + std::env::var("BASERT_HF_XET") .map(|v| !v.is_empty() && v != "0") .unwrap_or(false) } @@ -72,6 +81,23 @@ pub trait Fetcher { let _ = repo; None } + + /// Read `range` of `filename` without downloading the rest. + /// + /// A `.base` header is a 16-byte prefix plus a JSON blob, both at the front + /// of a file that may be tens of gigabytes. Two ranged reads are what let + /// [`crate::scan`] learn a published bundle's arch and backend without + /// fetching the bundle. + fn read_range( + &self, + repo: &str, + revision: &str, + filename: &str, + range: std::ops::Range, + ) -> Result> { + let _ = (repo, revision, filename, range); + anyhow::bail!("this fetcher cannot read byte ranges") + } } /// Terminal progress for one hf-hub-driven download. @@ -333,7 +359,11 @@ impl Fetcher for HfFetcher { // strictly worse. Small files take the same path: not worth a chunk // plan, and a retry costs less there than planning one. A file the // listing does not describe goes there too, rather than guessing. - let facts = facts.filter(|f| (!f.xet || force_ranged()) && f.size >= RANGED_MIN_BYTES); + // Large files take the resumable path whatever transport the Hub + // offers. Xet is opt-in (see `prefer_xet`): it cannot resume, and a + // 20GB pull that has to restart is the failure this release exists to + // fix. + let facts = facts.filter(|f| !(f.xet && prefer_xet()) && f.size >= RANGED_MIN_BYTES); let Some(facts) = facts else { return handle .download_file() @@ -384,6 +414,29 @@ impl Fetcher for HfFetcher { .join(format!("models--{}", repo.replace('/', "--"))), ) } + + fn read_range( + &self, + repo: &str, + revision: &str, + filename: &str, + range: std::ops::Range, + ) -> Result> { + let bytes = self + .repo(repo) + .download_file_to_bytes() + .filename(filename) + .revision(revision) + .range(range.clone()) + .send() + .with_context(|| { + format!( + "reading bytes {}..{} of {filename} from {repo}@{revision}", + range.start, range.end + ) + })?; + Ok(bytes.to_vec()) + } } /// Install a file returned by [`Fetcher::get_file`] at `dst`, leaving at most diff --git a/base-convert/crates/base-hub/src/gen.rs b/base-convert/crates/base-hub/src/gen.rs index 12445e1..8f0d9cf 100644 --- a/base-convert/crates/base-hub/src/gen.rs +++ b/base-convert/crates/base-hub/src/gen.rs @@ -73,14 +73,6 @@ fn sha256_file(path: &Path) -> Result { pub fn entry_from_base(path: &Path, id: &str, hf_repo: &str) -> Result { let header = base_format::BaseReader::read_header(path) .with_context(|| format!("read .base header {}", path.display()))?; - let backend = backend_tag(header.target_backend)?; - let bits = crate::registry::quant_bits(&header.quant_profile) - .map(str::to_string) - .unwrap_or_else(|| format!("{:?}", header.quant_scheme).to_ascii_lowercase()); - let quant = match &backend { - Some(b) => format!("{b}-{bits}"), - None => format!("default-{bits}"), - }; let size = std::fs::metadata(path) .with_context(|| format!("stat {}", path.display()))? .len(); @@ -90,10 +82,36 @@ pub fn entry_from_base(path: &Path, id: &str, hf_repo: &str) -> Result Result { + let backend = backend_tag(header.target_backend)?; + let bits = crate::registry::quant_bits(&header.quant_profile) + .map(str::to_string) + .unwrap_or_else(|| format!("{:?}", header.quant_scheme).to_ascii_lowercase()); + let quant = match &backend { + Some(b) => format!("{b}-{bits}"), + None => format!("default-{bits}"), + }; Ok(CatalogEntry { id: id.to_string(), hf_repo: hf_repo.to_string(), - file, + file: file.to_string(), revision: "main".to_string(), source_repo: None, arch: Some(header.arch.clone()), diff --git a/base-convert/crates/base-hub/src/lib.rs b/base-convert/crates/base-hub/src/lib.rs index 9f3269b..c2eef0b 100644 --- a/base-convert/crates/base-hub/src/lib.rs +++ b/base-convert/crates/base-hub/src/lib.rs @@ -12,6 +12,7 @@ pub mod download; pub mod fetch; pub mod gen; pub mod registry; +pub mod scan; pub use cache::{models_dir, HubSidecar}; pub use registry::{ModelEntry, ModelRef, Registry, SourceKind}; diff --git a/base-convert/crates/base-hub/src/registry.rs b/base-convert/crates/base-hub/src/registry.rs index 0fb81a5..87cd815 100644 --- a/base-convert/crates/base-hub/src/registry.rs +++ b/base-convert/crates/base-hub/src/registry.rs @@ -20,6 +20,22 @@ pub fn quant_bits(quant: &str) -> Option<&str> { let b = quant.as_bytes(); let mut i = 0; while i < b.len() { + // `f16` / `bf16` are bit widths too. Whisper publishes an F16 bundle + // beside its Q4 and Q8, and without this every one of them is + // uncatalogable — the catalog's own invariant is that a quant exposes + // its width so a bare `basert pull :` can match it. + if (b[i] == b'f' || (b[i] == b'b' && i + 1 < b.len() && b[i + 1] == b'f')) + && b[i..].len() > 1 + { + let fstart = if b[i] == b'b' { i + 1 } else { i }; + if fstart + 1 < b.len() && b[fstart + 1].is_ascii_digit() { + let mut j = fstart + 1; + while j < b.len() && b[j].is_ascii_digit() { + j += 1; + } + return Some(&quant[i..j]); + } + } if b[i] == b'q' && i + 1 < b.len() && b[i + 1].is_ascii_digit() { let start = i; let mut j = i + 1; @@ -605,7 +621,19 @@ mod tests { assert_eq!(quant_bits("cuda-q4mix"), Some("q4")); assert_eq!(quant_bits("q8"), Some("q8")); assert_eq!(quant_bits("cuda-q4"), Some("q4")); - assert_eq!(quant_bits("bf16"), None); // no qN token + // Float widths now count. This assertion used to read `None` ("no qN + // token"), which was defensible while every catalog row was quantized + // — and wrong the moment one was not: Whisper publishes an F16 bundle + // beside its Q4 and Q8, and a row whose quant hides its width fails + // `every_catalog_quant_exposes_its_bit_width` and cannot be selected + // by `basert pull :f16`. The alternative was to leave every + // published F16 bundle out of the catalog, which is not a smaller + // change, only a quieter one. + assert_eq!(quant_bits("bf16"), Some("bf16")); + assert_eq!(quant_bits("default-f16"), Some("f16")); + assert_eq!(quant_bits("f16"), Some("f16")); + // A bare `f` with no digits is not a width. + assert_eq!(quant_bits("default-fp"), None); assert_eq!(quant_bits("q40"), Some("q40")); // whole digit run, not "q4" } diff --git a/base-convert/crates/base-hub/src/scan.rs b/base-convert/crates/base-hub/src/scan.rs new file mode 100644 index 0000000..59ee228 --- /dev/null +++ b/base-convert/crates/base-hub/src/scan.rs @@ -0,0 +1,610 @@ +//! Derive the catalog by scanning an organization's published bundles. +//! +//! [`crate::gen`] makes a catalog row from a `.base` file the publisher has on +//! disk. That covers the moment a bundle is built and misses everything after +//! it: a model published from another machine, or by another person, or a year +//! ago, never reaches the catalog at all unless someone remembers to hand-write +//! a row. The result is silent drift — at the time this module was written, 14 +//! of the 37 repositories under `basecompute` were missing from the catalog, +//! including every Whisper bundle, so `basert list --remote` showed a third +//! less than was actually shipped. +//! +//! So the catalog is derived from the Hub instead, which is the thing that is +//! actually true. Three sources, none of which requires downloading an +//! artifact: +//! +//! * the org listing (`/api/models?author=…`) names the repositories; +//! * each repository's tree gives every file's path, size, and LFS sha256 — +//! two of the five fields a row carries, straight from the Hub; +//! * two ranged reads of each `.base` give its header, which knows the arch, +//! the backend it was packed for, and its quant profile. +//! +//! A 20GB bundle costs a few megabytes to describe. Rows are then built by +//! [`crate::gen::entry_from_header`] — the same function the local path uses, +//! so a scanned row and a published row cannot disagree. +//! +//! Scans are incremental against a known catalog: a file whose sha256 already +//! appears is described by the row that is already there, so a rescan reads +//! headers only for what is new or changed. + +use crate::catalog::{Catalog, CatalogEntry}; +use crate::fetch::Fetcher; +use anyhow::{Context, Result}; +use base_format::{Header, PREFIX_LEN}; + +/// A file as the Hub describes it, before anything is downloaded. +#[derive(Debug, Clone)] +pub struct RemoteFile { + pub path: String, + pub size: u64, + /// Content sha256 — the LFS `oid`, which is what a catalog row records. + pub sha256: Option, +} + +/// Where the file list comes from. A trait so the scan is testable without a +/// network, and because the real implementation cannot use hf-hub's typed +/// tree listing: its `BlobLfsInfo` decodes `sha256`/`pointer_size` while the +/// Hub sends `oid`/`pointerSize`, so every hash comes back `None` and every +/// bundle would look unpinnable. Reading the tree directly is both correct and +/// one less layer. +pub trait RepoIndex { + fn files(&self, repo: &str) -> Result>; +} + +/// The real Hub. +pub struct HubApi; + +impl RepoIndex for HubApi { + fn files(&self, repo: &str) -> Result> { + let url = format!("{}/api/models/{repo}/tree/main?recursive=true", endpoint()); + let body = ureq::get(&url) + .call() + .with_context(|| format!("listing files in {repo}"))? + .into_body() + .read_to_string() + .with_context(|| format!("reading the file listing for {repo}"))?; + let entries: Vec = + serde_json::from_str(&body).with_context(|| format!("parsing {repo}'s tree"))?; + Ok(entries + .iter() + .filter(|e| e.get("type").and_then(|t| t.as_str()) == Some("file")) + .filter_map(|e| { + let path = e.get("path")?.as_str()?.to_string(); + let lfs = e.get("lfs"); + Some(RemoteFile { + size: lfs + .and_then(|l| l.get("size")) + .or_else(|| e.get("size")) + .and_then(|v| v.as_u64()) + .unwrap_or(0), + sha256: lfs + .and_then(|l| l.get("oid")) + .and_then(|v| v.as_str()) + .map(str::to_string), + path, + }) + }) + .collect()) + } +} + +/// Hub API base. `$HF_ENDPOINT` redirects the whole scan at a mirror, the same +/// variable hf-hub honors for downloads. +fn endpoint() -> String { + std::env::var("HF_ENDPOINT") + .ok() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| "https://huggingface.co".to_string()) +} + +/// What a scan found, including what it deliberately did not publish. +#[derive(Debug, Default)] +pub struct ScanReport { + /// Rows for every publishable bundle, sorted for a stable catalog file. + pub entries: Vec, + /// Files that are `.base` but cannot be represented as a row, and why. + /// Reported rather than dropped: a bundle that is published but + /// uncatalogable is a fact someone needs to see. + pub skipped: Vec<(String, String)>, + /// Rows carried over from the known catalog because the file's sha256 was + /// unchanged — no header read needed. + pub reused: usize, + /// Repositories that contained no `.base` file at all. + pub empty_repos: Vec, +} + +impl ScanReport { + /// Today's date, for the catalog's `updated` field. Kept here so every + /// caller stamps it the same way. + pub fn updated_stamp(&self) -> String { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + // Civil date from a Unix timestamp (Howard Hinnant's algorithm), so + // the catalog carries a real date without a chrono dependency. + let days = (secs / 86_400) as i64; + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let y = yoe + era * 400; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = if m <= 2 { y + 1 } else { y }; + format!("{y:04}-{m:02}-{d:02}") + } +} + +/// List every model repository under `org`, following pagination. +pub fn list_org_repos(org: &str) -> Result> { + let mut out = Vec::new(); + let mut url = format!("{}/api/models?author={org}&limit=100", endpoint()); + // The Hub paginates with a `Link: <…>; rel="next"` header. Bounded so a + // malformed or cyclic Link chain cannot spin forever. + for _ in 0..50 { + let resp = ureq::get(&url) + .call() + .with_context(|| format!("listing models for {org}"))?; + let next = resp + .headers() + .get("link") + .and_then(|v| v.to_str().ok()) + .and_then(parse_next_link); + let body = resp + .into_body() + .read_to_string() + .context("reading the model listing")?; + let page: Vec = + serde_json::from_str(&body).context("parsing the model listing")?; + if page.is_empty() { + break; + } + out.extend( + page.iter() + .filter_map(|m| m.get("id").and_then(|v| v.as_str()).map(str::to_string)), + ); + match next { + Some(n) => url = n, + None => break, + } + } + out.sort(); + out.dedup(); + Ok(out) +} + +/// Pull the `rel="next"` target out of a `Link` header. +fn parse_next_link(link: &str) -> Option { + link.split(',').find_map(|part| { + if !part.contains("rel=\"next\"") { + return None; + } + let start = part.find('<')? + 1; + let end = part[start..].find('>')? + start; + Some(part[start..end].to_string()) + }) +} + +/// The file already occupying `candidate`'s catalog identity, if any. +/// +/// The catalog's uniqueness key is `(id, quant, backend)`. Two files claiming +/// it is a publishing mistake with no basis for picking a winner, so the second +/// is reported rather than emitted — and this is checked for carried-over rows +/// as well as freshly derived ones, since a duplicate that is already in the +/// catalog would otherwise be copied forward forever. +fn clashing_identity(entries: &[CatalogEntry], candidate: &CatalogEntry) -> Option { + entries + .iter() + .find(|e| { + e.id == candidate.id && e.quant == candidate.quant && e.backend == candidate.backend + }) + .map(|e| e.file.clone()) +} + +/// The `source_repo` a new row should carry: whatever a sibling row for the +/// same repository already records. +/// +/// Which upstream model a bundle was converted from is a property of the +/// model, and no `.base` header records it. Without this, publishing a new +/// quant into an existing repo would regenerate its rows with the field +/// dropped. +fn inherited_source_repo(known: &Catalog, repo: &str) -> Option { + known + .models + .iter() + .find(|m| m.hf_repo == repo && m.source_repo.is_some()) + .and_then(|m| m.source_repo.clone()) +} + +/// Read a published bundle's header without downloading the bundle. +/// +/// Two reads, because the header's length is in the header: the 16-byte prefix +/// carries magic, format version, and the JSON length; the second read takes +/// exactly that many bytes. The JSON is large — several megabytes on a model +/// with a big tokenizer — but it is still four orders of magnitude smaller than +/// the artifact, and a fixed-size guess would be wrong in both directions. +pub fn read_remote_header( + fetcher: &dyn Fetcher, + repo: &str, + revision: &str, + filename: &str, +) -> Result
{ + let prefix = fetcher.read_range(repo, revision, filename, 0..PREFIX_LEN)?; + anyhow::ensure!( + prefix.len() == PREFIX_LEN as usize, + "{filename}: short prefix ({} bytes)", + prefix.len() + ); + anyhow::ensure!(&prefix[0..4] == b"BASE", "{filename}: not a .base file"); + let header_len = u64::from_le_bytes(prefix[8..16].try_into().unwrap()); + // A header claiming to be enormous is corrupt or hostile; refuse rather + // than allocate it. 256MB is far past any real tokenizer. + anyhow::ensure!( + header_len > 0 && header_len < 256 * 1024 * 1024, + "{filename}: implausible header length {header_len}" + ); + let json = fetcher.read_range( + repo, + revision, + filename, + PREFIX_LEN..PREFIX_LEN + header_len, + )?; + Header::from_json_bytes(&json).with_context(|| format!("parsing {filename}'s header")) +} + +/// Build catalog rows for every `.base` bundle published under `org`. +/// +/// `known` supplies rows to reuse: when a file's sha256 is unchanged, its +/// existing row is kept verbatim, so the header read is skipped and any +/// hand-curated field on that row (`source_repo`, which no header knows) +/// survives the regeneration. +pub fn scan_org( + fetcher: &dyn Fetcher, + index: &dyn RepoIndex, + org: &str, + repos: &[String], + known: &Catalog, +) -> Result { + let mut report = ScanReport::default(); + for repo in repos { + let files = match index.files(repo) { + Ok(f) => f, + Err(e) => { + report + .skipped + .push((repo.clone(), format!("listing failed: {e}"))); + continue; + } + }; + let bundles: Vec<_> = files + .into_iter() + .filter(|f| f.path.ends_with(".base")) + .collect(); + if bundles.is_empty() { + report.empty_repos.push(repo.clone()); + continue; + } + for f in bundles { + let Some(sha256) = f.sha256.clone() else { + report.skipped.push(( + format!("{repo}/{}", f.path), + "the Hub reports no LFS sha256, so integrity could not be pinned".to_string(), + )); + continue; + }; + // Unchanged file: keep the row that already describes it. + if let Some(prev) = known.models.iter().find(|m| { + m.hf_repo == *repo && m.file == f.path && m.sha256.as_deref() == Some(&sha256) + }) { + if let Some(clash) = clashing_identity(&report.entries, prev) { + report.skipped.push(( + format!("{repo}/{}", f.path), + format!( + "claims the same catalog identity ({}, backend={:?}) as {}", + prev.quant, prev.backend, clash + ), + )); + continue; + } + report.entries.push(prev.clone()); + report.reused += 1; + continue; + } + let header = match read_remote_header(fetcher, repo, "main", &f.path) { + Ok(h) => h, + Err(e) => { + report + .skipped + .push((format!("{repo}/{}", f.path), format!("{e:#}"))); + continue; + } + }; + // `id` is the repo itself for a published bundle: the hub id and + // the HF coordinates are the same thing on this org. + match crate::gen::entry_from_header(&header, repo, repo, &f.path, f.size, sha256) { + Ok(mut entry) => { + // Two files claiming one identity is a publishing mistake, + // not something to pick a winner for: the catalog's key is + // (id, quant, backend) and the resolver would have no basis + // to choose. Seen in the wild — `Qwen3-0.6B-cuda-q4.base` + // carries a Metal header, so it derives `default-q4` and + // collides with `Qwen3-0.6B-Q4.base`. Report both names so + // whoever published them can fix the bundle. + if let Some(clash) = clashing_identity(&report.entries, &entry) { + report.skipped.push(( + format!("{repo}/{}", f.path), + format!( + "would claim the same catalog identity ({}, backend={:?}) as {} \ + — its header says target_backend={:?}, so check whether the \ + bundle or its filename is wrong", + entry.quant, entry.backend, clash, header.target_backend + ), + )); + continue; + } + // `source_repo` — which upstream model this was converted + // from — is a property of the model, and no header records + // it. Carry it across from any known row for the same repo + // so publishing a new quant does not quietly drop it from + // the ones already curated. + entry.source_repo = inherited_source_repo(known, repo); + report.entries.push(entry) + } + Err(e) => report + .skipped + .push((format!("{repo}/{}", f.path), format!("{e:#}"))), + } + } + } + let _ = org; + // Stable order: a regenerated catalog should diff only where the Hub + // changed, never because a listing came back in a different order. + // Universal rows before backend-locked ones, then by quant. `Catalog::find` + // returns the FIRST id match and the catalog's contract is that this is the + // recommended default — a plain alphabetical sort puts `cuda-q4mix` ahead + // of `default-q4` and quietly changes what `basert pull ` resolves to. + report.entries.sort_by(|a, b| { + (&a.id, a.backend.is_some(), &a.quant, &a.file).cmp(&( + &b.id, + b.backend.is_some(), + &b.quant, + &b.file, + )) + }); + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn next_link_is_extracted_and_absent_when_last_page() { + let h = "; rel=\"next\""; + assert_eq!( + parse_next_link(h).as_deref(), + Some("https://huggingface.co/api/models?cursor=abc") + ); + assert_eq!(parse_next_link("; rel=\"prev\""), None); + assert_eq!(parse_next_link(""), None); + } + + /// A fetcher that serves bytes from an in-memory `.base` prefix, so header + /// reading is testable without a network or a multi-GB fixture. + struct BytesFetcher { + files: Vec<(String, Vec, Option)>, + } + + impl Fetcher for BytesFetcher { + fn get_file(&self, _: &str, _: &str, _: &str) -> Result { + anyhow::bail!("unused") + } + fn list_files(&self, _: &str, _: &str) -> Result> { + Ok(self.files.iter().map(|(n, _, _)| n.clone()).collect()) + } + fn read_range( + &self, + _: &str, + _: &str, + filename: &str, + range: std::ops::Range, + ) -> Result> { + let (_, bytes, _) = self + .files + .iter() + .find(|(n, _, _)| n == filename) + .ok_or_else(|| anyhow::anyhow!("no such file"))?; + let start = range.start as usize; + let end = (range.end as usize).min(bytes.len()); + anyhow::ensure!(start <= end, "bad range"); + Ok(bytes[start..end].to_vec()) + } + } + + impl RepoIndex for BytesFetcher { + fn files(&self, _: &str) -> Result> { + Ok(self + .files + .iter() + .map(|(n, b, sha)| RemoteFile { + path: n.clone(), + size: b.len() as u64, + sha256: sha.clone(), + }) + .collect()) + } + } + + fn base_bytes(header_json: &str) -> Vec { + let mut v = b"BASE".to_vec(); + v.extend_from_slice(&1u32.to_le_bytes()); + v.extend_from_slice(&(header_json.len() as u64).to_le_bytes()); + v.extend_from_slice(header_json.as_bytes()); + v + } + + #[test] + fn a_non_base_file_is_rejected_before_any_second_read() { + let f = BytesFetcher { + files: vec![("junk.base".into(), b"NOTBASEnotbase!!".to_vec(), None)], + }; + let err = read_remote_header(&f, "org/m", "main", "junk.base").unwrap_err(); + assert!(err.to_string().contains("not a .base file"), "{err}"); + } + + #[test] + fn an_implausible_header_length_is_refused_not_allocated() { + let mut bytes = b"BASE".to_vec(); + bytes.extend_from_slice(&1u32.to_le_bytes()); + bytes.extend_from_slice(&u64::MAX.to_le_bytes()); + let f = BytesFetcher { + files: vec![("huge.base".into(), bytes, None)], + }; + let err = read_remote_header(&f, "org/m", "main", "huge.base").unwrap_err(); + assert!( + err.to_string().contains("implausible header length"), + "{err}" + ); + } + + #[test] + fn unchanged_files_reuse_their_row_and_read_no_header() { + // The bundle's bytes are deliberately NOT a valid header: if the scan + // tried to read one, this test would fail. Reuse must be decided by + // the sha256 alone. + let f = BytesFetcher { + files: vec![( + "m-Q4.base".into(), + b"not a header at all".to_vec(), + Some("abc123".into()), + )], + }; + let known = Catalog { + schema: 1, + updated: String::new(), + models: vec![CatalogEntry { + id: "basecompute/m".into(), + hf_repo: "basecompute/m".into(), + file: "m-Q4.base".into(), + revision: "main".into(), + source_repo: Some("Qwen/m".into()), + arch: Some("qwen35".into()), + quant: "default-q4".into(), + size: Some(19), + sha256: Some("abc123".into()), + backend: None, + }], + }; + let r = scan_org( + &f, + &f, + "basecompute", + &["basecompute/m".to_string()], + &known, + ) + .unwrap(); + assert_eq!(r.reused, 1); + assert_eq!(r.entries.len(), 1); + assert!(r.skipped.is_empty(), "{:?}", r.skipped); + // The curated field the header cannot know survives regeneration. + assert_eq!(r.entries[0].source_repo.as_deref(), Some("Qwen/m")); + } + + #[test] + fn a_changed_sha_forces_a_reread_and_a_bad_header_is_reported_not_dropped() { + let f = BytesFetcher { + files: vec![( + "m-Q4.base".into(), + base_bytes("{ this is not valid json"), + Some("newsha".into()), + )], + }; + let known = Catalog { + schema: 1, + updated: String::new(), + models: vec![], + }; + let r = scan_org( + &f, + &f, + "basecompute", + &["basecompute/m".to_string()], + &known, + ) + .unwrap(); + assert!(r.entries.is_empty()); + assert_eq!(r.skipped.len(), 1, "a broken bundle must be reported"); + assert!(r.skipped[0].0.contains("m-Q4.base")); + } + + #[test] + fn a_new_quant_inherits_the_repos_curated_source_repo() { + // The rule a newly published quant depends on: no header knows which + // upstream model it came from, so the value has to come from a sibling + // row or it is silently lost on regeneration. + let known = Catalog { + schema: 1, + updated: String::new(), + models: vec![ + CatalogEntry { + id: "basecompute/m".into(), + hf_repo: "basecompute/m".into(), + file: "m-Q4.base".into(), + revision: "main".into(), + source_repo: Some("Qwen/m".into()), + arch: Some("qwen35".into()), + quant: "default-q4".into(), + size: Some(1), + sha256: Some("q4sha".into()), + backend: None, + }, + CatalogEntry { + id: "basecompute/other".into(), + hf_repo: "basecompute/other".into(), + file: "other-Q4.base".into(), + revision: "main".into(), + source_repo: None, + arch: Some("llama".into()), + quant: "default-q4".into(), + size: Some(1), + sha256: Some("othersha".into()), + backend: None, + }, + ], + }; + assert_eq!( + inherited_source_repo(&known, "basecompute/m").as_deref(), + Some("Qwen/m") + ); + // A repo whose rows never had one stays None rather than borrowing a + // neighbour's. + assert_eq!(inherited_source_repo(&known, "basecompute/other"), None); + assert_eq!(inherited_source_repo(&known, "basecompute/unknown"), None); + } + + #[test] + fn a_repo_with_no_bundles_is_recorded_not_treated_as_failure() { + let f = BytesFetcher { + files: vec![("README.md".into(), b"hi".to_vec(), None)], + }; + let known = Catalog { + schema: 1, + updated: String::new(), + models: vec![], + }; + let r = scan_org( + &f, + &f, + "basecompute", + &["basecompute/docs".to_string()], + &known, + ) + .unwrap(); + assert_eq!(r.empty_repos, vec!["basecompute/docs".to_string()]); + assert!(r.entries.is_empty()); + assert!(r.skipped.is_empty()); + } +} diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 6806c5d..b16f265 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -88,11 +88,11 @@ basert serve --help | `BASERT_MODELS_DIR` | Where pulled/converted models are cached | `~/.cache/baseRT/models` | | `BASERT_LIB_PATH` / `BASERT_LIB_DIR` | Where bindings look for `libbaseRT.dylib` | `build/` at repo root | | `HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN` | HuggingFace auth for gated/private repos | unset | -| `BASERT_HF_CONNECTIONS` | Parallel range requests when a file is fetched over plain HTTPS | `8` | -| `BASERT_HF_CHUNK_MB` | Bytes per range request | `32` | +| `BASERT_HF_CONNECTIONS` | Parallel range requests per download | `24` | +| `BASERT_HF_CHUNK_MB` | Bytes per range request (peak memory is connections × chunk) | `16` | | `BASERT_HF_READ_TIMEOUT_SECS` | How long a download may stall before it is retried | `60` | | `BASERT_HF_MAX_RETRIES` | Retries per chunk (`0` fails on the first error) | `5` | -| `BASERT_HF_FORCE_RANGED` | Use ranged HTTPS even for a Xet-backed file | unset | +| `BASERT_HF_XET` | Use Xet's CAS path for large files (deduplicated, but cannot resume) | unset | ## Next steps diff --git a/docs/guides/models.md b/docs/guides/models.md index 16009c7..4544ece 100644 --- a/docs/guides/models.md +++ b/docs/guides/models.md @@ -37,35 +37,29 @@ pre-converted artifacts. ## How downloads behave -Model bundles are large — a 30B-class Q4 artifact is around 20 GB — so which -transport a file takes matters. The Hub decides per file, and `basert` follows: - -- **Xet-backed repositories** (the default for new uploads, including the - pre-converted `basecompute/` bundles) transfer over the deduplicated CAS path. - This is the fast one, and there is nothing to enable. -- **Everything else** is fetched over several connections at once, in fixed - chunks, rather than through a single stream. - -Either way, a stalled transfer is cut off and retried rather than left hanging, -and a transfer shorter than the size the Hub advertised is reported as an error -instead of being installed as a truncated model. - -### If a pull is interrupted - -The two transports differ, and the difference is worth knowing before an -overnight pull: - -- A **ranged** download resumes. Chunk completion is recorded beside the partial - file as each chunk lands, so re-running the same `basert pull` continues from - what is already on disk, including after the process has exited. -- A **Xet** download restarts from the beginning. It is fast enough that the - window is much smaller — but it is a restart, not a resume. - -On a link that drops often, `BASERT_HF_FORCE_RANGED=1` takes the ranged path -even for a Xet-backed file, trading throughput for the ability to pick up where -it stopped. The other `BASERT_HF_*` variables in -[Installation](../getting-started/installation.md#environment-variables) tune -connection count, chunk size, and the stall timeout. +Model bundles are large — a 30B-class Q4 artifact is around 20 GB — so `basert +pull` fetches them over many connections at once, in fixed chunks, rather than +through a single stream. + +- **An interrupted pull resumes.** Chunk completion is recorded beside the + partial file as each chunk lands, so a crash, a `^C`, or a link that drops + overnight continues from what is already on disk, including after the process + has exited. Re-run the same `basert pull` command. +- **A stalled transfer fails instead of hanging.** A connection that stops + delivering bytes without closing is cut off and retried. +- **A short transfer is an error.** A download smaller than the size the Hub + advertised is reported rather than installed as a truncated model. + +Defaults are 24 connections of 16 MB; peak memory is connections × chunk. Both +are tunable via the `BASERT_HF_*` variables in +[Installation](../getting-started/installation.md#environment-variables). + +### Xet + +Setting `BASERT_HF_XET=1` routes large files through Xet's content-addressed +store, which deduplicates chunks against models already fetched. It cannot +resume: an interrupted Xet transfer restarts from the beginning, which is why it +is not the default. ## Listing