sync: open-core source for 0.2.3

base-convert/Cargo.toml

@@ -12,11 +12,11 @@ members = [
 ]
 
 [workspace.package]
-version = "0.2.2"
+version = "0.2.3"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/basecompute/baseRT"
-rust-version = "1.80"
+rust-version = "1.85"
 
 [workspace.dependencies]
 # Internal crates
@@ -43,8 +43,14 @@ xxhash-rust = { version = "0.8", features = ["xxh64"] }
 indicatif = "0.17"
 rayon = "1"
 tempfile = "3"
-# Sync-only `api::sync` client (ureq 3 → rustls; no tokio/reqwest/OpenSSL).
-# 0.5 is required: it resolves HF's relative `Location` redirects, which 0.3.2
-# could not (HF now 307s `resolve/...` to a relative `/api/resolve-cache/...`).
-hf-hub = { version = "0.5", default-features = false, features = ["ureq"] }
+# 1.0 is required for Xet: `hf-xet` is a hard dependency there, so Xet-backed
+# repos (the default for new uploads on the Hub) transfer over the dedup CAS
+# path instead of falling back to plain HTTPS. 1.0 dropped the `ureq` feature —
+# it is reqwest/tokio underneath — so the sync surface is now the `blocking`
+# feature, which parks one current-thread runtime on a background thread.
+# `rustls-tls` keeps the build off OpenSSL, as the ureq stack did.
+hf-hub = { version = "1.0", default-features = false, features = [
+    "blocking",
+    "rustls-tls",
+] }
 dirs = "5"

base-convert/crates/base-arch/src/gemma.rs

@@ -846,15 +846,14 @@ mod tests {
         g.insert(
             "gemma4.attention.head_count_kv".into(),
             KvValue::Array(
-                std::iter::repeat([
+                std::iter::repeat_n([
                     KvValue::U32(8),
                     KvValue::U32(8),
                     KvValue::U32(8),
                     KvValue::U32(8),
                     KvValue::U32(8),
                     KvValue::U32(2),
-                ])
-                .take(5)
+                ], 5)
                 .flatten()
                 .collect(),
             ),
@@ -888,15 +887,14 @@ mod tests {
         g.insert(
             "gemma4.attention.sliding_window_pattern".into(),
             KvValue::Array(
-                std::iter::repeat([
+                std::iter::repeat_n([
                     KvValue::Bool(true),
                     KvValue::Bool(true),
                     KvValue::Bool(true),
                     KvValue::Bool(true),
                     KvValue::Bool(true),
                     KvValue::Bool(false),
-                ])
-                .take(5)
+                ], 5)
                 .flatten()
                 .collect(),
             ),

base-convert/crates/base-convert/src/hub.rs

@@ -888,6 +888,79 @@ fn human_size(bytes: u64) -> String {
     }
 }
 
+/// `basert catalog-scan` — regenerate the catalog from what an organization
+/// has actually published.
+///
+/// The catalog used to be hand-edited, one JSON row per (model, quant,
+/// backend), and drifted from the Hub the moment anyone published without
+/// remembering to add a row. This derives it instead: the org listing names
+/// the repos, each repo's tree gives size and sha256, and two ranged reads of
+/// each bundle give the header that knows its arch, backend and quant. A row
+/// whose sha256 is unchanged is carried over untouched, so a rescan is cheap
+/// and hand-curated fields (`source_repo`) survive.
+pub fn cmd_catalog_scan(org: String, out: Option<PathBuf>, dry_run: bool) -> Result<()> {
+    let root = cache::models_dir()?;
+    let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?;
+
+    let known = base_hub::catalog::Catalog::bundled()
+        .context("reading the bundled catalog to reuse unchanged rows")?;
+    eprintln!("scanning {org} …");
+    let repos = base_hub::scan::list_org_repos(&org)?;
+    eprintln!("  {} repositories", repos.len());
+
+    let report = base_hub::scan::scan_org(&fetcher, &base_hub::scan::HubApi, &org, &repos, &known)?;
+
+    // What the scan decided, before anything is written. Silence here would
+    // make a dropped bundle indistinguishable from one that does not exist.
+    eprintln!(
+        "  {} rows ({} reused unchanged), {} repos with no bundle",
+        report.entries.len(),
+        report.reused,
+        report.empty_repos.len()
+    );
+    for (what, why) in &report.skipped {
+        eprintln!("  SKIPPED {what}: {why}");
+    }
+
+    let before: std::collections::BTreeSet<_> = known
+        .models
+        .iter()
+        .map(|m| (m.hf_repo.clone(), m.file.clone()))
+        .collect();
+    let after: std::collections::BTreeSet<_> = report
+        .entries
+        .iter()
+        .map(|m| (m.hf_repo.clone(), m.file.clone()))
+        .collect();
+    for (repo, file) in after.difference(&before) {
+        eprintln!("  + {repo}/{file}");
+    }
+    for (repo, file) in before.difference(&after) {
+        eprintln!("  - {repo}/{file}  (no longer published)");
+    }
+
+    let path = out.unwrap_or_else(|| {
+        PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+            .parent()
+            .map(|p| p.join("base-hub/catalog.json"))
+            .unwrap_or_else(|| PathBuf::from("catalog.json"))
+    });
+    if dry_run {
+        eprintln!("dry run — {} not written", path.display());
+        return Ok(());
+    }
+
+    let doc = base_hub::catalog::Catalog {
+        schema: known.schema,
+        updated: report.updated_stamp(),
+        models: report.entries,
+    };
+    let json = serde_json::to_string_pretty(&doc)? + "\n";
+    std::fs::write(&path, json).with_context(|| format!("writing {}", path.display()))?;
+    eprintln!("wrote {}", path.display());
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

base-convert/crates/base-convert/src/main.rs

@@ -65,6 +65,8 @@ enum Cmd {
     Pull(PullArgs),
     /// List models in the local hub cache (and, with `--remote`, the catalog).
     List(ListArgs),
+    /// Regenerate the model catalog by scanning a published HF organization.
+    CatalogScan(CatalogScanArgs),
     /// Runtime commands — `serve`, `chat`, `complete`, `bench`, … — handled
     /// by the BaseRT runtime (dispatched in `hub::dispatch_external`).
     #[command(external_subcommand)]
@@ -266,6 +268,19 @@ struct PullArgs {
     dry_run: bool,
 }
 
+#[derive(Parser, Debug)]
+struct CatalogScanArgs {
+    /// Organization to scan.
+    #[arg(long, default_value = "basecompute")]
+    org: String,
+    /// Where to write the catalog. Defaults to the bundled one, in place.
+    #[arg(long)]
+    out: Option<PathBuf>,
+    /// Report what would change without writing anything.
+    #[arg(long)]
+    dry_run: bool,
+}
+
 #[derive(Parser, Debug)]
 struct ListArgs {
     /// Also list catalog models that aren't installed yet.
@@ -339,6 +354,7 @@ fn main() -> Result<()> {
         Cmd::Keygen(a) => cmd_keygen(a),
         Cmd::Pull(a) => hub::cmd_pull(a),
         Cmd::List(a) => hub::cmd_list(a),
+        Cmd::CatalogScan(a) => hub::cmd_catalog_scan(a.org, a.out, a.dry_run),
         Cmd::External(argv) => hub::dispatch_external(argv),
     }
 }
@@ -4943,7 +4959,7 @@ mod gguf_passthrough_tests {
         let row_bytes = 2 * 144; // 512 elements / 256 per block x 144 B
         // Give every row a distinct byte pattern so a mis-shuffle shows up.
         let src: Vec<u8> = (0..rows)
-            .flat_map(|r| std::iter::repeat((r as u8) + 1).take(row_bytes))
+            .flat_map(|r| std::iter::repeat_n((r as u8) + 1, row_bytes))
             .collect();
         let out = unpermute_rope_rows(&info(512, rows as u64, GgmlType::Q4K), &src, n_heads)
             .unwrap();

base-convert/crates/base-hub/Cargo.toml

@@ -13,8 +13,14 @@ sha2.workspace = true
 base-format.workspace = true
 hf-hub.workspace = true
 dirs.workspace = true
-# Generic HTTPS GET for the hosted catalog (same ureq 3 / rustls stack as hf-hub).
+# Download progress bars: hf-hub 1.0 reports progress as events and draws
+# nothing itself (0.5 owned the bar behind `with_progress`).
+indicatif.workspace = true
+# Generic HTTPS GET for the hosted catalog.
 ureq = "3"
+# The HTTP client handed to hf-hub, so downloads carry our own timeouts.
+# Version must track hf-hub's reqwest, or the `Client` types do not unify.
+reqwest = { version = "0.13", default-features = false }
 
 [dev-dependencies]
 tempfile.workspace = true