Add support for build-dir artifact duplication

Cargo.toml

@@ -86,6 +86,7 @@ percent-encoding = "2.3.2"
 proptest = "1.11.0"
 pulldown-cmark = { version = "0.13.3", default-features = false, features = ["html"] }
 rand = "0.10.1"
+reflink-copy = "0.1.30"
 regex = "1.12.3"
 rusqlite = { version = "0.40.0", features = ["bundled"] }
 rustc-hash = "2.1.2"
@@ -202,6 +203,7 @@ os_info.workspace = true
 pathdiff.workspace = true
 portable-atomic.workspace = true
 rand.workspace = true
+reflink-copy.workspace = true
 regex.workspace = true
 rusqlite = { workspace = true, features = ["fallible_uint"] }
 rustc-hash.workspace = true

crates/cargo-util/src/paths.rs

@@ -163,6 +163,49 @@ pub fn symlink_metadata<P: AsRef<Path>>(path: P) -> Result<Metadata> {
         .with_context(|| format!("failed to load metadata for path `{}`", path.display()))
 }
 
+/// Returns the number of hard links to a file.
+#[cfg(unix)]
+pub fn nlink(metadata: &Metadata, _path: &Path) -> io::Result<u64> {
+    use std::os::unix::fs::MetadataExt;
+    Ok(metadata.nlink())
+}
+
+/// Returns the number of hard links to a file.
+#[cfg(windows)]
+pub fn nlink(_metadata: &Metadata, path: &Path) -> io::Result<u64> {
+    use std::os::windows::ffi::OsStrExt;
+    use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle};
+    use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE;
+    use windows_sys::Win32::Storage::FileSystem::{
+        BY_HANDLE_FILE_INFORMATION, CreateFileW, FILE_ATTRIBUTE_NORMAL, FILE_READ_ATTRIBUTES,
+        FILE_SHARE_DELETE, FILE_SHARE_READ, FILE_SHARE_WRITE, GetFileInformationByHandle,
+        OPEN_EXISTING,
+    };
+
+    let mut wide: Vec<u16> = path.as_os_str().encode_wide().collect();
+    wide.push(0);
+    let handle = unsafe {
+        CreateFileW(
+            wide.as_ptr(),
+            FILE_READ_ATTRIBUTES,
+            FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+            std::ptr::null(),
+            OPEN_EXISTING,
+            FILE_ATTRIBUTE_NORMAL,
+            std::ptr::null_mut(),
+        )
+    };
+    if handle == INVALID_HANDLE_VALUE {
+        return Err(io::Error::last_os_error());
+    }
+    let handle = unsafe { OwnedHandle::from_raw_handle(handle) };
+    let mut info: BY_HANDLE_FILE_INFORMATION = unsafe { std::mem::zeroed() };
+    if unsafe { GetFileInformationByHandle(handle.as_raw_handle(), &mut info) } == 0 {
+        return Err(io::Error::last_os_error());
+    }
+    Ok(info.nNumberOfLinks as u64)
+}
+
 /// Reads a file to a string.
 ///
 /// Equivalent to [`std::fs::read_to_string`] with better error messages.
@@ -887,6 +930,7 @@ fn exclude_from_time_machine_and_cloud_sync(path: &Path) {
 #[cfg(test)]
 mod tests {
     use super::join_paths;
+    use super::nlink;
     use super::normalize_path;
     use super::write;
     use super::write_atomic;
@@ -1103,4 +1147,20 @@ mod tests {
         assert!(!symlink_path.exists());
         assert!(file_path.exists());
     }
+
+    #[test]
+    fn hard_link_count_tracks_links() {
+        let dir = tempfile::tempdir().unwrap();
+        let file = dir.path().join("file");
+        let linked = dir.path().join("linked");
+        std::fs::write(&file, "contents").unwrap();
+        assert_eq!(nlink(&file.metadata().unwrap(), &file).unwrap(), 1);
+
+        std::fs::hard_link(&file, &linked).unwrap();
+        assert_eq!(nlink(&file.metadata().unwrap(), &file).unwrap(), 2);
+        assert_eq!(nlink(&linked.metadata().unwrap(), &linked).unwrap(), 2);
+
+        std::fs::remove_file(&linked).unwrap();
+        assert_eq!(nlink(&file.metadata().unwrap(), &file).unwrap(), 1);
+    }
 }

doc/book/src/reference/unstable.md

@@ -101,6 +101,7 @@ Each new feature described below should explain how to use it.
     * [json-target-spec](#json-target-spec) --- Allows the use of `.json` custom target specs.
     * [hint-msrv](#hint-msrv) --- Allows Cargo to set `-Zhint-msrv`
     * [mem-stats](#mem-stats) --- Reports peak memory usage per build unit in `--timings`.
+    * [shared-blob-storage](#shared-blob-storage) --- Deduplicates build artifacts between workspaces
 * rustdoc
     * [rustdoc-map](#rustdoc-map) --- Provides mappings for documentation to link to external sites like [docs.rs](https://docs.rs/).
     * [scrape-examples](#scrape-examples) --- Shows examples within documentation.
@@ -2077,6 +2078,21 @@ option:
 hint-msrv = true
 ```
 
+## shared-blob-storage
+* Tracking Issue: [rust-lang/cargo#17453](https://github.com/rust-lang/cargo/issues/17453)
+
+Enables Cargo to automatically deduplicate files across build units and workspaces by hardlink/reflinking
+against a shared content addressable blob storage located in `~/.cargo/blobs`.
+
+You can set this via your global `~/.cargo/config.toml`, and nightly Cargo will
+automatically use it, while stable Cargo will silently ignore the unstable
+option:
+
+```toml
+[unstable]
+shared-blob-storage = true
+```
+
 ## builtin-dependencies
 * Tracking Issue: [rust-lang/cargo#16960](https://github.com/rust-lang/cargo/issues/16960)
 - RFC: [rust-lang/rfcs#3875](https://rust-lang.github.io/rfcs/3875-build-std-explicit-dependencies.html)

src/bin/cargo/commands/clean.rs

@@ -130,6 +130,15 @@ pub fn cli() -> Command {
                     )
                     .value_name("SIZE")
                     .value_parser(parse_human_size),
+                )
+                .arg(
+                    opt(
+                        "max-blob-size",
+                        "Deletes shared blobs until the cache is under the \
+                        given size (unstable)",
+                    )
+                    .value_name("SIZE")
+                    .value_parser(parse_human_size),
                 ),
         )
         .after_help(color_print::cstr!(
@@ -199,6 +208,7 @@ fn gc(gctx: &GlobalContext, args: &ArgMatches) -> CliResult {
         max_crate_size: size_opt("max-crate-size"),
         max_git_size: size_opt("max-git-size"),
         max_download_size: size_opt("max-download-size"),
+        max_blob_size: size_opt("max-blob-size"),
     };
     if let Some(age) = duration_opt("max-download-age") {
         gc_opts.set_max_download_age(age);