Enable Cargo's new build-dir layout

src/bootstrap/src/core/build_steps/compile.rs

@@ -2652,8 +2652,8 @@ pub fn run_cargo(
 ) -> Vec<PathBuf> {
     // `target_root_dir` looks like $dir/$target/release
     let target_root_dir = stamp.path().parent().unwrap();
-    // `target_deps_dir` looks like $dir/$target/release/deps
-    let target_deps_dir = target_root_dir.join("deps");
+    // `target_build_dir` looks like $dir/$target/release/build
+    let target_build_dir = target_root_dir.join("build");
     // `host_root_dir` looks like $dir/release
     let host_root_dir = target_root_dir
         .parent()
@@ -2724,7 +2724,7 @@ pub fn run_cargo(
 
             // If this was output in the `deps` dir then this is a precise file
             // name (hash included) so we start tracking it.
-            if filename.starts_with(&target_deps_dir) {
+            if filename.starts_with(&target_build_dir) {
                 deps.push((filename.to_path_buf(), DependencyType::Target));
                 continue;
             }
@@ -2759,11 +2759,18 @@ pub fn run_cargo(
 
     // Ok now we need to actually find all the files listed in `toplevel`. We've
     // got a list of prefix/extensions and we basically just need to find the
-    // most recent file in the `deps` folder corresponding to each one.
-    let contents = target_deps_dir
+    // most recent file in the `build` folder corresponding to each one.
+    //
+    // Cargo's build folder is structured as `build/<pkg>/<hash>/out/<artifacts>` so
+    // we need to traverse multiple directory layers to get to actual files.
+    let read_dir = |path: &Path| path.read_dir().ok().into_iter().flatten().filter_map(Result::ok);
+    let contents = target_build_dir
         .read_dir()
-        .unwrap_or_else(|e| panic!("Couldn't read {}: {}", target_deps_dir.display(), e))
-        .map(|e| t!(e))
+        .unwrap_or_else(|e| panic!("Couldn't read {}: {}", target_build_dir.display(), e))
+        .map(|e| e.unwrap())
+        .flat_map(|e| read_dir(&e.path()))
+        .flat_map(|e| read_dir(&e.path()))
+        .flat_map(|e| read_dir(&e.path()))
         .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata())))
         .collect::<Vec<_>>();
     for (prefix, extension, expected_len) in toplevel {

src/bootstrap/src/core/build_steps/tool.rs

@@ -10,7 +10,7 @@
 //! return `ToolBuildResult` and should never prepare `cargo` invocations manually.
 
 use std::ffi::OsStr;
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::{env, fs};
 
 use crate::core::build_steps::compile::is_lto_stage;
@@ -1623,7 +1623,7 @@ impl Builder<'_> {
         // Notably this munges the dynamic library lookup path to point to the
         // right location to run `compiler`.
         let mut lib_paths: Vec<PathBuf> =
-            vec![self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps")];
+            discover_out_dirs(self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("build"));
 
         // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make
         // mode) and that C compiler may need some extra PATH modification. Do
@@ -1651,3 +1651,20 @@ impl Builder<'_> {
         cmd
     }
 }
+
+/// Gets all of the `out` dirs in a given Cargo `build-dir/<profile>/build` dir.
+fn discover_out_dirs(dir: PathBuf) -> Vec<PathBuf> {
+    if !dir.exists() {
+        return Vec::new();
+    }
+
+    let read_dir = |path: &Path| path.read_dir().ok().into_iter().flatten().filter_map(Result::ok);
+    dir.read_dir()
+        .unwrap_or_else(|e| panic!("Couldn't read {}: {}", dir.display(), e))
+        .map(|e| e.unwrap())
+        .flat_map(|e| read_dir(&e.path()))
+        .flat_map(|e| read_dir(&e.path()))
+        .map(|e| e.path())
+        .filter(|path| path.ends_with("out"))
+        .collect::<Vec<_>>()
+}

src/bootstrap/src/core/builder/cargo.rs

@@ -596,6 +596,8 @@ impl Builder<'_> {
 
         let mut hostflags = HostFlags::default();
 
+        cargo.arg("-Zbuild-dir-new-layout");
+
         // Codegen backends are not yet tracked by -Zbinary-dep-depinfo,
         // so we need to explicitly clear out if they've been updated.
         for backend in self.codegen_backends(compiler) {

src/tools/compiletest/src/runtest/run_make.rs

@@ -1,3 +1,4 @@
+use std::path::{Path, PathBuf};
 use std::process::{Command, Output, Stdio};
 use std::{env, fs};
 
@@ -66,8 +67,8 @@ impl TestCx<'_> {
         // build/<target_triple>/
         // ├── bootstrap-tools/
         // │   ├── <host_triple>/release/librun_make_support.rlib   // <- support rlib itself
-        // │   ├── <host_triple>/release/deps/                      // <- deps
-        // │   └── release/deps/                                    // <- deps of deps
+        // │   ├── <host_triple>/release/build/<pkg>/<hash>/out     // <- deps
+        // │   └── release/build/<pkg>/<hash>/out                   // <- deps of deps
         // ```
         //
         // FIXME(jieyouxu): there almost certainly is a better way to do this (specifically how the
@@ -77,8 +78,8 @@ impl TestCx<'_> {
         let support_host_path = tools_bin.join(&self.config.host).join("release");
         let support_lib_path = support_host_path.join("librun_make_support.rlib");
 
-        let support_lib_deps = support_host_path.join("deps");
-        let support_lib_deps_deps = tools_bin.join("release").join("deps");
+        let support_lib_deps = discover_out_dirs(support_host_path.join("build"));
+        let support_lib_deps_deps = discover_out_dirs(tools_bin.join("release").join("build"));
 
         // To compile the recipe with rustc, we need to provide suitable dynamic library search
         // paths to rustc. This includes both:
@@ -100,6 +101,10 @@ impl TestCx<'_> {
             p
         };
 
+        let out_dirs_to_args = |paths: Vec<PathBuf>| {
+            paths.into_iter().map(|p| format!("-Ldependency={}", p.display())).collect::<Vec<_>>()
+        };
+
         // run-make-support and run-make tests are compiled using the stage0 compiler
         // If the stage is 0, then the compiler that we test (either bootstrap or an explicitly
         // set compiler) is the one that actually compiled run-make-support.
@@ -119,8 +124,8 @@ impl TestCx<'_> {
             .arg(&recipe_bin)
             // Specify library search paths for `run_make_support`.
             .arg(format!("-Ldependency={}", &support_lib_path.parent().unwrap()))
-            .arg(format!("-Ldependency={}", &support_lib_deps))
-            .arg(format!("-Ldependency={}", &support_lib_deps_deps))
+            .args(out_dirs_to_args(support_lib_deps))
+            .args(out_dirs_to_args(support_lib_deps_deps))
             // Provide `run_make_support` as extern prelude, so test writers don't need to write
             // `extern run_make_support;`.
             .arg("--extern")
@@ -337,3 +342,19 @@ impl TestCx<'_> {
         }
     }
 }
+
+/// Gets all of the `out` dirs in a given Cargo `build-dir/<profile>/build` dir.
+fn discover_out_dirs(dir: Utf8PathBuf) -> Vec<PathBuf> {
+    let read_dir = |path: &Path| path.read_dir().ok().into_iter().flatten().filter_map(Result::ok);
+    let contents = dir
+        .read_dir()
+        .unwrap_or_else(|e| panic!("Couldn't read {}: {}", dir, e))
+        .map(|e| e.unwrap())
+        .flat_map(|e| read_dir(&e.path()))
+        .flat_map(|e| read_dir(&e.path()))
+        .map(|e| e.path())
+        .filter(|path| path.ends_with("out"))
+        .collect::<Vec<_>>();
+
+    return contents;
+}

src/tools/miri/test-cargo-miri/run-test.py

@@ -215,7 +215,7 @@ def test_cargo_miri_multi_target():
     if os.listdir(target_dir) != ["miri"]:
         fail(f"`{target_dir}` contains unexpected files")
     # Ensure something exists inside that target dir.
-    os.access(os.path.join(target_dir, "miri", "debug", "deps"), os.F_OK)
+    os.access(os.path.join(target_dir, "miri", "debug"), os.F_OK)
 
 print("\nTEST SUCCESSFUL!")
 sys.exit(0)

src/tools/rustfmt/.github/workflows/linux.yml

@@ -36,4 +36,7 @@ jobs:
         rustup target add ${{ matrix.target }}
 
     - name: Build and Test
+      env:
+        RUSTFLAGS: -D warnings
+        CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT: true
       run: ./ci/build_and_test.sh

src/tools/rustfmt/.github/workflows/mac.yml

@@ -32,4 +32,7 @@ jobs:
         rustup target add ${{ matrix.target }}
 
     - name: Build and Test
+      env:
+        RUSTFLAGS: -D warnings
+        CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT: true
       run: ./ci/build_and_test.sh

src/tools/rustfmt/.github/workflows/windows.yml

@@ -59,4 +59,7 @@ jobs:
 
     - name: Build and Test
       shell: cmd
+      env:
+        RUSTFLAGS: -D warnings
+        CARGO_UNSTABLE_BUILD_DIR_NEW_LAYOUT: true
       run: ci\build_and_test.bat

src/tools/rustfmt/src/test/mod.rs

@@ -1077,8 +1077,23 @@ fn rustfmt() -> PathBuf {
     let mut me = env::current_exe().expect("failed to get current executable");
     // Chop of the test name.
     me.pop();
-    // Chop off `deps`.
-    me.pop();
+
+    // Handle Cargo's old and new filesystem layouts
+    // * v1: `target/<profile>/deps/test-bin-[HASH][EXE]`
+    // * v2: `target/<profile>/build/<pkgname>/[HASH]/out/test-bin-[HASH][EXE]`
+    if me.ends_with("deps") {
+        // Chop off `deps`.
+        me.pop();
+    } else if me.ends_with("out") {
+        // Chop off `out`.
+        me.pop();
+        // Chop off `<hash>`.
+        me.pop();
+        // Chop off `<pkgname>`.
+        me.pop();
+        // Chop off `build`.
+        me.pop();
+    }
 
     me.push("rustfmt");
     assert!(

src/tools/rustfmt/tests/cargo-fmt/main.rs

@@ -10,8 +10,17 @@ use rustfmt_config_proc_macro::rustfmt_only_ci_test;
 fn cargo_fmt(args: &[&str]) -> (String, String) {
     let mut bin_dir = env::current_exe().unwrap();
     bin_dir.pop(); // chop off test exe name
+
+    // Handle Cargo's old and new filesystem layouts
+    // * v1: `target/<profile>/deps/test-bin-[HASH][EXE]`
+    // * v2: `target/<profile>/build/<pkgname>/[HASH]/out/test-bin-[HASH][EXE]`
     if bin_dir.ends_with("deps") {
         bin_dir.pop();
+    } else if bin_dir.ends_with("out") {
+        bin_dir.pop(); // chop off `out`
+        bin_dir.pop(); // chop off `<hash>`
+        bin_dir.pop(); // chop off `<pkgname>`
+        bin_dir.pop(); // chop off `build`
     }
     let cmd = bin_dir.join(format!("cargo-fmt{}", env::consts::EXE_SUFFIX));