bootstrap: support shell argfiles in compiler shims

src/bootstrap/Cargo.lock

@@ -61,6 +61,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "sha2",
+ "shlex",
  "sysinfo",
  "tar",
  "tempfile",

src/bootstrap/Cargo.toml

@@ -50,6 +50,7 @@ serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
 sha2 = "0.10"
+shlex = "1.0"
 tar = { version = "0.4.45", default-features = false }
 termcolor = "1.4"
 toml = "0.5"

src/bootstrap/src/utils/shared_helpers.rs

@@ -130,17 +130,66 @@ pub fn parse_value_from_args<'a>(args: &'a [OsString], key: &str) -> Option<&'a
 
 /// Collect all the command line arguments, including the arguments from any `@argfile`
 pub fn collect_args() -> Vec<OsString> {
-    let mut args = Vec::with_capacity(env::args_os().len());
-    for arg in env::args_os().skip(1) {
-        if let Some(s) = arg.to_str()
-            && let Some(path) = s.strip_prefix('@')
-        {
-            args.extend(args_from_argfile(Path::new(path)));
+    collect_args_from(env::args_os().skip(1))
+}
+
+pub(crate) fn collect_args_from(args: impl IntoIterator<Item = OsString>) -> Vec<OsString> {
+    // The shim must inspect the same expanded arguments that rustc receives.
+    let mut expander = ArgExpander::default();
+    for arg in args {
+        expander.arg(arg);
+    }
+    expander.args
+}
+
+#[derive(Default)]
+struct ArgExpander {
+    shell_argfiles: bool,
+    next_is_unstable_option: bool,
+    args: Vec<OsString>,
+}
+
+impl ArgExpander {
+    fn arg(&mut self, arg: OsString) {
+        if let Some(argfile) = arg.to_str().and_then(|arg| arg.strip_prefix('@')) {
+            match argfile.split_once(':') {
+                Some(("shell", path)) if self.shell_argfiles => {
+                    for arg in shell_args_from_argfile(Path::new(path)) {
+                        self.push(arg);
+                    }
+                }
+                _ => {
+                    for arg in args_from_argfile(Path::new(argfile)) {
+                        self.push(arg);
+                    }
+                }
+            }
         } else {
-            args.push(arg)
+            self.push(arg);
+        }
+    }
+
+    fn push(&mut self, arg: OsString) {
+        if let Some(arg) = arg.to_str() {
+            if self.next_is_unstable_option {
+                self.inspect_unstable_option(arg);
+                self.next_is_unstable_option = false;
+            } else if let Some(option) = arg.strip_prefix("-Z") {
+                if option.is_empty() {
+                    self.next_is_unstable_option = true;
+                } else {
+                    self.inspect_unstable_option(option);
+                }
+            }
+        }
+        self.args.push(arg);
+    }
+
+    fn inspect_unstable_option(&mut self, option: &str) {
+        if option == "shell-argfiles" {
+            self.shell_argfiles = true;
         }
     }
-    args
 }
 
 /// Reads all the arguments from argfile given by `path`.
@@ -154,3 +203,14 @@ fn args_from_argfile(path: &Path) -> Vec<OsString> {
     }
     collect_lines(path).expect("read args from argfile {path:?}")
 }
+
+fn shell_args_from_argfile(path: &Path) -> Vec<OsString> {
+    let contents = std::fs::read_to_string(path).unwrap_or_else(|err| {
+        panic!("failed to read shell args from argfile {}: {err}", path.display())
+    });
+    shlex::split(&contents)
+        .unwrap_or_else(|| panic!("failed to parse shell args from argfile {}", path.display()))
+        .into_iter()
+        .map(OsString::from)
+        .collect()
+}

src/bootstrap/src/utils/tests/shared_helpers_tests.rs

@@ -5,7 +5,10 @@
 //! To prevent tidy from complaining about this file not being named `tests.rs`, it lives inside a
 //! submodule directory named `tests`.
 
-use crate::utils::shared_helpers::parse_value_from_args;
+use std::ffi::OsString;
+use std::fs;
+
+use crate::utils::shared_helpers::{collect_args_from, parse_value_from_args};
 
 #[test]
 fn test_parse_value_from_args() {
@@ -33,3 +36,41 @@ fn test_parse_value_from_args() {
     assert_eq!(parse_value_from_args(args.as_slice(), "--key").unwrap(), "value");
     assert_eq!(parse_value_from_args(args.as_slice(), "--sysroot").unwrap(), "/x/y/z");
 }
+
+#[test]
+fn test_collect_args_expands_shell_argfile_enabled_in_argfile() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let enable_shell_argfiles = tempdir.path().join("enable-shell-argfiles.args");
+    let shell_argfile = tempdir.path().join("shell-argfiles.args");
+    fs::write(&enable_shell_argfiles, "-Zshell-argfiles\n").unwrap();
+    fs::write(&shell_argfile, "--target 'x86_64-unknown-linux-gnu' --cfg shell_argfile\n").unwrap();
+
+    let args = vec![
+        format!("@{}", enable_shell_argfiles.display()).into(),
+        format!("@shell:{}", shell_argfile.display()).into(),
+    ];
+
+    assert_eq!(
+        collect_args_from(args),
+        ["-Zshell-argfiles", "--target", "x86_64-unknown-linux-gnu", "--cfg", "shell_argfile"]
+            .map(OsString::from)
+    );
+}
+
+#[test]
+fn test_collect_args_expands_shell_argfile_enabled_by_separate_option() {
+    let tempdir = tempfile::tempdir().unwrap();
+    let shell_argfile = tempdir.path().join("shell-argfiles.args");
+    fs::write(&shell_argfile, "--target 'x86_64-unknown-linux-gnu'\n").unwrap();
+
+    let args = vec![
+        "-Z".into(),
+        "shell-argfiles".into(),
+        format!("@shell:{}", shell_argfile.display()).into(),
+    ];
+
+    assert_eq!(
+        collect_args_from(args),
+        ["-Z", "shell-argfiles", "--target", "x86_64-unknown-linux-gnu"].map(OsString::from)
+    );
+}