Auto merge of rust-lang#139244 - jieyouxu:exp/auto-cross-run-make, r=<try>

[WIP] [WIP] Enable automatic cross-compilation in run-make tests

WIP for the WIP rust-lang#138066.

Based on rust-lang#138066 with rust-lang#139242 + rust-lang#139239 cherry-picked in, plus `rustdoc()` cross-compile changes.

r? `@ghost`

try-job: armhf-gnu
try-job: test-various
try-job: x86_64-msvc-1
try-job: i686-msvc-1
try-job: x86_64-mingw-1
try-job: i686-mingw-1
try-job: aarch64-apple
try-job: x86_64-apple-1

Author: bors

Diff for: src/tools/run-make-support/src/artifact_names.rs

@@ -1,11 +1,11 @@
 //! A collection of helpers to construct artifact names, such as names of dynamic or static
-//! librarys which are target-dependent.
-
-// FIXME(jieyouxu): convert these to return `PathBuf`s instead of strings!
+//! libraries which are target-dependent.
 
+use crate::target;
 use crate::targets::is_msvc;
 
 /// Construct the static library name based on the target.
+#[track_caller]
 #[must_use]
 pub fn static_lib_name(name: &str) -> String {
     assert!(!name.contains(char::is_whitespace), "static library name cannot contain whitespace");
@@ -14,15 +14,34 @@ pub fn static_lib_name(name: &str) -> String {
 }
 
 /// Construct the dynamic library name based on the target.
+#[track_caller]
 #[must_use]
 pub fn dynamic_lib_name(name: &str) -> String {
     assert!(!name.contains(char::is_whitespace), "dynamic library name cannot contain whitespace");
 
-    format!("{}{name}.{}", std::env::consts::DLL_PREFIX, std::env::consts::DLL_EXTENSION)
+    format!("{}{name}.{}", dynamic_lib_prefix(), dynamic_lib_extension())
+}
+
+fn dynamic_lib_prefix() -> &'static str {
+    if target().contains("windows") { "" } else { "lib" }
 }
 
-/// Construct the name of the import library for the dynamic library, exclusive to MSVC and
-/// accepted by link.exe.
+/// Construct the dynamic library extension based on the target.
+#[must_use]
+pub fn dynamic_lib_extension() -> &'static str {
+    let target = target();
+
+    if target.contains("apple") {
+        "dylib"
+    } else if target.contains("windows") {
+        "dll"
+    } else {
+        "so"
+    }
+}
+
+/// Construct the name of the import library for the dynamic library, exclusive to MSVC and accepted
+/// by link.exe.
 #[track_caller]
 #[must_use]
 pub fn msvc_import_dynamic_lib_name(name: &str) -> String {
@@ -32,20 +51,26 @@ pub fn msvc_import_dynamic_lib_name(name: &str) -> String {
     format!("{name}.dll.lib")
 }
 
-/// Construct the dynamic library extension based on the target.
-#[must_use]
-pub fn dynamic_lib_extension() -> &'static str {
-    std::env::consts::DLL_EXTENSION
-}
-
 /// Construct the name of a rust library (rlib).
+#[track_caller]
 #[must_use]
 pub fn rust_lib_name(name: &str) -> String {
     format!("lib{name}.rlib")
 }
 
 /// Construct the binary (executable) name based on the target.
+#[track_caller]
 #[must_use]
 pub fn bin_name(name: &str) -> String {
-    format!("{name}{}", std::env::consts::EXE_SUFFIX)
+    let target = target();
+
+    if target.contains("windows") {
+        format!("{name}.exe")
+    } else if target.contains("uefi") {
+        format!("{name}.efi")
+    } else if target.contains("wasm") {
+        format!("{name}.wasm")
+    } else {
+        name.to_string()
+    }
 }

Diff for: src/tools/run-make-support/src/external_deps/rustc.rs

@@ -6,7 +6,7 @@ use crate::command::Command;
 use crate::env::env_var;
 use crate::path_helpers::cwd;
 use crate::util::set_host_compiler_dylib_path;
-use crate::{is_aix, is_darwin, is_msvc, is_windows, uname};
+use crate::{is_aix, is_darwin, is_msvc, is_windows, target, uname};
 
 /// Construct a new `rustc` invocation. This will automatically set the library
 /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
@@ -27,9 +27,15 @@ pub fn bare_rustc() -> Rustc {
 #[must_use]
 pub struct Rustc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustc, |rustc: &mut Rustc| {
+    if let Some(target) = &rustc.target {
+        rustc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 pub fn rustc_path() -> String {
     env_var("RUSTC")
@@ -46,19 +52,22 @@ impl Rustc {
     // `rustc` invocation constructor methods
 
     /// Construct a new `rustc` invocation. This will automatically set the library
-    /// search path as `-L cwd()`. Use [`bare_rustc`] to avoid this.
+    /// search path as `-L cwd()` and also the compilation target.
+    /// Use [`bare_rustc`] to avoid this.
     #[track_caller]
     pub fn new() -> Self {
         let mut cmd = setup_common();
         cmd.arg("-L").arg(cwd());
-        Self { cmd }
+
+        // Automatically default to cross-compilation
+        Self { cmd, target: Some(target()) }
     }
 
     /// Construct a bare `rustc` invocation with no flags set.
     #[track_caller]
     pub fn bare() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: None }
     }
 
     // Argument provider methods
@@ -234,8 +243,9 @@ impl Rustc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in Rustc::new().
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

Diff for: src/tools/run-make-support/src/external_deps/rustdoc.rs

@@ -3,21 +3,35 @@ use std::path::Path;
 
 use crate::command::Command;
 use crate::env::env_var;
+use crate::target;
 use crate::util::set_host_compiler_dylib_path;
 
-/// Construct a new `rustdoc` invocation.
+/// Construct a new `rustdoc` invocation with target automatically set to cross-compile target. Use
+/// [`bare_rustdoc`] to avoid this.
 #[track_caller]
 pub fn rustdoc() -> Rustdoc {
     Rustdoc::new()
 }
 
+/// Bare `rustdoc` invocation, no args set.
+#[track_caller]
+pub fn bare_rustdoc() -> Rustdoc {
+    Rustdoc::bare()
+}
+
 #[derive(Debug)]
 #[must_use]
 pub struct Rustdoc {
     cmd: Command,
+    target: Option<String>,
 }
 
-crate::macros::impl_common_helpers!(Rustdoc);
+// Only fill in the target just before execution, so that it can be overridden.
+crate::macros::impl_common_helpers!(Rustdoc, |rustdoc: &mut Rustdoc| {
+    if let Some(target) = &rustdoc.target {
+        rustdoc.cmd.arg(&format!("--target={target}"));
+    }
+});
 
 #[track_caller]
 fn setup_common() -> Command {
@@ -28,11 +42,19 @@ fn setup_common() -> Command {
 }
 
 impl Rustdoc {
-    /// Construct a bare `rustdoc` invocation.
+    /// Construct a new `rustdoc` invocation with target automatically set to cross-compile target.
+    /// Use [`bare_rustdoc`] to avoid this.
     #[track_caller]
     pub fn new() -> Self {
         let cmd = setup_common();
-        Self { cmd }
+        Self { cmd, target: Some(target()) }
+    }
+
+    /// Bare `rustdoc` invocation, no args set.
+    #[track_caller]
+    pub fn bare() -> Self {
+        let cmd = setup_common();
+        Self { cmd, target: None }
     }
 
     /// Specify where an external library is located.
@@ -85,8 +107,9 @@ impl Rustdoc {
 
     /// Specify the target triple, or a path to a custom target json spec file.
     pub fn target<S: AsRef<str>>(&mut self, target: S) -> &mut Self {
-        let target = target.as_ref();
-        self.cmd.arg(format!("--target={target}"));
+        // We store the target as a separate field, so that it can be specified multiple times.
+        // This is in particular useful to override the default target set in `Rustdoc::new()`.
+        self.target = Some(target.as_ref().to_string());
         self
     }
 

Diff for: src/tools/run-make-support/src/lib.rs

@@ -69,7 +69,7 @@ pub use llvm::{
 };
 pub use python::python_command;
 pub use rustc::{bare_rustc, rustc, rustc_path, Rustc};
-pub use rustdoc::{rustdoc, Rustdoc};
+pub use rustdoc::{bare_rustdoc, rustdoc, Rustdoc};
 
 /// [`diff`][mod@diff] is implemented in terms of the [similar] library.
 ///

Diff for: src/tools/run-make-support/src/macros.rs

@@ -23,10 +23,16 @@
 /// }
 /// ```
 ///
+/// You can pass an optional second parameter which should be a function that is passed
+/// `&mut self` just before the command is executed.
+///
 /// [`Command`]: crate::command::Command
 /// [`CompletedProcess`]: crate::command::CompletedProcess
 macro_rules! impl_common_helpers {
     ($wrapper: ident) => {
+        $crate::macros::impl_common_helpers!($wrapper, |_| {});
+    };
+    ($wrapper: ident, $before_exec: expr) => {
         impl $wrapper {
             /// Specify an environment variable.
             pub fn env<K, V>(&mut self, key: K, value: V) -> &mut Self
@@ -118,12 +124,14 @@ macro_rules! impl_common_helpers {
             /// Run the constructed command and assert that it is successfully run.
             #[track_caller]
             pub fn run(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run()
             }
 
             /// Run the constructed command and assert that it does not successfully run.
             #[track_caller]
             pub fn run_fail(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_fail()
             }
 
@@ -133,6 +141,7 @@ macro_rules! impl_common_helpers {
             /// whenever possible.
             #[track_caller]
             pub fn run_unchecked(&mut self) -> crate::command::CompletedProcess {
+                $before_exec(&mut *self);
                 self.cmd.run_unchecked()
             }
 

Diff for: tests/run-make/apple-deployment-target/rmake.rs

@@ -41,7 +41,6 @@ fn main() {
 
     // Remove env vars to get `rustc`'s default
     let output = rustc()
-        .target(target())
         .env_remove("MACOSX_DEPLOYMENT_TARGET")
         .env_remove("IPHONEOS_DEPLOYMENT_TARGET")
         .env_remove("WATCHOS_DEPLOYMENT_TARGET")
@@ -58,7 +57,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("lib");
             rustc.emit("obj");
             rustc.input("foo.rs");
@@ -82,7 +80,6 @@ fn main() {
 
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("dylib");
             rustc.input("foo.rs");
             rustc.output("libfoo.dylib");
@@ -108,7 +105,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.crate_type("bin");
             rustc.input("foo.rs");
             rustc.output("foo");
@@ -147,7 +143,6 @@ fn main() {
     run_in_tmpdir(|| {
         let rustc = || {
             let mut rustc = rustc();
-            rustc.target(target());
             rustc.incremental("incremental");
             rustc.crate_type("lib");
             rustc.emit("obj");

Diff for: tests/run-make/apple-sdk-version/rmake.rs

@@ -24,8 +24,7 @@ fn has_sdk_version(file: &str, version: &str) {
 
 fn main() {
     // Fetch rustc's inferred deployment target.
-    let current_deployment_target =
-        rustc().target(target()).print("deployment-target").run().stdout_utf8();
+    let current_deployment_target = rustc().print("deployment-target").run().stdout_utf8();
     let current_deployment_target = current_deployment_target.split('=').last().unwrap().trim();
 
     // Fetch current SDK version via. xcrun.
@@ -45,15 +44,15 @@ fn main() {
     let current_sdk_version = current_sdk_version.trim();
 
     // Check the SDK version in the object file produced by the codegen backend.
-    rustc().target(target()).crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
+    rustc().crate_type("lib").emit("obj").input("foo.rs").output("foo.o").run();
     // Set to 0, which means not set or "n/a".
     has_sdk_version("foo.o", "n/a");
 
     // Check the SDK version in the .rmeta file, as set in `create_object_file`.
     //
     // This is just to ensure that we don't set some odd version in `create_object_file`,
     // if the rmeta file is packed in a different way in the future, this can safely be removed.
-    rustc().target(target()).crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
+    rustc().crate_type("rlib").input("foo.rs").output("libfoo.rlib").run();
     // Extra .rmeta file (which is encoded as an object file).
     cmd("ar").arg("-x").arg("libfoo.rlib").arg("lib.rmeta").run();
     has_sdk_version("lib.rmeta", "n/a");
@@ -69,7 +68,6 @@ fn main() {
         // Test with clang
         let file_name = format!("foo_cc{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=gcc")
             .input("foo.rs")
@@ -80,7 +78,6 @@ fn main() {
         // Test with ld64
         let file_name = format!("foo_ld{file_ext}");
         rustc()
-            .target(target())
             .crate_type("bin")
             .arg("-Clinker-flavor=ld")
             .input("foo.rs")