Add a Local environment to opt-dist

This makes it easier to build a PGO/BOLT optimized `rustc` locally, outside of CI.

Author: Kobzol

src/tools/opt-dist/src/environment.rs

@@ -7,15 +7,16 @@ pub struct Environment {
     python_binary: String,
     /// The rustc checkout, where the compiler source is located.
     checkout_dir: Utf8PathBuf,
-    /// The main directory where the build occurs.
+    /// The main directory where the build occurs. Stage0 rustc and cargo have to be available in
+    /// this directory before `opt-dist` is started.
     build_dir: Utf8PathBuf,
     /// Directory where the optimization artifacts (PGO/BOLT profiles, etc.)
     /// will be stored.
     artifact_dir: Utf8PathBuf,
     /// Path to the host LLVM used to compile LLVM in `src/llvm-project`.
     host_llvm_dir: Utf8PathBuf,
     /// List of test paths that should be skipped when testing the optimized artifacts.
-    skipped_tests: Vec<&'static str>,
+    skipped_tests: Vec<String>,
     use_bolt: bool,
     shared_llvm: bool,
 }
@@ -83,7 +84,7 @@ impl Environment {
         self.shared_llvm
     }
 
-    pub fn skipped_tests(&self) -> &[&'static str] {
+    pub fn skipped_tests(&self) -> &[String] {
         &self.skipped_tests
     }
 }

src/tools/opt-dist/src/main.rs

@@ -42,10 +42,51 @@ struct SharedArgs {
 
 #[derive(clap::Parser, Clone, Debug)]
 enum EnvironmentCmd {
+    /// Perform a custom local PGO/BOLT optimized build.
+    Local {
+        /// Target triple of the host.
+        #[arg(long)]
+        target_triple: String,
+
+        /// Checkout directory of `rustc`.
+        #[arg(long)]
+        checkout_dir: Utf8PathBuf,
+
+        /// Host LLVM installation directory.
+        #[arg(long)]
+        llvm_dir: Utf8PathBuf,
+
+        /// Python binary to use in bootstrap invocations.
+        #[arg(long, default_value = "python3")]
+        python: String,
+
+        /// Directory where artifacts (like PGO profiles or rustc-perf) of this workflow
+        /// will be stored.
+        #[arg(long, default_value = "opt-artifacts")]
+        artifact_dir: Utf8PathBuf,
+
+        /// Is LLVM for `rustc` built in shared library mode?
+        #[arg(long, default_value_t = true)]
+        llvm_shared: bool,
+
+        /// Should BOLT optimization be used? If yes, host LLVM must have BOLT binaries
+        /// (`llvm-bolt` and `merge-fdata`) available.
+        #[arg(long, default_value_t = false)]
+        use_bolt: bool,
+
+        /// Tests that should be skipped when testing the optimized compiler.
+        #[arg(long)]
+        skipped_tests: Vec<String>,
+
+        #[clap(flatten)]
+        shared: SharedArgs,
+    },
+    /// Perform an optimized build on Linux CI, from inside Docker.
     LinuxCi {
         #[clap(flatten)]
         shared: SharedArgs,
     },
+    /// Perform an optimized build on Windows CI, directly inside Github Actions.
     WindowsCi {
         #[clap(flatten)]
         shared: SharedArgs,
@@ -58,6 +99,34 @@ fn is_try_build() -> bool {
 
 fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)> {
     let (env, args) = match args.env {
+        EnvironmentCmd::Local {
+            target_triple,
+            checkout_dir,
+            llvm_dir,
+            python,
+            artifact_dir,
+            llvm_shared,
+            use_bolt,
+            skipped_tests,
+            shared,
+        } => {
+            let env = EnvironmentBuilder::default()
+                .host_triple(target_triple)
+                .python_binary(python)
+                .checkout_dir(checkout_dir.clone())
+                .host_llvm_dir(llvm_dir)
+                .artifact_dir(artifact_dir)
+                .build_dir(checkout_dir)
+                .shared_llvm(llvm_shared)
+                .use_bolt(use_bolt)
+                .skipped_tests(skipped_tests)
+                .build()?;
+            with_log_group("Building rustc-perf", || {
+                Ok::<(), anyhow::Error>(download_rustc_perf(&env)?)
+            })?;
+
+            (env, shared.build_args)
+        }
         EnvironmentCmd::LinuxCi { shared } => {
             let target_triple =
                 std::env::var("PGO_HOST").expect("PGO_HOST environment variable missing");
@@ -74,7 +143,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)>
                 .use_bolt(true)
                 .skipped_tests(vec![
                     // Fails because of linker errors, as of June 2023.
-                    "tests/ui/process/nofile-limit.rs",
+                    "tests/ui/process/nofile-limit.rs".to_string(),
                 ])
                 .build()?;
             // /tmp/rustc-perf comes from the x64 dist Dockerfile
@@ -100,7 +169,7 @@ fn create_environment(args: Args) -> anyhow::Result<(Environment, Vec<String>)>
                 .use_bolt(false)
                 .skipped_tests(vec![
                     // Fails as of June 2023.
-                    "tests\\codegen\\vec-shrink-panik.rs",
+                    "tests\\codegen\\vec-shrink-panik.rs".to_string(),
                 ])
                 .build()?;
 
@@ -309,6 +378,8 @@ fn copy_rustc_perf(env: &Environment, dir: &Utf8Path) -> anyhow::Result<()> {
 
 // Download and build rustc-perf into the given environment.
 fn download_rustc_perf(env: &Environment) -> anyhow::Result<()> {
+    reset_directory(&env.rustc_perf_dir())?;
+
     // FIXME: add some mechanism for synchronization of this commit SHA with
     // Linux (which builds rustc-perf in a Dockerfile)
     // rustc-perf version from 2023-05-30