Add support for downloading libgccjit.so file to prevent requiring to build it yourself

Author: GuillaumeGomez

compiler/rustc_codegen_gcc/build_system/src/config.rs

@@ -444,15 +444,17 @@ impl ConfigInfo {
             "build_sysroot/sysroot/lib/rustlib/{}/lib",
             self.target_triple,
         ));
-        let ld_library_path = format!(
-            "{target}:{sysroot}:{gcc_path}",
-            target = self.cargo_target_dir,
-            sysroot = sysroot.display(),
-            gcc_path = self.gcc_path,
-        );
-        env.insert("LIBRARY_PATH".to_string(), ld_library_path.clone());
-        env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
-        env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
+        if !use_system_gcc {
+            let ld_library_path = format!(
+                "{target}:{sysroot}:{gcc_path}",
+                target = self.cargo_target_dir,
+                sysroot = sysroot.display(),
+                gcc_path = self.gcc_path,
+            );
+            env.insert("LIBRARY_PATH".to_string(), ld_library_path.clone());
+            env.insert("LD_LIBRARY_PATH".to_string(), ld_library_path.clone());
+            env.insert("DYLD_LIBRARY_PATH".to_string(), ld_library_path);
+        }
 
         // NOTE: To avoid the -fno-inline errors, use /opt/gcc/bin/gcc instead of cc.
         // To do so, add a symlink for cc to /opt/gcc/bin/gcc in our PATH.

config.example.toml

@@ -155,6 +155,13 @@
 # Custom CMake defines to set when building LLVM.
 #build-config = {}
 
+# =============================================================================
+# General configuration options for the GCC backend
+# =============================================================================
+[gcc]
+# Change to `true` if you want to use the version used in Rust CI.
+# download-gccjit = false
+
 # =============================================================================
 # General build configuration options
 # =============================================================================

src/bootstrap/src/core/build_steps/compile.rs

@@ -20,6 +20,7 @@ use std::str;
 use serde_derive::Deserialize;
 
 use crate::core::build_steps::dist;
+use crate::core::build_steps::gcc::set_gccjit_library_path;
 use crate::core::build_steps::llvm;
 use crate::core::build_steps::tool::SourceType;
 use crate::core::builder;
@@ -1365,6 +1366,10 @@ impl Step for CodegenBackend {
             .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml")));
         rustc_cargo_env(builder, &mut cargo, target, compiler.stage);
 
+        if backend == "gcc" {
+            set_gccjit_library_path(&builder.config, &mut cargo);
+        }
+
         let tmp_stamp = out_dir.join(".tmp.stamp");
 
         let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target);

src/bootstrap/src/core/build_steps/gcc.rs

@@ -0,0 +1,36 @@
+//! Retrieves information related to GCC backend.
+
+use crate::builder::Cargo;
+
+/// This retrieves the gccjit sha we use.
+pub(crate) fn detect_gccjit_sha() -> String {
+    // FIXME: This is absolutely ugly. Please find another way to do that.
+    let content =
+        include_str!("../../../../ci/docker/host-x86_64/dist-x86_64-linux/build-gccjit.sh");
+    if let Some(commit) = content
+        .lines()
+        .filter_map(|line| line.strip_prefix("GIT_COMMIT=\""))
+        .filter_map(|commit| commit.strip_suffix("\""))
+        .next()
+    {
+        if !commit.is_empty() {
+            return commit.to_string();
+        }
+    }
+
+    eprintln!("error: could not find commit hash for downloading libgccjit");
+    panic!();
+}
+
+pub fn set_gccjit_library_path(config: &crate::Config, cargo: &mut Cargo) {
+    if config.gcc_download_gccjit {
+        let libgccjit_path = config.libgccjit_folder(&detect_gccjit_sha());
+        let mut library_path = std::env::var("LD_LIBRARY_PATH").unwrap_or_default();
+        if !library_path.is_empty() {
+            library_path.push(':');
+        }
+        library_path.push_str(&libgccjit_path.display().to_string());
+        cargo.env("LD_LIBRARY_PATH", &library_path);
+        cargo.env("LIBRARY_PATH", library_path);
+    }
+}

src/bootstrap/src/core/build_steps/mod.rs

@@ -5,6 +5,7 @@ pub(crate) mod compile;
 pub(crate) mod dist;
 pub(crate) mod doc;
 pub(crate) mod format;
+pub(crate) mod gcc;
 pub(crate) mod install;
 pub(crate) mod llvm;
 pub(crate) mod run;

src/bootstrap/src/core/build_steps/test.rs

@@ -16,6 +16,7 @@ use clap_complete::shells;
 use crate::core::build_steps::compile;
 use crate::core::build_steps::dist;
 use crate::core::build_steps::doc::DocumentationFormat;
+use crate::core::build_steps::gcc::set_gccjit_library_path;
 use crate::core::build_steps::llvm;
 use crate::core::build_steps::synthetic_targets::MirOptPanicAbortSyntheticTarget;
 use crate::core::build_steps::tool::{self, SourceType, Tool};
@@ -3512,6 +3513,8 @@ impl Step for CodegenGCC {
 
         let mut cargo = build_cargo();
 
+        set_gccjit_library_path(&builder.config, &mut cargo);
+
         cargo
             .arg("--")
             .arg("test")

src/bootstrap/src/core/config/config.rs

@@ -250,6 +250,8 @@ pub struct Config {
     pub llvm_ldflags: Option<String>,
     pub llvm_use_libcxx: bool,
 
+    pub gcc_download_gccjit: bool,
+
     // rust codegen options
     pub rust_optimize: RustOptimize,
     pub rust_codegen_units: Option<u32>,
@@ -617,6 +619,7 @@ pub(crate) struct TomlConfig {
     build: Option<Build>,
     install: Option<Install>,
     llvm: Option<Llvm>,
+    gcc: Option<Gcc>,
     rust: Option<Rust>,
     target: Option<HashMap<String, TomlTarget>>,
     dist: Option<Dist>,
@@ -651,7 +654,7 @@ trait Merge {
 impl Merge for TomlConfig {
     fn merge(
         &mut self,
-        TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id }: Self,
+        TomlConfig { build, install, llvm, rust, dist, target, profile: _, change_id, gcc }: Self,
         replace: ReplaceOpt,
     ) {
         fn do_merge<T: Merge>(x: &mut Option<T>, y: Option<T>, replace: ReplaceOpt) {
@@ -669,6 +672,7 @@ impl Merge for TomlConfig {
         do_merge(&mut self.llvm, llvm, replace);
         do_merge(&mut self.rust, rust, replace);
         do_merge(&mut self.dist, dist, replace);
+        do_merge(&mut self.gcc, gcc, replace);
         assert!(target.is_none(), "merging target-specific config is not currently supported");
     }
 }
@@ -923,6 +927,13 @@ define_config! {
     }
 }
 
+define_config! {
+    /// TOML representation of how the GCC backend is configured.
+    struct Gcc {
+        download_gccjit: Option<bool> = "download-gccjit",
+    }
+}
+
 #[derive(Clone, Debug, Deserialize)]
 #[serde(untagged)]
 pub enum StringOrBool {
@@ -1749,6 +1760,14 @@ impl Config {
         config.omit_git_hash = omit_git_hash.unwrap_or(default);
         config.rust_info = GitInfo::new(config.omit_git_hash, &config.src);
 
+        if let Some(gcc) = toml.gcc {
+            let Gcc { download_gccjit } = gcc;
+            config.gcc_download_gccjit = download_gccjit.unwrap_or(false);
+        }
+        if config.gcc_enabled(config.build) {
+            config.maybe_download_gccjit();
+        }
+
         if let Some(llvm) = toml.llvm {
             let Llvm {
                 optimize: optimize_toml,
@@ -2315,6 +2334,18 @@ impl Config {
         self.codegen_backends(target).contains(&"llvm".to_owned())
     }
 
+    pub fn gcc_enabled(&self, target: TargetSelection) -> bool {
+        self.codegen_backends(target).contains(&"gcc".to_owned())
+    }
+
+    pub fn libgccjit_folder(&self, gcc_sha: &str) -> PathBuf {
+        assert!(self.gcc_download_gccjit);
+        let cache_prefix = format!("libgccjit-{gcc_sha}");
+        let cache_dst =
+            self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache"));
+        cache_dst.join(cache_prefix)
+    }
+
     pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind {
         self.target_config
             .get(&target)

src/bootstrap/src/core/download.rs

@@ -13,7 +13,10 @@ use xz2::bufread::XzDecoder;
 
 use crate::core::config::RustfmtMetadata;
 use crate::utils::helpers::{check_run, exe, program_out_of_date};
-use crate::{core::build_steps::llvm::detect_llvm_sha, utils::helpers::hex_encode};
+use crate::{
+    core::build_steps::gcc::detect_gccjit_sha, core::build_steps::llvm::detect_llvm_sha,
+    utils::helpers::hex_encode,
+};
 use crate::{t, Config};
 
 static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock<bool> = OnceLock::new();
@@ -229,7 +232,7 @@ impl Config {
             tempfile.to_str().unwrap(),
             "--retry",
             "3",
-            "-SRf",
+            "-SRfL",
         ]);
         // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful.
         if CiEnv::is_ci() {
@@ -742,6 +745,38 @@ download-rustc = false
         let llvm_root = self.ci_llvm_root();
         self.unpack(&tarball, &llvm_root, "rust-dev");
     }
+
+    pub(crate) fn maybe_download_gccjit(&self) {
+        if !self.gcc_download_gccjit {
+            return;
+        }
+        self.download_gccjit(&detect_gccjit_sha());
+    }
+
+    fn download_gccjit(&self, gcc_sha: &str) {
+        let help_on_error = "ERROR: failed to download libgccjit
+
+    HELP: There could be two reasons behind this:
+        1) The host triple is not supported for `download-gccjit`.
+        2) Old builds get deleted after a certain time.
+    ";
+        if !self.build.triple.contains("linux") || !self.build.triple.contains("x86_64") {
+            eprintln!("{help_on_error}");
+            return;
+        }
+        let rustc_cache = self.libgccjit_folder(gcc_sha);
+        if !rustc_cache.exists() {
+            t!(fs::create_dir_all(&rustc_cache));
+        }
+
+        let lib_path = rustc_cache.join("libgccjit.so.0");
+        if !lib_path.exists() {
+            let url = format!(
+                "https://github.com/rust-lang/gcc/releases/download/master-{gcc_sha}/libgccjit.so",
+            );
+            self.download_file(&url, &lib_path, help_on_error);
+        }
+    }
 }
 
 fn path_is_dylib(path: &Path) -> bool {