Add support for new pass manager

The new pass manager can be enabled using
-Z new-llvm-pass-manager=on.

Author: nikic

src/librustc_codegen_llvm/back/lto.rs

@@ -584,6 +584,20 @@ pub(crate) fn run_pass_manager(
     //      tools/lto/LTOCodeGenerator.cpp
     debug!("running the pass manager");
     unsafe {
+        if write::should_use_new_llvm_pass_manager(config) {
+            let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
+            let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
+            // See comment below for why this is necessary.
+            let opt_level = if let config::OptLevel::No = opt_level {
+                config::OptLevel::Less
+            } else {
+                opt_level
+            };
+            write::optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage);
+            debug!("lto done");
+            return;
+        }
+
         let pm = llvm::LLVMCreatePassManager();
         llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm);
 

src/librustc_codegen_llvm/back/write.rs

@@ -111,6 +111,18 @@ pub fn to_llvm_opt_settings(
     }
 }
 
+fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel {
+    use config::OptLevel::*;
+    match cfg {
+        No => llvm::PassBuilderOptLevel::O0,
+        Less => llvm::PassBuilderOptLevel::O1,
+        Default => llvm::PassBuilderOptLevel::O2,
+        Aggressive => llvm::PassBuilderOptLevel::O3,
+        Size => llvm::PassBuilderOptLevel::Os,
+        SizeMin => llvm::PassBuilderOptLevel::Oz,
+    }
+}
+
 // If find_features is true this won't access `sess.crate_types` by assuming
 // that `is_pie_binary` is false. When we discover LLVM target features
 // `sess.crate_types` is uninitialized so we cannot access it.
@@ -303,6 +315,88 @@ unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void
     }
 }
 
+fn get_pgo_gen_path(config: &ModuleConfig) -> Option<CString> {
+    match config.pgo_gen {
+        SwitchWithOptPath::Enabled(ref opt_dir_path) => {
+            let path = if let Some(dir_path) = opt_dir_path {
+                dir_path.join("default_%m.profraw")
+            } else {
+                PathBuf::from("default_%m.profraw")
+            };
+
+            Some(CString::new(format!("{}", path.display())).unwrap())
+        }
+        SwitchWithOptPath::Disabled => None,
+    }
+}
+
+fn get_pgo_use_path(config: &ModuleConfig) -> Option<CString> {
+    config
+        .pgo_use
+        .as_ref()
+        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap())
+}
+
+pub(crate) fn should_use_new_llvm_pass_manager(config: &ModuleConfig) -> bool {
+    // We only support the new pass manager starting with LLVM 9.
+    if llvm_util::get_major_version() < 9 {
+        return false;
+    }
+
+    // The new pass manager is disabled by default.
+    config.new_llvm_pass_manager.unwrap_or(false)
+}
+
+pub(crate) unsafe fn optimize_with_new_llvm_pass_manager(
+    module: &ModuleCodegen<ModuleLlvm>,
+    config: &ModuleConfig,
+    opt_level: config::OptLevel,
+    opt_stage: llvm::OptStage,
+) {
+    let unroll_loops =
+        opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin;
+    let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed();
+    let pgo_gen_path = get_pgo_gen_path(config);
+    let pgo_use_path = get_pgo_use_path(config);
+    let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO;
+    // Sanitizer instrumentation is only inserted during the pre-link optimization stage.
+    let sanitizer_options = if !is_lto {
+        config.sanitizer.as_ref().map(|s| llvm::SanitizerOptions {
+            sanitize_memory: *s == Sanitizer::Memory,
+            sanitize_thread: *s == Sanitizer::Thread,
+            sanitize_address: *s == Sanitizer::Address,
+            sanitize_recover: config.sanitizer_recover.contains(s),
+            sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int,
+        })
+    } else {
+        None
+    };
+
+    // FIXME: NewPM doesn't provide a facility to pass custom InlineParams.
+    // We would have to add upstream support for this first, before we can support
+    // config.inline_threshold and our more aggressive default thresholds.
+    // FIXME: NewPM uses an different and more explicit way to textually represent
+    // pass pipelines. It would probably make sense to expose this, but it would
+    // require a different format than the current -C passes.
+    llvm::LLVMRustOptimizeWithNewPassManager(
+        module.module_llvm.llmod(),
+        &*module.module_llvm.tm,
+        to_pass_builder_opt_level(opt_level),
+        opt_stage,
+        config.no_prepopulate_passes,
+        config.verify_llvm_ir,
+        using_thin_buffers,
+        config.merge_functions,
+        unroll_loops,
+        config.vectorize_slp,
+        config.vectorize_loop,
+        config.no_builtins,
+        sanitizer_options.as_ref(),
+        pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
+        pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()),
+    );
+}
+
 // Unsafe due to LLVM calls.
 pub(crate) unsafe fn optimize(
     cgcx: &CodegenContext<LlvmCodegenBackend>,
@@ -327,6 +421,17 @@ pub(crate) unsafe fn optimize(
     }
 
     if let Some(opt_level) = config.opt_level {
+        if should_use_new_llvm_pass_manager(config) {
+            let opt_stage = match cgcx.lto {
+                Lto::Fat => llvm::OptStage::PreLinkFatLTO,
+                Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO,
+                _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO,
+                _ => llvm::OptStage::PreLinkNoLTO,
+            };
+            optimize_with_new_llvm_pass_manager(module, config, opt_level, opt_stage);
+            return Ok(());
+        }
+
         // Create the two optimizing pass managers. These mirror what clang
         // does, and are by populated by LLVM's default PassManagerBuilder.
         // Each manager has a different set of passes, but they also share
@@ -757,24 +862,8 @@ pub unsafe fn with_llvm_pmb(
     let opt_size =
         config.opt_size.map(|x| to_llvm_opt_settings(x).1).unwrap_or(llvm::CodeGenOptSizeNone);
     let inline_threshold = config.inline_threshold;
-
-    let pgo_gen_path = match config.pgo_gen {
-        SwitchWithOptPath::Enabled(ref opt_dir_path) => {
-            let path = if let Some(dir_path) = opt_dir_path {
-                dir_path.join("default_%m.profraw")
-            } else {
-                PathBuf::from("default_%m.profraw")
-            };
-
-            Some(CString::new(format!("{}", path.display())).unwrap())
-        }
-        SwitchWithOptPath::Disabled => None,
-    };
-
-    let pgo_use_path = config
-        .pgo_use
-        .as_ref()
-        .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap());
+    let pgo_gen_path = get_pgo_gen_path(config);
+    let pgo_use_path = get_pgo_use_path(config);
 
     llvm::LLVMRustConfigurePassManagerBuilder(
         builder,

src/librustc_codegen_llvm/llvm/ffi.rs

@@ -402,6 +402,38 @@ pub enum CodeGenOptLevel {
     Aggressive,
 }
 
+/// LLVMRustPassBuilderOptLevel
+#[repr(C)]
+pub enum PassBuilderOptLevel {
+    O0,
+    O1,
+    O2,
+    O3,
+    Os,
+    Oz,
+}
+
+/// LLVMRustOptStage
+#[derive(PartialEq)]
+#[repr(C)]
+pub enum OptStage {
+    PreLinkNoLTO,
+    PreLinkThinLTO,
+    PreLinkFatLTO,
+    ThinLTO,
+    FatLTO,
+}
+
+/// LLVMRustSanitizerOptions
+#[repr(C)]
+pub struct SanitizerOptions {
+    pub sanitize_memory: bool,
+    pub sanitize_thread: bool,
+    pub sanitize_address: bool,
+    pub sanitize_recover: bool,
+    pub sanitize_memory_track_origins: c_int,
+}
+
 /// LLVMRelocMode
 #[derive(Copy, Clone, PartialEq)]
 #[repr(C)]
@@ -1896,6 +1928,23 @@ extern "C" {
         Output: *const c_char,
         FileType: FileType,
     ) -> LLVMRustResult;
+    pub fn LLVMRustOptimizeWithNewPassManager(
+        M: &'a Module,
+        TM: &'a TargetMachine,
+        OptLevel: PassBuilderOptLevel,
+        OptStage: OptStage,
+        NoPrepopulatePasses: bool,
+        VerifyIR: bool,
+        UseThinLTOBuffers: bool,
+        MergeFunctions: bool,
+        UnrollLoops: bool,
+        SLPVectorize: bool,
+        LoopVectorize: bool,
+        DisableSimplifyLibCalls: bool,
+        SanitizerOptions: Option<&SanitizerOptions>,
+        PGOGenPath: *const c_char,
+        PGOUsePath: *const c_char,
+    );
     pub fn LLVMRustPrintModule(
         M: &'a Module,
         Output: *const c_char,

src/librustc_codegen_ssa/back/write.rs

@@ -88,6 +88,7 @@ pub struct ModuleConfig {
     pub vectorize_slp: bool,
     pub merge_functions: bool,
     pub inline_threshold: Option<usize>,
+    pub new_llvm_pass_manager: Option<bool>,
     // Instead of creating an object file by doing LLVM codegen, just
     // make the object file bitcode. Provides easy compatibility with
     // emscripten's ecc compiler, when used as the linker.
@@ -132,6 +133,7 @@ impl ModuleConfig {
             vectorize_slp: false,
             merge_functions: false,
             inline_threshold: None,
+            new_llvm_pass_manager: None,
         }
     }
 
@@ -140,6 +142,7 @@ impl ModuleConfig {
         self.no_prepopulate_passes = sess.opts.cg.no_prepopulate_passes;
         self.no_builtins = no_builtins || sess.target.target.options.no_builtins;
         self.inline_threshold = sess.opts.cg.inline_threshold;
+        self.new_llvm_pass_manager = sess.opts.debugging_opts.new_llvm_pass_manager;
         self.obj_is_bitcode =
             sess.target.target.options.obj_is_bitcode || sess.opts.cg.linker_plugin_lto.enabled();
         let embed_bitcode =

src/librustc_session/options.rs

@@ -968,4 +968,6 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options,
         "compile without linking"),
     link_only: bool = (false, parse_bool, [TRACKED],
         "link the `.rlink` file generated by `-Z no-link`"),
+    new_llvm_pass_manager: Option<bool> = (None, parse_opt_bool, [TRACKED],
+        "use new LLVM pass manager"),
 }

src/rustllvm/PassWrapper.cpp

@@ -12,6 +12,11 @@
 #include "llvm/IR/AutoUpgrade.h"
 #include "llvm/IR/AssemblyAnnotationWriter.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Passes/PassBuilder.h"
+#if LLVM_VERSION_GE(9, 0)
+#include "llvm/Passes/StandardInstrumentations.h"
+#endif
 #include "llvm/Support/CBindingWrapping.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Host.h"
@@ -32,9 +37,12 @@
 #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h"
 #include "llvm/Transforms/Instrumentation/MemorySanitizer.h"
 #endif
+#if LLVM_VERSION_GE(9, 0)
+#include "llvm/Transforms/Utils/CanonicalizeAliases.h"
+#endif
+#include "llvm/Transforms/Utils/NameAnonGlobals.h"
 
 using namespace llvm;
-using namespace llvm::legacy;
 
 typedef struct LLVMOpaquePass *LLVMPassRef;
 typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
@@ -314,6 +322,34 @@ static CodeGenOpt::Level fromRust(LLVMRustCodeGenOptLevel Level) {
   }
 }
 
+enum class LLVMRustPassBuilderOptLevel {
+  O0,
+  O1,
+  O2,
+  O3,
+  Os,
+  Oz,
+};
+
+static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
+  switch (Level) {
+  case LLVMRustPassBuilderOptLevel::O0:
+    return PassBuilder::O0;
+  case LLVMRustPassBuilderOptLevel::O1:
+    return PassBuilder::O1;
+  case LLVMRustPassBuilderOptLevel::O2:
+    return PassBuilder::O2;
+  case LLVMRustPassBuilderOptLevel::O3:
+    return PassBuilder::O3;
+  case LLVMRustPassBuilderOptLevel::Os:
+    return PassBuilder::Os;
+  case LLVMRustPassBuilderOptLevel::Oz:
+    return PassBuilder::Oz;
+  default:
+    report_fatal_error("Bad PassBuilderOptLevel.");
+  }
+}
+
 enum class LLVMRustRelocMode {
   Default,
   Static,
@@ -604,6 +640,212 @@ LLVMRustWriteOutputFile(LLVMTargetMachineRef Target, LLVMPassManagerRef PMR,
   return LLVMRustResult::Success;
 }
 
+enum class LLVMRustOptStage {
+  PreLinkNoLTO,
+  PreLinkThinLTO,
+  PreLinkFatLTO,
+  ThinLTO,
+  FatLTO,
+};
+
+struct LLVMRustSanitizerOptions {
+  bool SanitizeMemory;
+  bool SanitizeThread;
+  bool SanitizeAddress;
+  bool SanitizeRecover;
+  int SanitizeMemoryTrackOrigins;
+};
+
+extern "C" void
+LLVMRustOptimizeWithNewPassManager(
+    LLVMModuleRef ModuleRef,
+    LLVMTargetMachineRef TMRef,
+    LLVMRustPassBuilderOptLevel OptLevelRust,
+    LLVMRustOptStage OptStage,
+    bool NoPrepopulatePasses, bool VerifyIR, bool UseThinLTOBuffers,
+    bool MergeFunctions, bool UnrollLoops, bool SLPVectorize, bool LoopVectorize,
+    bool DisableSimplifyLibCalls,
+    LLVMRustSanitizerOptions *SanitizerOptions,
+    const char *PGOGenPath, const char *PGOUsePath) {
+#if LLVM_VERSION_GE(9, 0)
+  Module *TheModule = unwrap(ModuleRef);
+  TargetMachine *TM = unwrap(TMRef);
+  PassBuilder::OptimizationLevel OptLevel = fromRust(OptLevelRust);
+
+  // FIXME: MergeFunctions is not supported by NewPM yet.
+  (void) MergeFunctions;
+
+  PipelineTuningOptions PTO;
+  PTO.LoopUnrolling = UnrollLoops;
+  PTO.LoopInterleaving = UnrollLoops;
+  PTO.LoopVectorization = LoopVectorize;
+  PTO.SLPVectorization = SLPVectorize;
+
+  PassInstrumentationCallbacks PIC;
+  StandardInstrumentations SI;
+  SI.registerCallbacks(PIC);
+
+  Optional<PGOOptions> PGOOpt;
+  if (PGOGenPath) {
+    assert(!PGOUsePath);
+    PGOOpt = PGOOptions(PGOGenPath, "", "", PGOOptions::IRInstr);
+  } else if (PGOUsePath) {
+    assert(!PGOGenPath);
+    PGOOpt = PGOOptions(PGOUsePath, "", "", PGOOptions::IRUse);
+  }
+
+  PassBuilder PB(TM, PTO, PGOOpt, &PIC);
+
+  // FIXME: We may want to expose this as an option.
+  bool DebugPassManager = false;
+  LoopAnalysisManager LAM(DebugPassManager);
+  FunctionAnalysisManager FAM(DebugPassManager);
+  CGSCCAnalysisManager CGAM(DebugPassManager);
+  ModuleAnalysisManager MAM(DebugPassManager);
+
+  FAM.registerPass([&] { return PB.buildDefaultAAPipeline(); });
+
+  Triple TargetTriple(TheModule->getTargetTriple());
+  std::unique_ptr<TargetLibraryInfoImpl> TLII(new TargetLibraryInfoImpl(TargetTriple));
+  if (DisableSimplifyLibCalls)
+    TLII->disableAllFunctions();
+  FAM.registerPass([&] { return TargetLibraryAnalysis(*TLII); });
+
+  PB.registerModuleAnalyses(MAM);
+  PB.registerCGSCCAnalyses(CGAM);
+  PB.registerFunctionAnalyses(FAM);
+  PB.registerLoopAnalyses(LAM);
+  PB.crossRegisterProxies(LAM, FAM, CGAM, MAM);
+
+  // We manually collect pipeline callbacks so we can apply them at O0, where the
+  // PassBuilder does not create a pipeline.
+  std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks;
+  std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
+      OptimizerLastEPCallbacks;
+
+  if (VerifyIR) {
+    PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) {
+        MPM.addPass(VerifierPass());
+    });
+  }
+
+  if (SanitizerOptions) {
+    if (SanitizerOptions->SanitizeMemory) {
+      MemorySanitizerOptions Options(
+          SanitizerOptions->SanitizeMemoryTrackOrigins,
+          SanitizerOptions->SanitizeRecover,
+          /*CompileKernel=*/false);
+#if LLVM_VERSION_GE(10, 0)
+      PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) {
+        MPM.addPass(MemorySanitizerPass(Options));
+      });
+#endif
+      OptimizerLastEPCallbacks.push_back(
+        [Options](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
+          FPM.addPass(MemorySanitizerPass(Options));
+        }
+      );
+    }
+
+    if (SanitizerOptions->SanitizeThread) {
+#if LLVM_VERSION_GE(10, 0)
+      PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) {
+        MPM.addPass(ThreadSanitizerPass());
+      });
+#endif
+      OptimizerLastEPCallbacks.push_back(
+        [](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
+          FPM.addPass(ThreadSanitizerPass());
+        }
+      );
+    }
+
+    if (SanitizerOptions->SanitizeAddress) {
+      // FIXME: Rust does not expose the UseAfterScope option.
+      PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) {
+        MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
+      });
+      OptimizerLastEPCallbacks.push_back(
+        [SanitizerOptions](FunctionPassManager &FPM, PassBuilder::OptimizationLevel Level) {
+          FPM.addPass(AddressSanitizerPass(
+              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover));
+        }
+      );
+      PipelineStartEPCallbacks.push_back(
+        [SanitizerOptions](ModulePassManager &MPM) {
+          MPM.addPass(ModuleAddressSanitizerPass(
+              /*CompileKernel=*/false, SanitizerOptions->SanitizeRecover));
+        }
+      );
+    }
+  }
+
+  ModulePassManager MPM(DebugPassManager);
+  if (!NoPrepopulatePasses) {
+    if (OptLevel == PassBuilder::O0) {
+      for (const auto &C : PipelineStartEPCallbacks)
+        C(MPM);
+
+      if (!OptimizerLastEPCallbacks.empty()) {
+        FunctionPassManager FPM(DebugPassManager);
+        for (const auto &C : OptimizerLastEPCallbacks)
+          C(FPM, OptLevel);
+        MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+      }
+
+      MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/false));
+
+#if LLVM_VERSION_GE(10, 0)
+      if (PGOOpt) {
+        PB.addPGOInstrPassesForO0(
+            MPM, DebugPassManager, PGOOpt->Action == PGOOptions::IRInstr,
+            /*IsCS=*/false, PGOOpt->ProfileFile, PGOOpt->ProfileRemappingFile);
+      }
+#endif
+    } else {
+      for (const auto &C : PipelineStartEPCallbacks)
+        PB.registerPipelineStartEPCallback(C);
+      for (const auto &C : OptimizerLastEPCallbacks)
+        PB.registerOptimizerLastEPCallback(C);
+
+      switch (OptStage) {
+      case LLVMRustOptStage::PreLinkNoLTO:
+        MPM = PB.buildPerModuleDefaultPipeline(OptLevel, DebugPassManager);
+        break;
+      case LLVMRustOptStage::PreLinkThinLTO:
+        MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
+        break;
+      case LLVMRustOptStage::PreLinkFatLTO:
+        MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
+        break;
+      case LLVMRustOptStage::ThinLTO:
+        // FIXME: Does it make sense to pass the ModuleSummaryIndex?
+        // It only seems to be needed for C++ specific optimizations.
+        MPM = PB.buildThinLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
+        break;
+      case LLVMRustOptStage::FatLTO:
+        MPM = PB.buildLTODefaultPipeline(OptLevel, DebugPassManager, nullptr);
+        break;
+      }
+    }
+  }
+
+  if (UseThinLTOBuffers) {
+    MPM.addPass(CanonicalizeAliasesPass());
+    MPM.addPass(NameAnonGlobalPass());
+  }
+
+  // Upgrade all calls to old intrinsics first.
+  for (Module::iterator I = TheModule->begin(), E = TheModule->end(); I != E;)
+    UpgradeCallsToIntrinsic(&*I++); // must be post-increment, as we remove
+
+  MPM.run(*TheModule, MAM);
+#else
+  // The new pass manager has been available for a long time,
+  // but we don't bother supporting it on old LLVM versions.
+  report_fatal_error("New pass manager only supported since LLVM 9");
+#endif
+}
 
 // Callback to demangle function name
 // Parameters:

src/test/codegen/sanitizer-memory-track-orgins.rs

@@ -15,10 +15,10 @@
 #![crate_type="lib"]
 
 // MSAN-0-NOT: @__msan_track_origins
-// MSAN-1:     @__msan_track_origins = weak_odr local_unnamed_addr constant i32 1
-// MSAN-2:     @__msan_track_origins = weak_odr local_unnamed_addr constant i32 2
-// MSAN-1-LTO: @__msan_track_origins = weak_odr local_unnamed_addr constant i32 1
-// MSAN-2-LTO: @__msan_track_origins = weak_odr local_unnamed_addr constant i32 2
+// MSAN-1:     @__msan_track_origins = weak_odr {{.*}}constant i32 1
+// MSAN-2:     @__msan_track_origins = weak_odr {{.*}}constant i32 2
+// MSAN-1-LTO: @__msan_track_origins = weak_odr {{.*}}constant i32 1
+// MSAN-2-LTO: @__msan_track_origins = weak_odr {{.*}}constant i32 2
 //
 // MSAN-0-LABEL: define void @copy(
 // MSAN-1-LABEL: define void @copy(