Enable MIR inlining of very small functions in debug builds

saethlin · saethlin · commit 2848a107e96a · 2023-10-18T00:07:36.000-04:00
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
@@ -10,7 +10,6 @@ use rustc_middle::mir::visit::*;
 use rustc_middle::mir::*;
 use rustc_middle::ty::TypeVisitableExt;
 use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
-use rustc_session::config::OptLevel;
 use rustc_target::abi::FieldIdx;
 use rustc_target::spec::abi::Abi;
 
@@ -47,12 +46,8 @@ impl<'tcx> MirPass<'tcx> for Inline {
         }
 
         match sess.mir_opt_level() {
-            0 | 1 => false,
-            2 => {
-                (sess.opts.optimize == OptLevel::Default
-                    || sess.opts.optimize == OptLevel::Aggressive)
-                    && sess.opts.incremental == None
-            }
+            0 => false,
+            1 | 2 => sess.opts.incremental.is_none(),
             _ => true,
         }
     }
@@ -70,6 +65,14 @@ impl<'tcx> MirPass<'tcx> for Inline {
 }
 
 fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
+    let has_call = body
+        .basic_blocks
+        .iter()
+        .any(|b| matches!(b.terminator().kind, TerminatorKind::Call { .. }));
+    if !has_call {
+        return false;
+    }
+
     let def_id = body.source.def_id().expect_local();
 
     // Only do inlining into fn bodies.
@@ -475,7 +478,16 @@ impl<'tcx> Inliner<'tcx> {
         if callee_body.basic_blocks.len() <= 3 {
             threshold += threshold / 4;
         }
-        debug!("    final inline threshold = {}", threshold);
+
+        // At MIR opt level 1 we only want to do a tiny bit of inlining.
+        // 2 * INSTR_COST is the cost of one statement plus a return terminator.
+        if tcx.sess.mir_opt_level() == 1 {
+            threshold = 2 * INSTR_COST;
+            if callee_body.basic_blocks.len() > 1 {
+                return Err("cost above threshold");
+            }
+        }
+        debug!("final inline threshold = {}", threshold);
 
         // FIXME: Give a bonus to functions with only a single caller
 
diff --git a/tests/codegen-units/polymorphization/unused_type_parameters.rs b/tests/codegen-units/polymorphization/unused_type_parameters.rs
@@ -1,4 +1,4 @@
-// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1
+// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0
 
 #![crate_type = "rlib"]
 
diff --git a/tests/coverage-map/status-quo/async.cov-map b/tests/coverage-map/status-quo/async.cov-map
@@ -283,22 +283,19 @@ Number of file 0 mappings: 5
 - Code(Zero) at (prev + 2, 1) to (start + 0, 2)
 
 Function name: async::l
-Raw bytes (37): 0x[01, 01, 04, 01, 07, 09, 05, 09, 0f, 05, 02, 05, 01, 53, 01, 01, 0c, 02, 02, 0e, 00, 10, 05, 01, 0e, 00, 10, 09, 01, 0e, 00, 10, 0b, 02, 01, 00, 02]
+Raw bytes (33): 0x[01, 01, 02, 09, 07, 05, 00, 05, 01, 53, 01, 01, 0c, 00, 02, 0e, 00, 10, 00, 01, 0e, 00, 10, 09, 01, 0e, 00, 10, 03, 02, 01, 00, 02]
 Number of files: 1
 - file 0 => global file 1
-Number of expressions: 4
-- expression 0 operands: lhs = Counter(0), rhs = Expression(1, Add)
-- expression 1 operands: lhs = Counter(2), rhs = Counter(1)
-- expression 2 operands: lhs = Counter(2), rhs = Expression(3, Add)
-- expression 3 operands: lhs = Counter(1), rhs = Expression(0, Sub)
+Number of expressions: 2
+- expression 0 operands: lhs = Counter(2), rhs = Expression(1, Add)
+- expression 1 operands: lhs = Counter(1), rhs = Zero
 Number of file 0 mappings: 5
 - Code(Counter(0)) at (prev + 83, 1) to (start + 1, 12)
-- Code(Expression(0, Sub)) at (prev + 2, 14) to (start + 0, 16)
-    = (c0 - (c2 + c1))
-- Code(Counter(1)) at (prev + 1, 14) to (start + 0, 16)
+- Code(Zero) at (prev + 2, 14) to (start + 0, 16)
+- Code(Zero) at (prev + 1, 14) to (start + 0, 16)
 - Code(Counter(2)) at (prev + 1, 14) to (start + 0, 16)
-- Code(Expression(2, Add)) at (prev + 2, 1) to (start + 0, 2)
-    = (c2 + (c1 + (c0 - (c2 + c1))))
+- Code(Expression(0, Add)) at (prev + 2, 1) to (start + 0, 2)
+    = (c2 + (c1 + Zero))
 
 Function name: async::m
 Raw bytes (9): 0x[01, 01, 00, 01, 01, 5b, 01, 00, 19]
diff --git a/tests/debuginfo/function-names.rs b/tests/debuginfo/function-names.rs
@@ -1,7 +1,7 @@
 // Function names are formatted differently in old versions of GDB
 // min-gdb-version: 10.1
 
-// compile-flags:-g
+// compile-flags: -g -Zmir-opt-level=0
 
 // === GDB TESTS ===================================================================================
 
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.rs b/tests/mir-opt/inline/trivial_fn_debug_build.rs
@@ -0,0 +1,22 @@
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+// compile-flags: -Copt-level=0 -Zmir-opt-level=1
+#![crate_type = "lib"]
+
+// Test that we still inline trivial functions even in a debug build
+
+pub struct Thing {
+    inner: u8,
+}
+
+impl Thing {
+    #[inline]
+    pub fn get(&self) -> u8 {
+        self.inner
+    }
+}
+
+// EMIT_MIR trivial_fn_debug_build.wrapper.Inline.diff
+// EMIT_MIR trivial_fn_debug_build.wrapper.PreCodegen.after.mir
+pub fn wrapper(t: &Thing) -> u8 {
+    t.get()
+}
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-abort.diff
@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind unreachable];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.Inline.panic-unwind.diff
@@ -0,0 +1,24 @@
+- // MIR for `wrapper` before Inline
++ // MIR for `wrapper` after Inline
+  
+  fn wrapper(_1: &Thing) -> u8 {
+      debug t => _1;
+      let mut _0: u8;
+      let mut _2: &Thing;
++     scope 1 (inlined Thing::get) {
++         debug self => _2;
++     }
+  
+      bb0: {
+          StorageLive(_2);
+          _2 = &(*_1);
+-         _0 = Thing::get(move _2) -> [return: bb1, unwind continue];
+-     }
+- 
+-     bb1: {
++         _0 = ((*_2).0: u8);
+          StorageDead(_2);
+          return;
+      }
+  }
+  
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-abort.mir
@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}
diff --git a/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/inline/trivial_fn_debug_build.wrapper.PreCodegen.after.panic-unwind.mir
@@ -0,0 +1,14 @@
+// MIR for `wrapper` after PreCodegen
+
+fn wrapper(_1: &Thing) -> u8 {
+    debug t => _1;
+    let mut _0: u8;
+    scope 1 (inlined Thing::get) {
+        debug self => _1;
+    }
+
+    bb0: {
+        _0 = ((*_1).0: u8);
+        return;
+    }
+}
diff --git a/tests/run-make/sepcomp-cci-copies/Makefile b/tests/run-make/sepcomp-cci-copies/Makefile
@@ -8,5 +8,5 @@ include ../tools.mk
 all:
 	$(RUSTC) cci_lib.rs
 	$(RUSTC) foo.rs --emit=llvm-ir -C codegen-units=6 \
-		-Z inline-in-all-cgus
+		-Z inline-in-all-cgus -Zmir-opt-level=0
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ .*cci_fn)" -eq "2" ]
diff --git a/tests/run-make/sepcomp-inlining/Makefile b/tests/run-make/sepcomp-inlining/Makefile
@@ -8,7 +8,7 @@ include ../tools.mk
 
 all:
 	$(RUSTC) foo.rs --emit=llvm-ir -C codegen-units=3 \
-		-Z inline-in-all-cgus
+		-Z inline-in-all-cgus -Zmir-opt-level=0
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ i32\ .*inlined)" -eq "0" ]
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ internal\ i32\ .*inlined)" -eq "2" ]
 	[ "$$(cat "$(TMPDIR)"/foo.*.ll | grep -c define\ hidden\ i32\ .*normal)" -eq "1" ]
diff --git a/tests/run-make/symbol-visibility/Makefile b/tests/run-make/symbol-visibility/Makefile
@@ -32,12 +32,12 @@ endif
 RE_ANY_RUST_SYMBOL="_ZN.*h.*E\|_R[a-zA-Z0-9_]+"
 
 all:
-	$(RUSTC) -Zshare-generics=no an_rlib.rs
-	$(RUSTC) -Zshare-generics=no a_cdylib.rs
-	$(RUSTC) -Zshare-generics=no a_rust_dylib.rs
-	$(RUSTC) -Zshare-generics=no a_proc_macro.rs
-	$(RUSTC) -Zshare-generics=no an_executable.rs
-	$(RUSTC) -Zshare-generics=no a_cdylib.rs --crate-name combined_rlib_dylib --crate-type=rlib,cdylib
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 an_rlib.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_cdylib.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_rust_dylib.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_proc_macro.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 an_executable.rs
+	$(RUSTC) -Zshare-generics=no -Zmir-opt-level=0 a_cdylib.rs --crate-name combined_rlib_dylib --crate-type=rlib,cdylib
 
 	# Check that a cdylib exports its public #[no_mangle] functions
 	[ "$$($(NM) $(TMPDIR)/$(CDYLIB_NAME) | grep -v __imp_ | grep -c public_c_function_from_cdylib)" -eq "1" ]
@@ -85,11 +85,11 @@ endif
 	[ "$$($(NM) $(TMPDIR)/$(COMBINED_CDYLIB_NAME) | grep -v __imp_ | grep -c $(RE_ANY_RUST_SYMBOL))" -eq "0" ]
 
 
-	$(RUSTC) -Zshare-generics=yes an_rlib.rs
-	$(RUSTC) -Zshare-generics=yes a_cdylib.rs
-	$(RUSTC) -Zshare-generics=yes a_rust_dylib.rs
-	$(RUSTC) -Zshare-generics=yes a_proc_macro.rs
-	$(RUSTC) -Zshare-generics=yes an_executable.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 an_rlib.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_cdylib.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_rust_dylib.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 a_proc_macro.rs
+	$(RUSTC) -Zshare-generics=yes -Zmir-opt-level=0 an_executable.rs
 
 	# Check that a cdylib exports its public #[no_mangle] functions
 	[ "$$($(NM) $(TMPDIR)/$(CDYLIB_NAME) | grep -v __imp_ | grep -c public_c_function_from_cdylib)" -eq "1" ]
diff --git a/tests/ui-fulldeps/stable-mir/crate-info.rs b/tests/ui-fulldeps/stable-mir/crate-info.rs
@@ -59,7 +59,7 @@ fn test_stable_mir(_tcx: TyCtxt<'_>) -> ControlFlow<()> {
     let foo_bar = get_item(&items, (DefKind::Fn, "foo_bar")).unwrap();
     let body = foo_bar.body();
     assert_eq!(body.locals.len(), 5);
-    assert_eq!(body.blocks.len(), 4);
+    assert_eq!(body.blocks.len(), 3);
     let block = &body.blocks[0];
     match &block.terminator.kind {
         stable_mir::mir::TerminatorKind::Call { .. } => {}
diff --git a/tests/ui/consts/const-eval/index-out-of-bounds-never-type.rs b/tests/ui/consts/const-eval/index-out-of-bounds-never-type.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 // Regression test for #66975
 #![warn(unconditional_panic)]
diff --git a/tests/ui/consts/const-eval/index-out-of-bounds-never-type.stderr b/tests/ui/consts/const-eval/index-out-of-bounds-never-type.stderr
@@ -1,11 +1,11 @@
 error[E0080]: evaluation of `PrintName::<()>::VOID` failed
-  --> $DIR/index-out-of-bounds-never-type.rs:10:61
+  --> $DIR/index-out-of-bounds-never-type.rs:11:61
    |
 LL |     const VOID: ! = { let x = 0 * std::mem::size_of::<T>(); [][x] };
    |                                                             ^^^^^ index out of bounds: the length is 0 but the index is 0
 
 note: the above error was encountered while instantiating `fn f::<()>`
-  --> $DIR/index-out-of-bounds-never-type.rs:20:5
+  --> $DIR/index-out-of-bounds-never-type.rs:21:5
    |
 LL |     f::<()>();
    |     ^^^^^^^^^
diff --git a/tests/ui/consts/const-eval/issue-50814-2.rs b/tests/ui/consts/const-eval/issue-50814-2.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 trait C {
     const BOO: usize;
diff --git a/tests/ui/consts/const-eval/issue-50814-2.stderr b/tests/ui/consts/const-eval/issue-50814-2.stderr
@@ -1,17 +1,17 @@
 error[E0080]: evaluation of `<A<()> as Foo<()>>::BAR` failed
-  --> $DIR/issue-50814-2.rs:14:24
+  --> $DIR/issue-50814-2.rs:15:24
    |
 LL |     const BAR: usize = [5, 6, 7][T::BOO];
    |                        ^^^^^^^^^^^^^^^^^ index out of bounds: the length is 3 but the index is 42
 
 note: erroneous constant encountered
-  --> $DIR/issue-50814-2.rs:18:6
+  --> $DIR/issue-50814-2.rs:19:6
    |
 LL |     &<A<T> as Foo<T>>::BAR
    |      ^^^^^^^^^^^^^^^^^^^^^
 
 note: the above error was encountered while instantiating `fn foo::<()>`
-  --> $DIR/issue-50814-2.rs:30:22
+  --> $DIR/issue-50814-2.rs:31:22
    |
 LL |     println!("{:x}", foo::<()>() as *const usize as usize);
    |                      ^^^^^^^^^^^
diff --git a/tests/ui/consts/const-eval/issue-85155.rs b/tests/ui/consts/const-eval/issue-85155.rs
@@ -11,6 +11,7 @@
 // aux-build: post_monomorphization_error.rs
 // build-fail: this is a post-monomorphization error, it passes check runs and requires building
 //             to actually fail.
+// compile-flags: -Zmir-opt-level=0
 
 extern crate post_monomorphization_error;
 
diff --git a/tests/ui/consts/const-eval/issue-85155.stderr b/tests/ui/consts/const-eval/issue-85155.stderr
@@ -5,7 +5,7 @@ LL |         let _ = 1 / ((IMM >= MIN && IMM <= MAX) as usize);
    |                 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ attempt to divide `1_usize` by zero
 
 note: the above error was encountered while instantiating `fn post_monomorphization_error::stdarch_intrinsic::<2>`
-  --> $DIR/issue-85155.rs:19:5
+  --> $DIR/issue-85155.rs:20:5
    |
 LL |     post_monomorphization_error::stdarch_intrinsic::<2>();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/tests/ui/generics/post_monomorphization_error_backtrace.rs b/tests/ui/generics/post_monomorphization_error_backtrace.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 
 fn assert_zst<T>() {
     struct F<T>(T);
diff --git a/tests/ui/generics/post_monomorphization_error_backtrace.stderr b/tests/ui/generics/post_monomorphization_error_backtrace.stderr
@@ -1,27 +1,27 @@
 error[E0080]: evaluation of `assert_zst::F::<u32>::V` failed
-  --> $DIR/post_monomorphization_error_backtrace.rs:6:23
+  --> $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
 LL |         const V: () = assert!(std::mem::size_of::<T>() == 0);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn assert_zst::<u32>`
-  --> $DIR/post_monomorphization_error_backtrace.rs:18:5
+  --> $DIR/post_monomorphization_error_backtrace.rs:19:5
    |
 LL |     assert_zst::<U>()
    |     ^^^^^^^^^^^^^^^^^
 
 error[E0080]: evaluation of `assert_zst::F::<i32>::V` failed
-  --> $DIR/post_monomorphization_error_backtrace.rs:6:23
+  --> $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
 LL |         const V: () = assert!(std::mem::size_of::<T>() == 0);
-   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:6:23
+   |                       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ the evaluated program panicked at 'assertion failed: std::mem::size_of::<T>() == 0', $DIR/post_monomorphization_error_backtrace.rs:7:23
    |
    = note: this error originates in the macro `assert` (in Nightly builds, run with -Z macro-backtrace for more info)
 
 note: the above error was encountered while instantiating `fn assert_zst::<i32>`
-  --> $DIR/post_monomorphization_error_backtrace.rs:18:5
+  --> $DIR/post_monomorphization_error_backtrace.rs:19:5
    |
 LL |     assert_zst::<U>()
    |     ^^^^^^^^^^^^^^^^^
diff --git a/tests/ui/inline-const/const-expr-generic-err.rs b/tests/ui/inline-const/const-expr-generic-err.rs
@@ -1,4 +1,5 @@
 // build-fail
+// compile-flags: -Zmir-opt-level=0
 #![feature(inline_const)]
 
 fn foo<T>() {
diff --git a/tests/ui/inline-const/const-expr-generic-err.stderr b/tests/ui/inline-const/const-expr-generic-err.stderr
diff --git a/tests/ui/polymorphization/const_parameters/functions.rs b/tests/ui/polymorphization/const_parameters/functions.rs
diff --git a/tests/ui/polymorphization/type_parameters/functions.rs b/tests/ui/polymorphization/type_parameters/functions.rs
diff --git a/tests/ui/type_length_limit.rs b/tests/ui/type_length_limit.rs

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1`
	`1`	`+// compile-flags:-Zpolymorphize=on -Zprint-mono-items=lazy -Copt-level=1 -Zmir-opt-level=0`
`2`	`2`
`3`	`3`	`#![crate_type = "rlib"]`
`4`	`4`