Skip to content

Commit 226202d

Browse files
borsbors
committed
Auto merge of #105119 - JakobDegen:inline-experiments, r=cjgillot
Disable top down MIR inlining The current MIR inliner has exponential behavior in some cases: <https://godbolt.org/z/7jnWah4fE>. The cause of this is top-down inlining, where we repeatedly do inlining like `call_a() => { call_b(); call_b(); }`. Each decision on its own seems to make sense, but the result is exponential. Disabling top-down inlining fundamentally prevents this. Each call site in the original, unoptimized source code is now considered for inlining exactly one time, which means that the total growth in MIR size is limited to number of call sites * inlining threshold. Top down inlining may be worth re-introducing at some point, but it needs to be accompanied with a principled way to prevent this kind of behavior.
2 parents 8e440b0 + f4f7777 commit 226202d

10 files changed

+243
-115
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

+6-22
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ use rustc_middle::mir::visit::*;
88
use rustc_middle::mir::*;
99
use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
1010
use rustc_session::config::OptLevel;
11-
use rustc_span::def_id::DefId;
1211
use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span};
1312
use rustc_target::abi::VariantIdx;
1413
use rustc_target::spec::abi::Abi;
@@ -87,13 +86,8 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
8786

8887
let param_env = tcx.param_env_reveal_all_normalized(def_id);
8988

90-
let mut this = Inliner {
91-
tcx,
92-
param_env,
93-
codegen_fn_attrs: tcx.codegen_fn_attrs(def_id),
94-
history: Vec::new(),
95-
changed: false,
96-
};
89+
let mut this =
90+
Inliner { tcx, param_env, codegen_fn_attrs: tcx.codegen_fn_attrs(def_id), changed: false };
9791
let blocks = BasicBlock::new(0)..body.basic_blocks.next_index();
9892
this.process_blocks(body, blocks);
9993
this.changed
@@ -104,12 +98,6 @@ struct Inliner<'tcx> {
10498
param_env: ParamEnv<'tcx>,
10599
/// Caller codegen attributes.
106100
codegen_fn_attrs: &'tcx CodegenFnAttrs,
107-
/// Stack of inlined instances.
108-
/// We only check the `DefId` and not the substs because we want to
109-
/// avoid inlining cases of polymorphic recursion.
110-
/// The number of `DefId`s is finite, so checking history is enough
111-
/// to ensure that we do not loop endlessly while inlining.
112-
history: Vec<DefId>,
113101
/// Indicates that the caller body has been modified.
114102
changed: bool,
115103
}
@@ -134,12 +122,12 @@ impl<'tcx> Inliner<'tcx> {
134122
debug!("not-inlined {} [{}]", callsite.callee, reason);
135123
continue;
136124
}
137-
Ok(new_blocks) => {
125+
Ok(_) => {
138126
debug!("inlined {}", callsite.callee);
139127
self.changed = true;
140-
self.history.push(callsite.callee.def_id());
141-
self.process_blocks(caller_body, new_blocks);
142-
self.history.pop();
128+
// We could process the blocks returned by `try_inlining` here. However, that
129+
// leads to exponential compile times due to the top-down nature of this kind
130+
// of inlining.
143131
}
144132
}
145133
}
@@ -313,10 +301,6 @@ impl<'tcx> Inliner<'tcx> {
313301
return None;
314302
}
315303

316-
if self.history.contains(&callee.def_id()) {
317-
return None;
318-
}
319-
320304
let fn_sig = self.tcx.bound_fn_sig(def_id).subst(self.tcx, substs);
321305

322306
return Some(CallSite {

‎src/test/mir-opt/inline/cycle.g.Inline.diff

+12-11
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,6 @@
1010
+ let _3: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
1111
+ let mut _4: &fn() {main}; // in scope 1 at $DIR/cycle.rs:6:5: 6:6
1212
+ let mut _5: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
13-
+ scope 2 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { // at $DIR/cycle.rs:6:5: 6:8
14-
+ }
1513
+ }
1614

1715
bb0: {
@@ -29,7 +27,10 @@
2927
+ StorageLive(_4); // scope 1 at $DIR/cycle.rs:6:5: 6:6
3028
+ _4 = &_2; // scope 1 at $DIR/cycle.rs:6:5: 6:6
3129
+ StorageLive(_5); // scope 1 at $DIR/cycle.rs:6:5: 6:8
32-
+ _3 = move (*_4)() -> [return: bb4, unwind: bb2]; // scope 2 at $SRC_DIR/core/src/ops/function.rs:LL:COL
30+
+ _3 = <fn() {main} as Fn<()>>::call(move _4, move _5) -> [return: bb2, unwind: bb3]; // scope 1 at $DIR/cycle.rs:6:5: 6:8
31+
+ // mir::Constant
32+
+ // + span: $DIR/cycle.rs:6:5: 6:6
33+
+ // + literal: Const { ty: for<'a> extern "rust-call" fn(&'a fn() {main}, ()) -> <fn() {main} as FnOnce<()>>::Output {<fn() {main} as Fn<()>>::call}, val: Value(<ZST>) }
3334
}
3435

3536
bb1: {
@@ -39,19 +40,19 @@
3940
return; // scope 0 at $DIR/cycle.rs:+2:2: +2:2
4041
+ }
4142
+
42-
+ bb2 (cleanup): {
43-
+ drop(_2) -> bb3; // scope 1 at $DIR/cycle.rs:7:1: 7:2
43+
+ bb2: {
44+
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
45+
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
46+
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
47+
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
4448
+ }
4549
+
4650
+ bb3 (cleanup): {
47-
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
51+
+ drop(_2) -> bb4; // scope 1 at $DIR/cycle.rs:7:1: 7:2
4852
+ }
4953
+
50-
+ bb4: {
51-
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
52-
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
53-
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
54-
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
54+
+ bb4 (cleanup): {
55+
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
5556
}
5657
}
5758

‎src/test/mir-opt/inline/cycle.main.Inline.diff

+12-28
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,6 @@
1010
+ let _3: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
1111
+ let mut _4: &fn() {g}; // in scope 1 at $DIR/cycle.rs:6:5: 6:6
1212
+ let mut _5: (); // in scope 1 at $DIR/cycle.rs:6:5: 6:8
13-
+ scope 2 (inlined <fn() {g} as Fn<()>>::call - shim(fn() {g})) { // at $DIR/cycle.rs:6:5: 6:8
14-
+ scope 3 (inlined g) { // at $SRC_DIR/core/src/ops/function.rs:LL:COL
15-
+ let mut _6: fn() {main}; // in scope 3 at $DIR/cycle.rs:12:5: 12:12
16-
+ scope 4 (inlined f::<fn() {main}>) { // at $DIR/cycle.rs:12:5: 12:12
17-
+ debug g => _6; // in scope 4 at $DIR/cycle.rs:5:6: 5:7
18-
+ let _7: (); // in scope 4 at $DIR/cycle.rs:6:5: 6:8
19-
+ let mut _8: &fn() {main}; // in scope 4 at $DIR/cycle.rs:6:5: 6:6
20-
+ scope 5 (inlined <fn() {main} as Fn<()>>::call - shim(fn() {main})) { // at $DIR/cycle.rs:6:5: 6:8
21-
+ }
22-
+ }
23-
+ }
24-
+ }
2513
+ }
2614

2715
bb0: {
@@ -39,11 +27,10 @@
3927
+ StorageLive(_4); // scope 1 at $DIR/cycle.rs:6:5: 6:6
4028
+ _4 = &_2; // scope 1 at $DIR/cycle.rs:6:5: 6:6
4129
+ StorageLive(_5); // scope 1 at $DIR/cycle.rs:6:5: 6:8
42-
+ StorageLive(_6); // scope 3 at $DIR/cycle.rs:12:5: 12:12
43-
+ StorageLive(_7); // scope 4 at $DIR/cycle.rs:6:5: 6:8
44-
+ StorageLive(_8); // scope 4 at $DIR/cycle.rs:6:5: 6:6
45-
+ _8 = &_6; // scope 4 at $DIR/cycle.rs:6:5: 6:6
46-
+ _7 = move (*_8)() -> [return: bb4, unwind: bb2]; // scope 5 at $SRC_DIR/core/src/ops/function.rs:LL:COL
30+
+ _3 = <fn() {g} as Fn<()>>::call(move _4, move _5) -> [return: bb2, unwind: bb3]; // scope 1 at $DIR/cycle.rs:6:5: 6:8
31+
+ // mir::Constant
32+
+ // + span: $DIR/cycle.rs:6:5: 6:6
33+
+ // + literal: Const { ty: for<'a> extern "rust-call" fn(&'a fn() {g}, ()) -> <fn() {g} as FnOnce<()>>::Output {<fn() {g} as Fn<()>>::call}, val: Value(<ZST>) }
4734
}
4835

4936
bb1: {
@@ -53,22 +40,19 @@
5340
return; // scope 0 at $DIR/cycle.rs:+2:2: +2:2
5441
+ }
5542
+
56-
+ bb2 (cleanup): {
57-
+ drop(_2) -> bb3; // scope 1 at $DIR/cycle.rs:7:1: 7:2
43+
+ bb2: {
44+
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
45+
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
46+
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
47+
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
5848
+ }
5949
+
6050
+ bb3 (cleanup): {
61-
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
51+
+ drop(_2) -> bb4; // scope 1 at $DIR/cycle.rs:7:1: 7:2
6252
+ }
6353
+
64-
+ bb4: {
65-
+ StorageDead(_8); // scope 4 at $DIR/cycle.rs:6:7: 6:8
66-
+ StorageDead(_7); // scope 4 at $DIR/cycle.rs:6:8: 6:9
67-
+ StorageDead(_6); // scope 3 at $DIR/cycle.rs:12:5: 12:12
68-
+ StorageDead(_5); // scope 1 at $DIR/cycle.rs:6:7: 6:8
69-
+ StorageDead(_4); // scope 1 at $DIR/cycle.rs:6:7: 6:8
70-
+ StorageDead(_3); // scope 1 at $DIR/cycle.rs:6:8: 6:9
71-
+ drop(_2) -> bb1; // scope 1 at $DIR/cycle.rs:7:1: 7:2
54+
+ bb4 (cleanup): {
55+
+ resume; // scope 1 at $DIR/cycle.rs:5:1: 7:2
7256
}
7357
}
7458

‎src/test/mir-opt/inline/exponential_runtime.main.Inline.diff

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
- // MIR for `main` before Inline
2+
+ // MIR for `main` after Inline
3+
4+
fn main() -> () {
5+
let mut _0: (); // return place in scope 0 at $DIR/exponential_runtime.rs:+0:11: +0:11
6+
let _1: (); // in scope 0 at $DIR/exponential_runtime.rs:+1:5: +1:22
7+
+ scope 1 (inlined <() as G>::call) { // at $DIR/exponential_runtime.rs:86:5: 86:22
8+
+ let _2: (); // in scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
9+
+ let _3: (); // in scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
10+
+ let _4: (); // in scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
11+
+ }
12+
13+
bb0: {
14+
StorageLive(_1); // scope 0 at $DIR/exponential_runtime.rs:+1:5: +1:22
15+
- _1 = <() as G>::call() -> bb1; // scope 0 at $DIR/exponential_runtime.rs:+1:5: +1:22
16+
+ StorageLive(_2); // scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
17+
+ _2 = <() as F>::call() -> bb1; // scope 1 at $DIR/exponential_runtime.rs:73:9: 73:25
18+
// mir::Constant
19+
- // + span: $DIR/exponential_runtime.rs:86:5: 86:20
20+
- // + literal: Const { ty: fn() {<() as G>::call}, val: Value(<ZST>) }
21+
+ // + span: $DIR/exponential_runtime.rs:73:9: 73:23
22+
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
23+
}
24+
25+
bb1: {
26+
+ StorageDead(_2); // scope 1 at $DIR/exponential_runtime.rs:73:25: 73:26
27+
+ StorageLive(_3); // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
28+
+ _3 = <() as F>::call() -> bb2; // scope 1 at $DIR/exponential_runtime.rs:74:9: 74:25
29+
+ // mir::Constant
30+
+ // + span: $DIR/exponential_runtime.rs:74:9: 74:23
31+
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
32+
+ }
33+
+
34+
+ bb2: {
35+
+ StorageDead(_3); // scope 1 at $DIR/exponential_runtime.rs:74:25: 74:26
36+
+ StorageLive(_4); // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
37+
+ _4 = <() as F>::call() -> bb3; // scope 1 at $DIR/exponential_runtime.rs:75:9: 75:25
38+
+ // mir::Constant
39+
+ // + span: $DIR/exponential_runtime.rs:75:9: 75:23
40+
+ // + literal: Const { ty: fn() {<() as F>::call}, val: Value(<ZST>) }
41+
+ }
42+
+
43+
+ bb3: {
44+
+ StorageDead(_4); // scope 1 at $DIR/exponential_runtime.rs:75:25: 75:26
45+
StorageDead(_1); // scope 0 at $DIR/exponential_runtime.rs:+1:22: +1:23
46+
_0 = const (); // scope 0 at $DIR/exponential_runtime.rs:+0:11: +2:2
47+
return; // scope 0 at $DIR/exponential_runtime.rs:+2:2: +2:2
48+
}
49+
}
50+

‎src/test/mir-opt/inline/exponential_runtime.rs

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
// Checks that code with exponential runtime does not have exponential behavior in inlining.
2+
3+
trait A {
4+
fn call();
5+
}
6+
7+
trait B {
8+
fn call();
9+
}
10+
impl<T: A> B for T {
11+
#[inline]
12+
fn call() {
13+
<T as A>::call();
14+
<T as A>::call();
15+
<T as A>::call();
16+
}
17+
}
18+
19+
trait C {
20+
fn call();
21+
}
22+
impl<T: B> C for T {
23+
#[inline]
24+
fn call() {
25+
<T as B>::call();
26+
<T as B>::call();
27+
<T as B>::call();
28+
}
29+
}
30+
31+
trait D {
32+
fn call();
33+
}
34+
impl<T: C> D for T {
35+
#[inline]
36+
fn call() {
37+
<T as C>::call();
38+
<T as C>::call();
39+
<T as C>::call();
40+
}
41+
}
42+
43+
trait E {
44+
fn call();
45+
}
46+
impl<T: D> E for T {
47+
#[inline]
48+
fn call() {
49+
<T as D>::call();
50+
<T as D>::call();
51+
<T as D>::call();
52+
}
53+
}
54+
55+
trait F {
56+
fn call();
57+
}
58+
impl<T: E> F for T {
59+
#[inline]
60+
fn call() {
61+
<T as E>::call();
62+
<T as E>::call();
63+
<T as E>::call();
64+
}
65+
}
66+
67+
trait G {
68+
fn call();
69+
}
70+
impl<T: F> G for T {
71+
#[inline]
72+
fn call() {
73+
<T as F>::call();
74+
<T as F>::call();
75+
<T as F>::call();
76+
}
77+
}
78+
79+
impl A for () {
80+
#[inline(never)]
81+
fn call() {}
82+
}
83+
84+
// EMIT_MIR exponential_runtime.main.Inline.diff
85+
fn main() {
86+
<() as G>::call();
87+
}

‎src/test/mir-opt/inline/inline_cycle.one.Inline.diff

+4-7
Original file line numberDiff line numberDiff line change
@@ -5,20 +5,17 @@
55
let mut _0: (); // return place in scope 0 at $DIR/inline_cycle.rs:+0:10: +0:10
66
let _1: (); // in scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
77
+ scope 1 (inlined <C as Call>::call) { // at $DIR/inline_cycle.rs:14:5: 14:24
8-
+ scope 2 (inlined <A<C> as Call>::call) { // at $DIR/inline_cycle.rs:43:9: 43:23
9-
+ scope 3 (inlined <B<C> as Call>::call) { // at $DIR/inline_cycle.rs:28:9: 28:31
10-
+ }
11-
+ }
128
+ }
139

1410
bb0: {
1511
StorageLive(_1); // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
1612
- _1 = <C as Call>::call() -> bb1; // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:24
17-
+ _1 = <C as Call>::call() -> bb1; // scope 3 at $DIR/inline_cycle.rs:36:9: 36:28
13+
+ _1 = <A<C> as Call>::call() -> bb1; // scope 1 at $DIR/inline_cycle.rs:43:9: 43:23
1814
// mir::Constant
1915
- // + span: $DIR/inline_cycle.rs:14:5: 14:22
20-
+ // + span: $DIR/inline_cycle.rs:36:9: 36:26
21-
// + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
16+
- // + literal: Const { ty: fn() {<C as Call>::call}, val: Value(<ZST>) }
17+
+ // + span: $DIR/inline_cycle.rs:43:9: 43:21
18+
+ // + literal: Const { ty: fn() {<A<C> as Call>::call}, val: Value(<ZST>) }
2219
}
2320

2421
bb1: {

‎src/test/mir-opt/inline/inline_cycle.two.Inline.diff

+7-16
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,6 @@
99
+ debug f => _2; // in scope 1 at $DIR/inline_cycle.rs:53:22: 53:23
1010
+ let _3: (); // in scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
1111
+ let mut _4: (); // in scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
12-
+ scope 2 (inlined <fn() {f} as FnOnce<()>>::call_once - shim(fn() {f})) { // at $DIR/inline_cycle.rs:54:5: 54:8
13-
+ scope 3 (inlined f) { // at $SRC_DIR/core/src/ops/function.rs:LL:COL
14-
+ let _5: (); // in scope 3 at $DIR/inline_cycle.rs:59:5: 59:12
15-
+ }
16-
+ }
1712
+ }
1813

1914
bb0: {
@@ -23,23 +18,19 @@
2318
+ _2 = f; // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:12
2419
// mir::Constant
2520
- // + span: $DIR/inline_cycle.rs:49:5: 49:9
26-
+ // + span: $DIR/inline_cycle.rs:49:10: 49:11
27-
+ // + literal: Const { ty: fn() {f}, val: Value(<ZST>) }
21+
- // + literal: Const { ty: fn(fn() {f}) {call::<fn() {f}>}, val: Value(<ZST>) }
22+
- // mir::Constant
23+
// + span: $DIR/inline_cycle.rs:49:10: 49:11
24+
// + literal: Const { ty: fn() {f}, val: Value(<ZST>) }
2825
+ StorageLive(_3); // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
2926
+ StorageLive(_4); // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
30-
+ StorageLive(_5); // scope 3 at $DIR/inline_cycle.rs:59:5: 59:12
31-
+ _5 = call::<fn() {f}>(f) -> bb1; // scope 3 at $DIR/inline_cycle.rs:59:5: 59:12
27+
+ _3 = <fn() {f} as FnOnce<()>>::call_once(move _2, move _4) -> bb1; // scope 1 at $DIR/inline_cycle.rs:54:5: 54:8
3228
+ // mir::Constant
33-
+ // + span: $DIR/inline_cycle.rs:59:5: 59:9
34-
// + literal: Const { ty: fn(fn() {f}) {call::<fn() {f}>}, val: Value(<ZST>) }
35-
// mir::Constant
36-
- // + span: $DIR/inline_cycle.rs:49:10: 49:11
37-
+ // + span: $DIR/inline_cycle.rs:59:10: 59:11
38-
// + literal: Const { ty: fn() {f}, val: Value(<ZST>) }
29+
+ // + span: $DIR/inline_cycle.rs:54:5: 54:6
30+
+ // + literal: Const { ty: extern "rust-call" fn(fn() {f}, ()) -> <fn() {f} as FnOnce<()>>::Output {<fn() {f} as FnOnce<()>>::call_once}, val: Value(<ZST>) }
3931
}
4032

4133
bb1: {
42-
+ StorageDead(_5); // scope 3 at $DIR/inline_cycle.rs:59:12: 59:13
4334
+ StorageDead(_4); // scope 1 at $DIR/inline_cycle.rs:54:7: 54:8
4435
+ StorageDead(_3); // scope 1 at $DIR/inline_cycle.rs:54:8: 54:9
4536
+ StorageDead(_2); // scope 0 at $DIR/inline_cycle.rs:+1:5: +1:12

0 commit comments

Comments
 (0)