Skip to content

Commit 52e8117

Browse files
cjgillotcjgillot
committed
Do not count inline(always) in inlining cost.
1 parent aec2861 commit 52e8117

6 files changed

+185
-110
lines changed

‎compiler/rustc_mir_transform/src/inline.rs

+32-18
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
9595
changed: false,
9696
};
9797
let blocks = START_BLOCK..body.basic_blocks.next_index();
98-
this.process_blocks(body, blocks);
98+
this.process_blocks(body, blocks, 0);
9999
this.changed
100100
}
101101

@@ -115,10 +115,15 @@ struct Inliner<'tcx> {
115115
}
116116

117117
impl<'tcx> Inliner<'tcx> {
118-
fn process_blocks(&mut self, caller_body: &mut Body<'tcx>, blocks: Range<BasicBlock>) {
118+
fn process_blocks(
119+
&mut self,
120+
caller_body: &mut Body<'tcx>,
121+
blocks: Range<BasicBlock>,
122+
depth: usize,
123+
) {
119124
// How many callsites in this body are we allowed to inline? We need to limit this in order
120125
// to prevent super-linear growth in MIR size
121-
let inline_limit = match self.history.len() {
126+
let inline_limit = match depth {
122127
0 => usize::MAX,
123128
1..=TOP_DOWN_DEPTH_LIMIT => 1,
124129
_ => return,
@@ -137,26 +142,35 @@ impl<'tcx> Inliner<'tcx> {
137142
let span = trace_span!("process_blocks", %callsite.callee, ?bb);
138143
let _guard = span.enter();
139144

140-
match self.try_inlining(caller_body, &callsite) {
145+
let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
146+
let inline_always = matches!(callee_attrs.inline, InlineAttr::Always);
147+
148+
if inlined_count >= inline_limit && !inline_always {
149+
debug!("inline count reached");
150+
continue;
151+
}
152+
153+
let new_blocks = match self.try_inlining(caller_body, &callsite, callee_attrs) {
154+
Ok(new_blocks) => new_blocks,
141155
Err(reason) => {
142156
debug!("not-inlined {} [{}]", callsite.callee, reason);
143157
continue;
144158
}
145-
Ok(new_blocks) => {
146-
debug!("inlined {}", callsite.callee);
147-
self.changed = true;
159+
};
148160

149-
self.history.push(callsite.callee.def_id());
150-
self.process_blocks(caller_body, new_blocks);
151-
self.history.pop();
161+
debug!("inlined {}", callsite.callee);
162+
self.changed = true;
152163

153-
inlined_count += 1;
154-
if inlined_count == inline_limit {
155-
debug!("inline count reached");
156-
return;
157-
}
158-
}
159-
}
164+
let new_depth = if inline_always {
165+
// This call was `inline(always)`. Do not count it in the inline cost.
166+
depth
167+
} else {
168+
inlined_count += 1;
169+
depth + 1
170+
};
171+
self.history.push(callsite.callee.def_id());
172+
self.process_blocks(caller_body, new_blocks, new_depth);
173+
self.history.pop();
160174
}
161175
}
162176

@@ -167,8 +181,8 @@ impl<'tcx> Inliner<'tcx> {
167181
&self,
168182
caller_body: &mut Body<'tcx>,
169183
callsite: &CallSite<'tcx>,
184+
callee_attrs: &CodegenFnAttrs,
170185
) -> Result<std::ops::Range<BasicBlock>, &'static str> {
171-
let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
172186
self.check_codegen_attributes(callsite, callee_attrs)?;
173187

174188
let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();

‎tests/mir-opt/inline/inline_diverging.h.Inline.diff

+5
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
let _1: (!, !); // in scope 0 at $DIR/inline_diverging.rs:+1:5: +1:22
77
+ let mut _2: fn() -> ! {sleep}; // in scope 0 at $DIR/inline_diverging.rs:+1:5: +1:22
88
+ let mut _8: (); // in scope 0 at $DIR/inline_diverging.rs:27:13: 27:16
9+
+ let mut _9: (); // in scope 0 at $DIR/inline_diverging.rs:28:13: 28:16
910
+ scope 1 (inlined call_twice::<!, fn() -> ! {sleep}>) { // at $DIR/inline_diverging.rs:22:5: 22:22
1011
+ debug f => _2; // in scope 1 at $DIR/inline_diverging.rs:26:36: 26:37
1112
+ let mut _3: &fn() -> ! {sleep}; // in scope 1 at $DIR/inline_diverging.rs:27:13: 27:14
@@ -18,6 +19,10 @@
1819
+ scope 3 {
1920
+ debug b => _6; // in scope 3 at $DIR/inline_diverging.rs:28:9: 28:10
2021
+ }
22+
+ scope 6 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { // at $DIR/inline_diverging.rs:28:13: 28:16
23+
+ scope 7 (inlined sleep) { // at $SRC_DIR/core/src/ops/function.rs:LL:COL
24+
+ }
25+
+ }
2126
+ }
2227
+ scope 4 (inlined <fn() -> ! {sleep} as Fn<()>>::call - shim(fn() -> ! {sleep})) { // at $DIR/inline_diverging.rs:27:13: 27:16
2328
+ scope 5 (inlined sleep) { // at $SRC_DIR/core/src/ops/function.rs:LL:COL

‎tests/mir-opt/pre-codegen/loops.int_range.PreCodegen.after.mir

+31-17
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,14 @@ fn int_range(_1: usize, _2: usize) -> () {
88
let mut _4: std::ops::Range<usize>; // in scope 0 at $DIR/loops.rs:+1:14: +1:24
99
let mut _5: &mut std::ops::Range<usize>; // in scope 0 at $DIR/loops.rs:+1:14: +1:24
1010
let mut _11: std::option::Option<usize>; // in scope 0 at $DIR/loops.rs:+1:14: +1:24
11-
let mut _14: isize; // in scope 0 at $DIR/loops.rs:+1:5: +3:6
12-
let _16: (); // in scope 0 at $DIR/loops.rs:+1:14: +1:24
11+
let mut _13: usize; // in scope 0 at $SRC_DIR/core/src/iter/range.rs:LL:COL
12+
let mut _15: isize; // in scope 0 at $DIR/loops.rs:+1:5: +3:6
13+
let _17: (); // in scope 0 at $DIR/loops.rs:+1:14: +1:24
1314
scope 1 {
1415
debug iter => _4; // in scope 1 at $DIR/loops.rs:+1:14: +1:24
15-
let _15: usize; // in scope 1 at $DIR/loops.rs:+1:9: +1:10
16+
let _16: usize; // in scope 1 at $DIR/loops.rs:+1:9: +1:10
1617
scope 2 {
17-
debug i => _15; // in scope 2 at $DIR/loops.rs:+1:9: +1:10
18+
debug i => _16; // in scope 2 at $DIR/loops.rs:+1:9: +1:10
1819
}
1920
scope 4 (inlined iter::range::<impl Iterator for std::ops::Range<usize>>::next) { // at $DIR/loops.rs:8:14: 8:24
2021
debug self => _5; // in scope 4 at $SRC_DIR/core/src/iter/range.rs:LL:COL
@@ -24,10 +25,22 @@ fn int_range(_1: usize, _2: usize) -> () {
2425
let mut _7: &usize; // in scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
2526
let mut _10: bool; // in scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
2627
let _12: usize; // in scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
27-
let mut _13: usize; // in scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
28+
let mut _14: usize; // in scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
2829
scope 6 {
2930
debug old => _12; // in scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
3031
scope 7 {
32+
scope 9 (inlined <usize as Step>::forward_unchecked) { // at $SRC_DIR/core/src/iter/range.rs:LL:COL
33+
debug start => _12; // in scope 9 at $SRC_DIR/core/src/iter/range.rs:LL:COL
34+
debug n => const 1_usize; // in scope 9 at $SRC_DIR/core/src/iter/range.rs:LL:COL
35+
scope 10 {
36+
scope 11 (inlined core::num::<impl usize>::unchecked_add) { // at $SRC_DIR/core/src/iter/range.rs:LL:COL
37+
debug self => _12; // in scope 11 at $SRC_DIR/core/src/num/uint_macros.rs:LL:COL
38+
debug rhs => const 1_usize; // in scope 11 at $SRC_DIR/core/src/num/uint_macros.rs:LL:COL
39+
scope 12 {
40+
}
41+
}
42+
}
43+
}
3144
}
3245
}
3346
scope 8 (inlined cmp::impls::<impl PartialOrd for usize>::lt) { // at $SRC_DIR/core/src/iter/range.rs:LL:COL
@@ -53,7 +66,6 @@ fn int_range(_1: usize, _2: usize) -> () {
5366
bb1: {
5467
StorageLive(_11); // scope 1 at $DIR/loops.rs:+1:14: +1:24
5568
_5 = &mut _4; // scope 1 at $DIR/loops.rs:+1:14: +1:24
56-
StorageLive(_12); // scope 4 at $SRC_DIR/core/src/iter/range.rs:LL:COL
5769
StorageLive(_10); // scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
5870
StorageLive(_6); // scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
5971
_6 = &((*_5).0: usize); // scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
@@ -78,25 +90,27 @@ fn int_range(_1: usize, _2: usize) -> () {
7890

7991
bb3: {
8092
_12 = ((*_5).0: usize); // scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
81-
StorageLive(_13); // scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
82-
_13 = <usize as Step>::forward_unchecked(_12, const 1_usize) -> bb4; // scope 7 at $SRC_DIR/core/src/iter/range.rs:LL:COL
93+
StorageLive(_14); // scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
94+
StorageLive(_13); // scope 7 at $SRC_DIR/core/src/iter/range.rs:LL:COL
95+
_13 = const 1_usize; // scope 7 at $SRC_DIR/core/src/iter/range.rs:LL:COL
96+
_14 = unchecked_add::<usize>(_12, _13) -> [return: bb4, unwind unreachable]; // scope 12 at $SRC_DIR/core/src/num/uint_macros.rs:LL:COL
8397
// mir::Constant
84-
// + span: $SRC_DIR/core/src/iter/range.rs:LL:COL
85-
// + literal: Const { ty: unsafe fn(usize, usize) -> usize {<usize as Step>::forward_unchecked}, val: Value(<ZST>) }
98+
// + span: $SRC_DIR/core/src/num/uint_macros.rs:LL:COL
99+
// + literal: Const { ty: unsafe extern "rust-intrinsic" fn(usize, usize) -> usize {unchecked_add::<usize>}, val: Value(<ZST>) }
86100
}
87101

88102
bb4: {
89-
((*_5).0: usize) = move _13; // scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
90-
StorageDead(_13); // scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
103+
StorageDead(_13); // scope 7 at $SRC_DIR/core/src/iter/range.rs:LL:COL
104+
((*_5).0: usize) = move _14; // scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
105+
StorageDead(_14); // scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
91106
_11 = Option::<usize>::Some(_12); // scope 6 at $SRC_DIR/core/src/iter/range.rs:LL:COL
92107
goto -> bb5; // scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
93108
}
94109

95110
bb5: {
96111
StorageDead(_10); // scope 5 at $SRC_DIR/core/src/iter/range.rs:LL:COL
97-
StorageDead(_12); // scope 4 at $SRC_DIR/core/src/iter/range.rs:LL:COL
98-
_14 = discriminant(_11); // scope 1 at $DIR/loops.rs:+1:14: +1:24
99-
switchInt(move _14) -> [0: bb6, 1: bb7, otherwise: bb9]; // scope 1 at $DIR/loops.rs:+1:14: +1:24
112+
_15 = discriminant(_11); // scope 1 at $DIR/loops.rs:+1:14: +1:24
113+
switchInt(move _15) -> [0: bb6, 1: bb7, otherwise: bb9]; // scope 1 at $DIR/loops.rs:+1:14: +1:24
100114
}
101115

102116
bb6: {
@@ -106,8 +120,8 @@ fn int_range(_1: usize, _2: usize) -> () {
106120
}
107121

108122
bb7: {
109-
_15 = ((_11 as Some).0: usize); // scope 1 at $DIR/loops.rs:+1:9: +1:10
110-
_16 = opaque::<usize>(_15) -> bb8; // scope 2 at $DIR/loops.rs:+2:9: +2:18
123+
_16 = ((_11 as Some).0: usize); // scope 1 at $DIR/loops.rs:+1:9: +1:10
124+
_17 = opaque::<usize>(_16) -> bb8; // scope 2 at $DIR/loops.rs:+2:9: +2:18
111125
// mir::Constant
112126
// + span: $DIR/loops.rs:9:9: 9:15
113127
// + literal: Const { ty: fn(usize) {opaque::<usize>}, val: Value(<ZST>) }

0 commit comments

Comments
 (0)