Skip to content

Commit 0b521e5

Browse files
jonas-schievinkjonas-schievink
authored
Rollup merge of #78899 - tmiasko:inline-diverging, r=oli-obk
Support inlining diverging function calls The existing heuristic does penalize diverging calls to some degree, but since it never inlined them previously it might need some further modifications. Additionally introduce storage markers for all temporaries created by the inliner. The temporary introduced for destination rebrorrow, didn't use them previously.
2 parents 919177f + ffa70d7 commit 0b521e5

File tree

7 files changed

+256
-76
lines changed

7 files changed

+256
-76
lines changed

‎compiler/rustc_mir/src/transform/inline.rs‎

Lines changed: 76 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ pub struct Inline;
2828
#[derive(Copy, Clone, Debug)]
2929
struct CallSite<'tcx> {
3030
callee: Instance<'tcx>,
31-
bb: BasicBlock,
31+
block: BasicBlock,
32+
target: Option<BasicBlock>,
3233
source_info: SourceInfo,
3334
}
3435

@@ -172,8 +173,7 @@ impl Inliner<'tcx> {
172173

173174
// Only consider direct calls to functions
174175
let terminator = bb_data.terminator();
175-
// FIXME: Handle inlining of diverging calls
176-
if let TerminatorKind::Call { func: ref op, destination: Some(_), .. } = terminator.kind {
176+
if let TerminatorKind::Call { func: ref op, ref destination, .. } = terminator.kind {
177177
if let ty::FnDef(callee_def_id, substs) = *op.ty(caller_body, self.tcx).kind() {
178178
// To resolve an instance its substs have to be fully normalized, so
179179
// we do this here.
@@ -187,7 +187,12 @@ impl Inliner<'tcx> {
187187
return None;
188188
}
189189

190-
return Some(CallSite { callee, bb, source_info: terminator.source_info });
190+
return Some(CallSite {
191+
callee,
192+
block: bb,
193+
target: destination.map(|(_, target)| target),
194+
source_info: terminator.source_info,
195+
});
191196
}
192197
}
193198

@@ -399,9 +404,9 @@ impl Inliner<'tcx> {
399404
caller_body: &mut Body<'tcx>,
400405
mut callee_body: Body<'tcx>,
401406
) {
402-
let terminator = caller_body[callsite.bb].terminator.take().unwrap();
407+
let terminator = caller_body[callsite.block].terminator.take().unwrap();
403408
match terminator.kind {
404-
TerminatorKind::Call { args, destination: Some(destination), cleanup, .. } => {
409+
TerminatorKind::Call { args, destination, cleanup, .. } => {
405410
// If the call is something like `a[*i] = f(i)`, where
406411
// `i : &mut usize`, then just duplicating the `a[*i]`
407412
// Place could result in two different locations if `f`
@@ -418,43 +423,39 @@ impl Inliner<'tcx> {
418423
false
419424
}
420425

421-
let dest = if dest_needs_borrow(destination.0) {
422-
trace!("creating temp for return destination");
423-
let dest = Rvalue::Ref(
424-
self.tcx.lifetimes.re_erased,
425-
BorrowKind::Mut { allow_two_phase_borrow: false },
426-
destination.0,
427-
);
428-
429-
let ty = dest.ty(caller_body, self.tcx);
430-
431-
let temp = LocalDecl::new(ty, callsite.source_info.span);
432-
433-
let tmp = caller_body.local_decls.push(temp);
434-
let tmp = Place::from(tmp);
435-
436-
let stmt = Statement {
437-
source_info: callsite.source_info,
438-
kind: StatementKind::Assign(box (tmp, dest)),
439-
};
440-
caller_body[callsite.bb].statements.push(stmt);
441-
self.tcx.mk_place_deref(tmp)
426+
let dest = if let Some((destination_place, _)) = destination {
427+
if dest_needs_borrow(destination_place) {
428+
trace!("creating temp for return destination");
429+
let dest = Rvalue::Ref(
430+
self.tcx.lifetimes.re_erased,
431+
BorrowKind::Mut { allow_two_phase_borrow: false },
432+
destination_place,
433+
);
434+
let dest_ty = dest.ty(caller_body, self.tcx);
435+
let temp = Place::from(self.new_call_temp(caller_body, &callsite, dest_ty));
436+
caller_body[callsite.block].statements.push(Statement {
437+
source_info: callsite.source_info,
438+
kind: StatementKind::Assign(box (temp, dest)),
439+
});
440+
self.tcx.mk_place_deref(temp)
441+
} else {
442+
destination_place
443+
}
442444
} else {
443-
destination.0
445+
trace!("creating temp for return place");
446+
Place::from(self.new_call_temp(caller_body, &callsite, callee_body.return_ty()))
444447
};
445448

446-
let return_block = destination.1;
447-
448449
// Copy the arguments if needed.
449-
let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, return_block);
450+
let args: Vec<_> = self.make_call_args(args, &callsite, caller_body);
450451

451452
let mut integrator = Integrator {
452453
args: &args,
453454
new_locals: Local::new(caller_body.local_decls.len())..,
454455
new_scopes: SourceScope::new(caller_body.source_scopes.len())..,
455456
new_blocks: BasicBlock::new(caller_body.basic_blocks().len())..,
456457
destination: dest,
457-
return_block,
458+
return_block: callsite.target,
458459
cleanup_block: cleanup,
459460
in_cleanup_block: false,
460461
tcx: self.tcx,
@@ -503,7 +504,7 @@ impl Inliner<'tcx> {
503504
caller_body.var_debug_info.extend(callee_body.var_debug_info.drain(..));
504505
caller_body.basic_blocks_mut().extend(callee_body.basic_blocks_mut().drain(..));
505506

506-
caller_body[callsite.bb].terminator = Some(Terminator {
507+
caller_body[callsite.block].terminator = Some(Terminator {
507508
source_info: callsite.source_info,
508509
kind: TerminatorKind::Goto { target: integrator.map_block(START_BLOCK) },
509510
});
@@ -527,7 +528,6 @@ impl Inliner<'tcx> {
527528
args: Vec<Operand<'tcx>>,
528529
callsite: &CallSite<'tcx>,
529530
caller_body: &mut Body<'tcx>,
530-
return_block: BasicBlock,
531531
) -> Vec<Local> {
532532
let tcx = self.tcx;
533533

@@ -558,18 +558,8 @@ impl Inliner<'tcx> {
558558
// `callee_body.spread_arg == None`, instead of special-casing closures.
559559
if tcx.is_closure(callsite.callee.def_id()) {
560560
let mut args = args.into_iter();
561-
let self_ = self.create_temp_if_necessary(
562-
args.next().unwrap(),
563-
callsite,
564-
caller_body,
565-
return_block,
566-
);
567-
let tuple = self.create_temp_if_necessary(
568-
args.next().unwrap(),
569-
callsite,
570-
caller_body,
571-
return_block,
572-
);
561+
let self_ = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
562+
let tuple = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_body);
573563
assert!(args.next().is_none());
574564

575565
let tuple = Place::from(tuple);
@@ -589,13 +579,13 @@ impl Inliner<'tcx> {
589579
Operand::Move(tcx.mk_place_field(tuple, Field::new(i), ty.expect_ty()));
590580

591581
// Spill to a local to make e.g., `tmp0`.
592-
self.create_temp_if_necessary(tuple_field, callsite, caller_body, return_block)
582+
self.create_temp_if_necessary(tuple_field, callsite, caller_body)
593583
});
594584

595585
closure_ref_arg.chain(tuple_tmp_args).collect()
596586
} else {
597587
args.into_iter()
598-
.map(|a| self.create_temp_if_necessary(a, callsite, caller_body, return_block))
588+
.map(|a| self.create_temp_if_necessary(a, callsite, caller_body))
599589
.collect()
600590
}
601591
}
@@ -607,46 +597,52 @@ impl Inliner<'tcx> {
607597
arg: Operand<'tcx>,
608598
callsite: &CallSite<'tcx>,
609599
caller_body: &mut Body<'tcx>,
610-
return_block: BasicBlock,
611600
) -> Local {
612-
// FIXME: Analysis of the usage of the arguments to avoid
613-
// unnecessary temporaries.
614-
601+
// Reuse the operand if it is a moved temporary.
615602
if let Operand::Move(place) = &arg {
616603
if let Some(local) = place.as_local() {
617604
if caller_body.local_kind(local) == LocalKind::Temp {
618-
// Reuse the operand if it's a temporary already
619605
return local;
620606
}
621607
}
622608
}
623609

610+
// Otherwise, create a temporary for the argument.
624611
trace!("creating temp for argument {:?}", arg);
625-
// Otherwise, create a temporary for the arg
626-
let arg = Rvalue::Use(arg);
627-
628-
let ty = arg.ty(caller_body, self.tcx);
629-
630-
let arg_tmp = LocalDecl::new(ty, callsite.source_info.span);
631-
let arg_tmp = caller_body.local_decls.push(arg_tmp);
632-
633-
caller_body[callsite.bb].statements.push(Statement {
612+
let arg_ty = arg.ty(caller_body, self.tcx);
613+
let local = self.new_call_temp(caller_body, callsite, arg_ty);
614+
caller_body[callsite.block].statements.push(Statement {
634615
source_info: callsite.source_info,
635-
kind: StatementKind::StorageLive(arg_tmp),
616+
kind: StatementKind::Assign(box (Place::from(local), Rvalue::Use(arg))),
636617
});
637-
caller_body[callsite.bb].statements.push(Statement {
618+
local
619+
}
620+
621+
/// Introduces a new temporary into the caller body that is live for the duration of the call.
622+
fn new_call_temp(
623+
&self,
624+
caller_body: &mut Body<'tcx>,
625+
callsite: &CallSite<'tcx>,
626+
ty: Ty<'tcx>,
627+
) -> Local {
628+
let local = caller_body.local_decls.push(LocalDecl::new(ty, callsite.source_info.span));
629+
630+
caller_body[callsite.block].statements.push(Statement {
638631
source_info: callsite.source_info,
639-
kind: StatementKind::Assign(box (Place::from(arg_tmp), arg)),
632+
kind: StatementKind::StorageLive(local),
640633
});
641-
caller_body[return_block].statements.insert(
642-
0,
643-
Statement {
644-
source_info: callsite.source_info,
645-
kind: StatementKind::StorageDead(arg_tmp),
646-
},
647-
);
648-
649-
arg_tmp
634+
635+
if let Some(block) = callsite.target {
636+
caller_body[block].statements.insert(
637+
0,
638+
Statement {
639+
source_info: callsite.source_info,
640+
kind: StatementKind::StorageDead(local),
641+
},
642+
);
643+
}
644+
645+
local
650646
}
651647
}
652648

@@ -671,7 +667,7 @@ struct Integrator<'a, 'tcx> {
671667
new_scopes: RangeFrom<SourceScope>,
672668
new_blocks: RangeFrom<BasicBlock>,
673669
destination: Place<'tcx>,
674-
return_block: BasicBlock,
670+
return_block: Option<BasicBlock>,
675671
cleanup_block: Option<BasicBlock>,
676672
in_cleanup_block: bool,
677673
tcx: TyCtxt<'tcx>,
@@ -817,7 +813,11 @@ impl<'a, 'tcx> MutVisitor<'tcx> for Integrator<'a, 'tcx> {
817813
}
818814
}
819815
TerminatorKind::Return => {
820-
terminator.kind = TerminatorKind::Goto { target: self.return_block };
816+
terminator.kind = if let Some(tgt) = self.return_block {
817+
TerminatorKind::Goto { target: tgt }
818+
} else {
819+
TerminatorKind::Unreachable
820+
}
821821
}
822822
TerminatorKind::Resume => {
823823
if let Some(tgt) = self.cleanup_block {

‎src/test/mir-opt/inline/inline-diverging.rs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Tests inlining of diverging calls.
2+
//
3+
// ignore-wasm32-bare compiled with panic=abort by default
4+
#![crate_type = "lib"]
5+
6+
// EMIT_MIR inline_diverging.f.Inline.diff
7+
pub fn f() {
8+
sleep();
9+
}
10+
11+
// EMIT_MIR inline_diverging.g.Inline.diff
12+
pub fn g(i: i32) -> u32 {
13+
if i > 0 {
14+
i as u32
15+
} else {
16+
panic();
17+
}
18+
}
19+
20+
// EMIT_MIR inline_diverging.h.Inline.diff
21+
pub fn h() {
22+
call_twice(sleep);
23+
}
24+
25+
#[inline(always)]
26+
pub fn call_twice<R, F: Fn() -> R>(f: F) -> (R, R) {
27+
let a = f();
28+
let b = f();
29+
(a, b)
30+
}
31+
32+
#[inline(always)]
33+
fn panic() -> ! {
34+
panic!();
35+
}
36+
37+
#[inline(always)]
38+
fn sleep() -> ! {
39+
loop {}
40+
}

‎src/test/mir-opt/inline/inline_diverging.f.Inline.diff‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
- // MIR for `f` before Inline
2+
+ // MIR for `f` after Inline
3+
4+
fn f() -> () {
5+
let mut _0: (); // return place in scope 0 at $DIR/inline-diverging.rs:7:12: 7:12
6+
let mut _1: !; // in scope 0 at $DIR/inline-diverging.rs:7:12: 9:2
7+
let _2: !; // in scope 0 at $DIR/inline-diverging.rs:8:5: 8:12
8+
+ let mut _3: !; // in scope 0 at $DIR/inline-diverging.rs:8:5: 8:12
9+
+ scope 1 (inlined sleep) { // at $DIR/inline-diverging.rs:8:5: 8:12
10+
+ }
11+
12+
bb0: {
13+
StorageLive(_2); // scope 0 at $DIR/inline-diverging.rs:8:5: 8:12
14+
- sleep(); // scope 0 at $DIR/inline-diverging.rs:8:5: 8:12
15+
- // mir::Constant
16+
- // + span: $DIR/inline-diverging.rs:8:5: 8:10
17+
- // + literal: Const { ty: fn() -> ! {sleep}, val: Value(Scalar(<ZST>)) }
18+
+ StorageLive(_3); // scope 0 at $DIR/inline-diverging.rs:8:5: 8:12
19+
+ goto -> bb1; // scope 0 at $DIR/inline-diverging.rs:8:5: 8:12
20+
+ }
21+
+
22+
+ bb1: {
23+
+ goto -> bb1; // scope 1 at $DIR/inline-diverging.rs:8:5: 8:12
24+
}
25+
}
26+

‎src/test/mir-opt/inline/inline_diverging.g.Inline.diff‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
- // MIR for `g` before Inline
2+
+ // MIR for `g` after Inline
3+
4+
fn g(_1: i32) -> u32 {
5+
debug i => _1; // in scope 0 at $DIR/inline-diverging.rs:12:10: 12:11
6+
let mut _0: u32; // return place in scope 0 at $DIR/inline-diverging.rs:12:21: 12:24
7+
let mut _2: bool; // in scope 0 at $DIR/inline-diverging.rs:13:8: 13:13
8+
let mut _3: i32; // in scope 0 at $DIR/inline-diverging.rs:13:8: 13:9
9+
let mut _4: i32; // in scope 0 at $DIR/inline-diverging.rs:14:9: 14:10
10+
let mut _5: !; // in scope 0 at $DIR/inline-diverging.rs:15:12: 17:6
11+
let _6: !; // in scope 0 at $DIR/inline-diverging.rs:16:9: 16:16
12+
+ let mut _7: !; // in scope 0 at $DIR/inline-diverging.rs:16:9: 16:16
13+
+ scope 1 (inlined panic) { // at $DIR/inline-diverging.rs:16:9: 16:16
14+
+ }
15+
16+
bb0: {
17+
StorageLive(_2); // scope 0 at $DIR/inline-diverging.rs:13:8: 13:13
18+
StorageLive(_3); // scope 0 at $DIR/inline-diverging.rs:13:8: 13:9
19+
_3 = _1; // scope 0 at $DIR/inline-diverging.rs:13:8: 13:9
20+
_2 = Gt(move _3, const 0_i32); // scope 0 at $DIR/inline-diverging.rs:13:8: 13:13
21+
StorageDead(_3); // scope 0 at $DIR/inline-diverging.rs:13:12: 13:13
22+
switchInt(_2) -> [false: bb1, otherwise: bb2]; // scope 0 at $DIR/inline-diverging.rs:13:5: 17:6
23+
}
24+
25+
bb1: {
26+
StorageLive(_6); // scope 0 at $DIR/inline-diverging.rs:16:9: 16:16
27+
- panic(); // scope 0 at $DIR/inline-diverging.rs:16:9: 16:16
28+
+ StorageLive(_7); // scope 0 at $DIR/inline-diverging.rs:16:9: 16:16
29+
+ begin_panic::<&str>(const "explicit panic"); // scope 1 at $DIR/inline-diverging.rs:16:9: 16:16
30+
// mir::Constant
31+
- // + span: $DIR/inline-diverging.rs:16:9: 16:14
32+
- // + literal: Const { ty: fn() -> ! {panic}, val: Value(Scalar(<ZST>)) }
33+
+ // + span: $DIR/inline-diverging.rs:16:9: 16:16
34+
+ // + literal: Const { ty: fn(&str) -> ! {std::rt::begin_panic::<&str>}, val: Value(Scalar(<ZST>)) }
35+
+ // ty::Const
36+
+ // + ty: &str
37+
+ // + val: Value(Slice { data: Allocation { bytes: [101, 120, 112, 108, 105, 99, 105, 116, 32, 112, 97, 110, 105, 99], relocations: Relocations(SortedMap { data: [] }), init_mask: InitMask { blocks: [16383], len: Size { raw: 14 } }, size: Size { raw: 14 }, align: Align { pow2: 0 }, mutability: Not, extra: () }, start: 0, end: 14 })
38+
+ // mir::Constant
39+
+ // + span: $DIR/inline-diverging.rs:16:9: 16:16
40+
+ // + literal: Const { ty: &str, val: Value(Slice { data: Allocation { bytes: [101, 120, 112, 108, 105, 99, 105, 116, 32, 112, 97, 110, 105, 99], relocations: Relocations(SortedMap { data: [] }), init_mask: InitMask { blocks: [16383], len: Size { raw: 14 } }, size: Size { raw: 14 }, align: Align { pow2: 0 }, mutability: Not, extra: () }, start: 0, end: 14 }) }
41+
}
42+
43+
bb2: {
44+
StorageLive(_4); // scope 0 at $DIR/inline-diverging.rs:14:9: 14:10
45+
_4 = _1; // scope 0 at $DIR/inline-diverging.rs:14:9: 14:10
46+
_0 = move _4 as u32 (Misc); // scope 0 at $DIR/inline-diverging.rs:14:9: 14:17
47+
StorageDead(_4); // scope 0 at $DIR/inline-diverging.rs:14:16: 14:17
48+
StorageDead(_2); // scope 0 at $DIR/inline-diverging.rs:18:1: 18:2
49+
return; // scope 0 at $DIR/inline-diverging.rs:18:2: 18:2
50+
}
51+
}
52+

0 commit comments

Comments
 (0)