diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index f48dba9663a52..df5bf43725881 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -115,6 +115,7 @@ trait Inliner<'tcx> { /// Has the caller body been changed? fn changed(self) -> bool; + fn set_changed(&mut self); /// Should inlining happen for a given callee? fn should_inline_for_callee(&self, def_id: DefId) -> bool; @@ -187,6 +188,10 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> { self.changed } + fn set_changed(&mut self) { + self.changed = true; + } + fn should_inline_for_callee(&self, def_id: DefId) -> bool { ForceInline::should_run_pass_for_callee(self.tcx(), def_id) } @@ -334,6 +339,10 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> { self.changed } + fn set_changed(&mut self) { + self.changed = true; + } + fn should_inline_for_callee(&self, _: DefId) -> bool { true } @@ -529,10 +538,35 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>( let span = trace_span!("process_blocks", %callsite.callee, ?bb); let _guard = span.enter(); - match try_inlining(inliner, caller_body, &callsite) { + let mut unwind_unreachable = Err("did not reach analysis"); + match try_inlining(inliner, caller_body, &callsite, &mut unwind_unreachable) { Err(reason) => { debug!("not-inlined {} [{}]", callsite.callee, reason); inliner.on_inline_failure(&callsite, reason); + + match unwind_unreachable { + Ok(()) => { + if let Some(TerminatorKind::Call { unwind, .. }) = + caller_body[callsite.block].terminator.as_mut().map(|v| &mut v.kind) + { + inliner.set_changed(); + tracing::info!("marked {} unwind unreachable", callsite.callee); + *unwind = UnwindAction::Unreachable; + } else { + bug!( + "unexpected terminator: {:?}", + caller_body[callsite.block].terminator + ); + } + } + Err(reason) => { + tracing::info!( + "not marking unwind unreachable {}: {}", + callsite.callee, + reason + ); + } + } } Ok(new_blocks) => { debug!("inlined {}", callsite.callee); @@ -595,6 +629,54 @@ fn resolve_callsite<'tcx, I: Inliner<'tcx>>( None } +/// Ok indicates yes, Err(reason) otherwise. +fn should_mark_nounwind<'tcx>(_tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> Result<(), &'static str> { + // Unwinds can only start at certain terminators. + for block in body.basic_blocks.iter() { + let unwind = match block.terminator().kind { + // These never unwind. + TerminatorKind::Goto { .. } + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::UnwindTerminate(_) + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::CoroutineDrop + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } => continue, + + // Resume will *continue* unwinding, but if there's no other unwinding terminator it + // will never be reached. + TerminatorKind::UnwindResume => continue, + + TerminatorKind::Yield { .. } => { + return Err("impl limitation: yield"); + } + + TerminatorKind::Drop { unwind, .. } + | TerminatorKind::Call { unwind, .. } + | TerminatorKind::Assert { unwind, .. } => unwind, + + TerminatorKind::InlineAsm { .. } => return Err("inlineasm"), + + TerminatorKind::TailCall { .. } => { + return Err("impl limitation: tail call"); + } + }; + + match unwind { + UnwindAction::Continue => return Err("unwind: continue"), + // cannot unwind + UnwindAction::Unreachable => {} + // cannot unwind either -- will terminate instead + UnwindAction::Terminate(_) => {} + UnwindAction::Cleanup(_) => return Err("unwind: cleanup"), + } + } + + // If we didn't find an unwinding terminator, the function cannot unwind. + Ok(()) +} + /// Attempts to inline a callsite into the caller body. When successful returns basic blocks /// containing the inlined body. Otherwise returns an error describing why inlining didn't take /// place. @@ -602,10 +684,14 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>( inliner: &I, caller_body: &mut Body<'tcx>, callsite: &CallSite<'tcx>, + unwind: &mut Result<(), &'static str>, ) -> Result, &'static str> { let tcx = inliner.tcx(); check_mir_is_available(inliner, caller_body, callsite.callee)?; + let callee_body = try_instance_mir(tcx, callsite.callee.def)?; + *unwind = should_mark_nounwind(tcx, callee_body); + let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id()); check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?; check_codegen_attributes(inliner, callsite, callee_attrs)?; @@ -622,7 +708,6 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>( } } - let callee_body = try_instance_mir(tcx, callsite.callee.def)?; check_inline::is_inline_valid_on_body(tcx, callee_body)?; inliner.check_callee_mir_body(callsite, callee_body, callee_attrs)?; diff --git a/tests/codegen/drop.rs b/tests/codegen/drop.rs index b22a8ef27d239..fd5640cb699df 100644 --- a/tests/codegen/drop.rs +++ b/tests/codegen/drop.rs @@ -11,7 +11,9 @@ impl Drop for SomeUniqueName { } #[inline(never)] -pub fn possibly_unwinding() {} +pub fn possibly_unwinding() { + panic!(); +} // CHECK-LABEL: @droppy #[no_mangle] diff --git a/tests/codegen/mem-replace-big-type.rs b/tests/codegen/mem-replace-big-type.rs index 0b2229ba7d104..b28b505b4e5c9 100644 --- a/tests/codegen/mem-replace-big-type.rs +++ b/tests/codegen/mem-replace-big-type.rs @@ -25,7 +25,7 @@ pub fn replace_big(dst: &mut Big, src: Big) -> Big { // CHECK-NOT: call void @llvm.memcpy // For a large type, we expect exactly three `memcpy`s -// CHECK-LABEL: define internal void @{{.+}}mem{{.+}}replace{{.+}}(ptr +// CHECK-LABEL: define void @{{.+}}mem{{.+}}replace{{.+}}(ptr // CHECK-SAME: sret([56 x i8]){{.+}}[[RESULT:%.+]], ptr{{.+}}%dest, ptr{{.+}}%src) // CHECK-NOT: call void @llvm.memcpy // CHECK: call void @llvm.memcpy.{{.+}}(ptr align 8 [[RESULT]], ptr align 8 %dest, i{{.*}} 56, i1 false) diff --git a/tests/codegen/personality_lifetimes.rs b/tests/codegen/personality_lifetimes.rs index cd81db6395349..0f922ad9c0b9a 100644 --- a/tests/codegen/personality_lifetimes.rs +++ b/tests/codegen/personality_lifetimes.rs @@ -13,7 +13,9 @@ impl Drop for S { } #[inline(never)] -fn might_unwind() {} +fn might_unwind() { + panic!(); +} // CHECK-LABEL: @test #[no_mangle] diff --git a/tests/mir-opt/inline/caller_with_trivial_bound.foo.Inline.panic-unwind.diff b/tests/mir-opt/inline/caller_with_trivial_bound.foo.Inline.panic-unwind.diff index d4427b2a8076e..1c6385b4905ad 100644 --- a/tests/mir-opt/inline/caller_with_trivial_bound.foo.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/caller_with_trivial_bound.foo.Inline.panic-unwind.diff @@ -10,7 +10,8 @@ bb0: { StorageLive(_1); - _1 = bar::() -> [return: bb1, unwind continue]; +- _1 = bar::() -> [return: bb1, unwind continue]; ++ _1 = bar::() -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff b/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff index 3f334779e18f8..ae36ab405781f 100644 --- a/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/exponential_runtime.main.Inline.panic-unwind.diff @@ -56,7 +56,7 @@ + StorageLive(_17); + StorageLive(_18); + StorageLive(_19); -+ _17 = <() as A>::call() -> [return: bb12, unwind continue]; ++ _17 = <() as A>::call() -> [return: bb12, unwind unreachable]; } bb1: { @@ -124,11 +124,11 @@ + } + + bb12: { -+ _18 = <() as A>::call() -> [return: bb13, unwind continue]; ++ _18 = <() as A>::call() -> [return: bb13, unwind unreachable]; + } + + bb13: { -+ _19 = <() as A>::call() -> [return: bb11, unwind continue]; ++ _19 = <() as A>::call() -> [return: bb11, unwind unreachable]; } } diff --git a/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir b/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir index df0cab513eca7..3218c4e567378 100644 --- a/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir +++ b/tests/mir-opt/inline/inline_options.main.Inline.after.panic-unwind.mir @@ -12,7 +12,7 @@ fn main() -> () { bb0: { StorageLive(_1); - _1 = not_inlined() -> [return: bb1, unwind continue]; + _1 = not_inlined() -> [return: bb1, unwind unreachable]; } bb1: { @@ -21,7 +21,7 @@ fn main() -> () { StorageLive(_3); StorageLive(_4); StorageLive(_5); - _3 = g() -> [return: bb3, unwind continue]; + _3 = g() -> [return: bb3, unwind unreachable]; } bb2: { @@ -34,10 +34,10 @@ fn main() -> () { } bb3: { - _4 = g() -> [return: bb4, unwind continue]; + _4 = g() -> [return: bb4, unwind unreachable]; } bb4: { - _5 = g() -> [return: bb2, unwind continue]; + _5 = g() -> [return: bb2, unwind unreachable]; } } diff --git a/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff b/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff index 834851b75adf1..60dbbf552d98a 100644 --- a/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/issue_106141.outer.Inline.panic-unwind.diff @@ -20,7 +20,7 @@ + StorageLive(_1); + StorageLive(_2); + _1 = const inner::promoted[0]; -+ _0 = index() -> [return: bb1, unwind continue]; ++ _0 = index() -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/inline/rustc_no_mir_inline.caller.Inline.panic-unwind.diff b/tests/mir-opt/inline/rustc_no_mir_inline.caller.Inline.panic-unwind.diff index 4506a338edd7a..5689446ba5aac 100644 --- a/tests/mir-opt/inline/rustc_no_mir_inline.caller.Inline.panic-unwind.diff +++ b/tests/mir-opt/inline/rustc_no_mir_inline.caller.Inline.panic-unwind.diff @@ -7,7 +7,8 @@ bb0: { StorageLive(_1); - _1 = callee() -> [return: bb1, unwind continue]; +- _1 = callee() -> [return: bb1, unwind continue]; ++ _1 = callee() -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/inline/rustc_no_mir_inline.caller.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/inline/rustc_no_mir_inline.caller.PreCodegen.after.panic-unwind.mir index 39ad4f1010b79..d0772e51a07fe 100644 --- a/tests/mir-opt/inline/rustc_no_mir_inline.caller.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/inline/rustc_no_mir_inline.caller.PreCodegen.after.panic-unwind.mir @@ -5,7 +5,7 @@ fn caller() -> () { let _1: (); bb0: { - _1 = callee() -> [return: bb1, unwind continue]; + _1 = callee() -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/inline/unsized_argument.caller.Inline.diff b/tests/mir-opt/inline/unsized_argument.caller.Inline.diff index 644d6d320de04..57a9750e906ed 100644 --- a/tests/mir-opt/inline/unsized_argument.caller.Inline.diff +++ b/tests/mir-opt/inline/unsized_argument.caller.Inline.diff @@ -13,11 +13,13 @@ StorageLive(_3); _3 = move _1; _4 = copy ((_3.0: std::ptr::Unique<[i32]>).0: std::ptr::NonNull<[i32]>) as *const [i32] (Transmute); - _2 = callee(move (*_4)) -> [return: bb1, unwind: bb3]; +- _2 = callee(move (*_4)) -> [return: bb1, unwind: bb3]; ++ _2 = callee(move (*_4)) -> [return: bb1, unwind unreachable]; } bb1: { - drop(_3) -> [return: bb2, unwind: bb4]; +- drop(_3) -> [return: bb2, unwind: bb4]; ++ drop(_3) -> [return: bb2, unwind: bb3]; } bb2: { @@ -28,10 +30,10 @@ } bb3 (cleanup): { - drop(_3) -> [return: bb4, unwind terminate(cleanup)]; - } - - bb4 (cleanup): { +- drop(_3) -> [return: bb4, unwind terminate(cleanup)]; +- } +- +- bb4 (cleanup): { resume; } } diff --git a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff index d81bfa9310bc1..993857f225a81 100644 --- a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.GVN.panic-unwind.diff @@ -20,7 +20,7 @@ + nop; + _6 = copy (*_1); _2 = copy (*_6); - _3 = unknown() -> [return: bb1, unwind continue]; + _3 = unknown() -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.PreCodegen.after.panic-unwind.mir index 4c01e9464bf49..23b1c3f3f43ad 100644 --- a/tests/mir-opt/pre-codegen/deref_nested_borrows.src.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/deref_nested_borrows.src.PreCodegen.after.panic-unwind.mir @@ -15,7 +15,7 @@ fn src(_1: &&u8) -> bool { bb0: { _2 = copy (*_1); _3 = copy (*_2); - _4 = unknown() -> [return: bb1, unwind continue]; + _4 = unknown() -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff index 0c52f1e058367..69507ad13ac48 100644 --- a/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff +++ b/tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.main.GVN.64bit.panic-unwind.diff @@ -76,9 +76,9 @@ _7 = copy _9; StorageLive(_8); - _8 = copy _1; -- _6 = std::alloc::Global::alloc_impl(move _7, move _8, const false) -> [return: bb5, unwind continue]; +- _6 = std::alloc::Global::alloc_impl(move _7, move _8, const false) -> [return: bb5, unwind unreachable]; + _8 = const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(8 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x0000000000000000): std::ptr::alignment::AlignmentEnum) }}; -+ _6 = std::alloc::Global::alloc_impl(copy _9, const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(8 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x0000000000000000): std::ptr::alignment::AlignmentEnum) }}, const false) -> [return: bb5, unwind continue]; ++ _6 = std::alloc::Global::alloc_impl(copy _9, const Layout {{ size: Indirect { alloc_id: ALLOC0, offset: Size(8 bytes) }: usize, align: std::ptr::Alignment(Scalar(0x0000000000000000): std::ptr::alignment::AlignmentEnum) }}, const false) -> [return: bb5, unwind unreachable]; } bb5: { diff --git a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir index e537dd6a28ef8..02544c050dc04 100644 --- a/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir +++ b/tests/mir-opt/pre-codegen/loops.vec_move.PreCodegen.after.mir @@ -19,7 +19,7 @@ fn vec_move(_1: Vec) -> () { bb0: { StorageLive(_2); - _2 = as IntoIterator>::into_iter(move _1) -> [return: bb1, unwind continue]; + _2 = as IntoIterator>::into_iter(move _1) -> [return: bb1, unwind unreachable]; } bb1: { @@ -31,12 +31,12 @@ fn vec_move(_1: Vec) -> () { bb2: { StorageLive(_5); _4 = &mut _3; - _5 = as Iterator>::next(move _4) -> [return: bb3, unwind: bb9]; + _5 = as Iterator>::next(move _4) -> [return: bb3, unwind unreachable]; } bb3: { _6 = discriminant(_5); - switchInt(move _6) -> [0: bb4, 1: bb6, otherwise: bb8]; + switchInt(move _6) -> [0: bb4, 1: bb6, otherwise: bb10]; } bb4: { @@ -52,7 +52,7 @@ fn vec_move(_1: Vec) -> () { bb6: { _7 = move ((_5 as Some).0: impl Sized); - _8 = opaque::(move _7) -> [return: bb7, unwind: bb9]; + _8 = opaque::(move _7) -> [return: bb7, unwind: bb8]; } bb7: { @@ -60,15 +60,15 @@ fn vec_move(_1: Vec) -> () { goto -> bb2; } - bb8: { - unreachable; + bb8 (cleanup): { + drop(_3) -> [return: bb9, unwind terminate(cleanup)]; } bb9 (cleanup): { - drop(_3) -> [return: bb10, unwind terminate(cleanup)]; + resume; } - bb10 (cleanup): { - resume; + bb10: { + unreachable; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir index 12b54b57b8448..c1b3148dcdf26 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.reverse_loop.PreCodegen.after.panic-unwind.mir @@ -112,13 +112,13 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { StorageLive(_15); StorageLive(_14); _14 = &mut (_13.0: std::slice::Iter<'_, T>); - _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind: bb11]; + _15 = as DoubleEndedIterator>::next_back(move _14) -> [return: bb5, unwind unreachable]; } bb5: { StorageDead(_14); _16 = discriminant(_15); - switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb10]; + switchInt(move _16) -> [0: bb6, 1: bb8, otherwise: bb12]; } bb6: { @@ -137,7 +137,7 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { _18 = &_2; StorageLive(_19); _19 = (copy _17,); - _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb11]; + _20 = >::call(move _18, move _19) -> [return: bb9, unwind: bb10]; } bb9: { @@ -147,15 +147,15 @@ fn reverse_loop(_1: &[T], _2: impl Fn(&T)) -> () { goto -> bb4; } - bb10: { - unreachable; + bb10 (cleanup): { + drop(_2) -> [return: bb11, unwind terminate(cleanup)]; } bb11 (cleanup): { - drop(_2) -> [return: bb12, unwind terminate(cleanup)]; + resume; } - bb12 (cleanup): { - resume; + bb12: { + unreachable; } } diff --git a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir index dfe5e206fadaf..78f96bf419559 100644 --- a/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/pre-codegen/slice_iter.slice_iter_mut_next_back.PreCodegen.after.panic-unwind.mir @@ -5,7 +5,7 @@ fn slice_iter_mut_next_back(_1: &mut std::slice::IterMut<'_, T>) -> Option<&mut let mut _0: std::option::Option<&mut T>; bb0: { - _0 = as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind continue]; + _0 = as DoubleEndedIterator>::next_back(move _1) -> [return: bb1, unwind unreachable]; } bb1: { diff --git a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff index 6c346e20e5893..c3076fb67c231 100644 --- a/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff +++ b/tests/mir-opt/simplify_if.main.SimplifyConstCondition-after-const-prop.panic-unwind.diff @@ -11,7 +11,7 @@ } bb1: { - _1 = noop() -> [return: bb2, unwind continue]; + _1 = noop() -> [return: bb2, unwind unreachable]; } bb2: { diff --git a/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff b/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff index e243ff45ab0b2..9e798cbcac0c1 100644 --- a/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff +++ b/tests/mir-opt/simplify_match.main.GVN.panic-unwind.diff @@ -23,7 +23,7 @@ } bb1: { - _0 = noop() -> [return: bb2, unwind continue]; + _0 = noop() -> [return: bb2, unwind unreachable]; } bb2: {