EXPERIMENT: Use the force-inline pass for more stuff

compiler/rustc_attr_parsing/src/attributes/inline.rs

@@ -93,3 +93,16 @@ impl<S: Stage> SingleAttributeParser<S> for RustcForceInlineParser {
         ))
     }
 }
+
+pub(crate) struct RustcEarlyInlineParser;
+
+impl<S: Stage> SingleAttributeParser<S> for RustcEarlyInlineParser {
+    const PATH: &'static [Symbol] = &[sym::rustc_early_inline];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
+    const TEMPLATE: AttributeTemplate = template!(Word);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
+        Some(AttributeKind::Inline(InlineAttr::Early, cx.attr_span))
+    }
+}

compiler/rustc_attr_parsing/src/context.rs

@@ -24,7 +24,7 @@ use crate::attributes::codegen_attrs::{
 use crate::attributes::confusables::ConfusablesParser;
 use crate::attributes::deprecation::DeprecationParser;
 use crate::attributes::dummy::DummyParser;
-use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
+use crate::attributes::inline::{InlineParser, RustcEarlyInlineParser, RustcForceInlineParser};
 use crate::attributes::link_attrs::{
     ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser,
     LinkSectionParser, StdInternalSymbolParser,
@@ -170,6 +170,7 @@ attribute_parsers!(
         Single<PathAttributeParser>,
         Single<ProcMacroDeriveParser>,
         Single<RustcBuiltinMacroParser>,
+        Single<RustcEarlyInlineParser>,
         Single<RustcForceInlineParser>,
         Single<RustcLayoutScalarValidRangeEnd>,
         Single<RustcLayoutScalarValidRangeStart>,

compiler/rustc_codegen_gcc/src/attributes.rs

@@ -67,7 +67,7 @@ fn inline_attr<'gcc, 'tcx>(
                 Some(FnAttribute::AlwaysInline)
             }
         }
-        InlineAttr::Hint => Some(FnAttribute::Inline),
+        InlineAttr::Hint | InlineAttr::Early => Some(FnAttribute::Inline),
         InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -52,7 +52,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
         return Some(AttributeKind::NoInline.create_attr(cx.llcx));
     }
     match inline {
-        InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
+        InlineAttr::Hint | InlineAttr::Early => {
+            Some(AttributeKind::InlineHint.create_attr(cx.llcx))
+        }
         InlineAttr::Always | InlineAttr::Force { .. } => {
             Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
         }

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -473,7 +473,7 @@ fn check_result(
 
     // warn that inline has no effect when no_sanitize is present
     if !codegen_fn_attrs.no_sanitize.is_empty()
-        && codegen_fn_attrs.inline.always()
+        && codegen_fn_attrs.inline.always_in_codegen()
         && let (Some(no_sanitize_span), Some(inline_span)) =
             (interesting_spans.no_sanitize, interesting_spans.inline)
     {

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1123,6 +1123,11 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_no_mir_inline]` prevents the MIR inliner from inlining a function while not affecting codegen"
     ),
+    rustc_attr!(
+        rustc_early_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
+        "`#[rustc_early_inline]` inlines non-generic calls to trivial methods even in debug, \
+         while still allowing them to be codegen'd for generic calls"
+    ),
     rustc_attr!(
         rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_force_inline]` forces a free function to be inlined"

compiler/rustc_hir/src/attrs/data_structures.rs

@@ -24,13 +24,20 @@ pub enum InlineAttr {
         attr_span: Span,
         reason: Option<Symbol>,
     },
+    /// `#[rustc_early_inline]` will always inline calls to a known impl in MIR.
+    ///
+    /// You can think of this as either
+    /// - Force, but without the "do not codegen as a function ever" restriction.
+    /// - Always, but only for MIR.
+    Early,
 }
 
 impl InlineAttr {
-    pub fn always(&self) -> bool {
+    pub fn always_in_codegen(&self) -> bool {
         match self {
-            InlineAttr::Always | InlineAttr::Force { .. } => true,
-            InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
+            InlineAttr::Always => true,
+            InlineAttr::None | InlineAttr::Hint | InlineAttr::Early | InlineAttr::Never => false,
+            InlineAttr::Force { .. } => panic!("Shouldn't be codegen'ing {self:?}"),
         }
     }
 }
@@ -342,7 +349,7 @@ pub enum AttributeKind {
         reason: Option<Symbol>,
     },
 
-    /// Represents `#[inline]` and `#[rustc_force_inline]`.
+    /// Represents `#[inline]` and `#[rustc_force_inline]` and `#[rustc_early_inline]`.
     Inline(InlineAttr, Span),
 
     /// Represents `#[link_name]`.

compiler/rustc_middle/src/mir/mono.rs

@@ -205,7 +205,7 @@ impl<'tcx> MonoItem<'tcx> {
 
         // To ensure that #[inline(always)] can be inlined as much as possible, especially in unoptimized
         // builds, we always select LocalCopy.
-        if codegen_fn_attrs.inline.always() {
+        if codegen_fn_attrs.inline.always_in_codegen() {
             return InstantiationMode::LocalCopy;
         }
 

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -46,8 +46,10 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
-        _ => {}
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Early | InlineAttr::Force { .. } => {
+            return true;
+        }
+        InlineAttr::None => {}
     }
 
     // If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until

compiler/rustc_mir_transform/src/inline.rs

@@ -77,7 +77,7 @@ pub struct ForceInline;
 
 impl ForceInline {
     pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
-        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
+        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. } | InlineAttr::Early)
     }
 }
 
@@ -195,7 +195,7 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
         &self,
         callee_attrs: &CodegenFnAttrs,
     ) -> Result<(), &'static str> {
-        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
+        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. } | InlineAttr::Early);
         Ok(())
     }
 
@@ -247,23 +247,26 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
 
     fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
         let tcx = self.tcx();
-        let InlineAttr::Force { attr_span, reason: justification } =
-            tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
-        else {
-            bug!("called on item without required inlining");
-        };
-
-        let call_span = callsite.source_info.span;
-        tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
-            call_span,
-            attr_span,
-            caller_span: tcx.def_span(self.def_id),
-            caller: tcx.def_path_str(self.def_id),
-            callee_span: tcx.def_span(callsite.callee.def_id()),
-            callee: tcx.def_path_str(callsite.callee.def_id()),
-            reason,
-            justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
-        });
+        match tcx.codegen_fn_attrs(callsite.callee.def_id()).inline {
+            InlineAttr::Early => {
+                // Ok, we don't actually mind if this fails.
+            }
+            InlineAttr::Force { attr_span, reason: justification } => {
+                let call_span = callsite.source_info.span;
+                tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
+                    call_span,
+                    attr_span,
+                    caller_span: tcx.def_span(self.def_id),
+                    caller: tcx.def_path_str(self.def_id),
+                    callee_span: tcx.def_span(callsite.callee.def_id()),
+                    callee: tcx.def_path_str(callsite.callee.def_id()),
+                    reason,
+                    justification: justification
+                        .map(|sym| crate::errors::ForceInlineJustification { sym }),
+                });
+            }
+            _ => bug!("called on item without required inlining"),
+        }
     }
 }
 

compiler/rustc_span/src/symbol.rs

@@ -1848,6 +1848,7 @@ symbols! {
         rustc_dump_predicates,
         rustc_dump_user_args,
         rustc_dump_vtable,
+        rustc_early_inline,
         rustc_effective_visibility,
         rustc_evaluate_where_clauses,
         rustc_expected_cgu_reuse,

library/core/src/num/int_macros.rs

@@ -225,7 +225,7 @@ macro_rules! int_impl {
         #[rustc_const_stable(feature = "integer_sign_cast", since = "1.87.0")]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[inline(always)]
+        #[rustc_early_inline]
         pub const fn cast_unsigned(self) -> $UnsignedT {
             self as $UnsignedT
         }
@@ -1915,7 +1915,7 @@ macro_rules! int_impl {
         #[rustc_const_stable(feature = "const_int_methods", since = "1.32.0")]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[inline(always)]
+        #[rustc_early_inline]
         pub const fn wrapping_add(self, rhs: Self) -> Self {
             intrinsics::wrapping_add(self, rhs)
         }

library/core/src/num/uint_macros.rs

@@ -277,7 +277,7 @@ macro_rules! uint_impl {
         #[rustc_const_stable(feature = "integer_sign_cast", since = "1.87.0")]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[inline(always)]
+        #[rustc_early_inline]
         pub const fn cast_signed(self) -> $SignedT {
             self as $SignedT
         }
@@ -2070,7 +2070,7 @@ macro_rules! uint_impl {
         #[rustc_const_stable(feature = "const_wrapping_math", since = "1.32.0")]
         #[must_use = "this returns the result of the operation, \
                       without modifying the original"]
-        #[inline(always)]
+        #[rustc_early_inline]
         pub const fn wrapping_add(self, rhs: Self) -> Self {
             intrinsics::wrapping_add(self, rhs)
         }

tests/mir-opt/inline/early_inline.call_early.ForceInline.panic-abort.diff

@@ -0,0 +1,32 @@
+- // MIR for `call_early` before ForceInline
++ // MIR for `call_early` after ForceInline
+
+  fn call_early(_1: i32) -> i32 {
+      debug x => _1;
+      let mut _0: i32;
+      let mut _2: i32;
++     let mut _3: i32;
++     scope 1 (inlined do_early) {
++         let mut _4: (i32, bool);
++     }
+
+      bb0: {
+          StorageLive(_2);
+          _2 = copy _1;
+-         _0 = do_early(move _2, const 42_i32) -> [return: bb1, unwind unreachable];
++         StorageLive(_3);
++         _3 = const 42_i32;
++         StorageLive(_4);
++         _4 = AddWithOverflow(copy _2, copy _3);
++         assert(!move (_4.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _2, copy _3) -> [success: bb1, unwind unreachable];
+      }
+
+      bb1: {
++         _0 = move (_4.0: i32);
++         StorageDead(_4);
++         StorageDead(_3);
+          StorageDead(_2);
+          return;
+      }
+  }
+

tests/mir-opt/inline/early_inline.call_early.ForceInline.panic-unwind.diff

@@ -0,0 +1,32 @@
+- // MIR for `call_early` before ForceInline
++ // MIR for `call_early` after ForceInline
+
+  fn call_early(_1: i32) -> i32 {
+      debug x => _1;
+      let mut _0: i32;
+      let mut _2: i32;
++     let mut _3: i32;
++     scope 1 (inlined do_early) {
++         let mut _4: (i32, bool);
++     }
+
+      bb0: {
+          StorageLive(_2);
+          _2 = copy _1;
+-         _0 = do_early(move _2, const 42_i32) -> [return: bb1, unwind continue];
++         StorageLive(_3);
++         _3 = const 42_i32;
++         StorageLive(_4);
++         _4 = AddWithOverflow(copy _2, copy _3);
++         assert(!move (_4.1: bool), "attempt to compute `{} + {}`, which would overflow", copy _2, copy _3) -> [success: bb1, unwind continue];
+      }
+
+      bb1: {
++         _0 = move (_4.0: i32);
++         StorageDead(_4);
++         StorageDead(_3);
+          StorageDead(_2);
+          return;
+      }
+  }
+

tests/mir-opt/inline/early_inline.early_as_fn.ForceInline.panic-abort.diff

@@ -0,0 +1,12 @@
+- // MIR for `early_as_fn` before ForceInline
++ // MIR for `early_as_fn` after ForceInline
+
+  fn early_as_fn() -> fn(i32, i32) -> i32 {
+      let mut _0: fn(i32, i32) -> i32;
+
+      bb0: {
+          _0 = do_early as fn(i32, i32) -> i32 (PointerCoercion(ReifyFnPointer, Implicit));
+          return;
+      }
+  }
+

tests/mir-opt/inline/early_inline.early_as_fn.ForceInline.panic-unwind.diff

@@ -0,0 +1,12 @@
+- // MIR for `early_as_fn` before ForceInline
++ // MIR for `early_as_fn` after ForceInline
+
+  fn early_as_fn() -> fn(i32, i32) -> i32 {
+      let mut _0: fn(i32, i32) -> i32;
+
+      bb0: {
+          _0 = do_early as fn(i32, i32) -> i32 (PointerCoercion(ReifyFnPointer, Implicit));
+          return;
+      }
+  }
+

tests/mir-opt/inline/early_inline.rs

@@ -0,0 +1,32 @@
+//@ compile-flags: -Copt-level=0 -Zmir-opt-level=1 -Cdebuginfo=limited
+// EMIT_MIR_FOR_EACH_PANIC_STRATEGY
+
+#![feature(rustc_attrs)]
+
+#[rustc_early_inline]
+fn do_early(x: i32, y: i32) -> i32 {
+    x + y
+}
+
+// EMIT_MIR early_inline.early_as_fn.ForceInline.diff
+fn early_as_fn() -> fn(i32, i32) -> i32 {
+    // CHECK-LABEL: fn early_as_fn() -> fn(i32, i32) -> i32
+    // CHECK: _0 = do_early as fn(i32, i32) -> i32 (PointerCoercion(ReifyFnPointer, Implicit));
+    do_early
+}
+
+// EMIT_MIR early_inline.call_early.ForceInline.diff
+fn call_early(x: i32) -> i32 {
+    // CHECK-LABEL: fn call_early(_1: i32) -> i32
+    // CHECK: (inlined do_early)
+    // CHECK: _2 = const 42_i32;
+    // CHECK: _3 = AddWithOverflow(copy _1, copy _2);
+    // CHECK: _0 = move (_3.0: i32);
+    do_early(x, 42)
+}
+
+fn main() {
+    let f = early_as_fn();
+    let _z = f(1, 2);
+    call_early(7);
+}