Auto merge of #144483 - scottmcm:early-inline, r=<try>

EXPERIMENT: Use the force-inline pass for more stuff

Author: rust-bors[bot]

compiler/rustc_attr_data_structures/src/attributes.rs

@@ -21,13 +21,20 @@ pub enum InlineAttr {
         attr_span: Span,
         reason: Option<Symbol>,
     },
+    /// `#[rustc_early_inline]` will always inline calls to a known impl in MIR.
+    ///
+    /// You can think of this as either
+    /// - Force, but without the "do not codegen as a function ever" restriction.
+    /// - Always, but only for MIR.
+    Early,
 }
 
 impl InlineAttr {
-    pub fn always(&self) -> bool {
+    pub fn always_in_codegen(&self) -> bool {
         match self {
-            InlineAttr::Always | InlineAttr::Force { .. } => true,
-            InlineAttr::None | InlineAttr::Hint | InlineAttr::Never => false,
+            InlineAttr::Always => true,
+            InlineAttr::None | InlineAttr::Hint | InlineAttr::Early | InlineAttr::Never => false,
+            InlineAttr::Force { .. } => panic!("Shouldn't be codegen'ing {self:?}"),
         }
     }
 }
@@ -349,7 +356,7 @@ pub enum AttributeKind {
         reason: Option<Symbol>,
     },
 
-    /// Represents `#[inline]` and `#[rustc_force_inline]`.
+    /// Represents `#[inline]` and `#[rustc_force_inline]` and `#[rustc_early_inline]`.
     Inline(InlineAttr, Span),
 
     /// Represents `#[link_name]`.

compiler/rustc_attr_parsing/src/attributes/inline.rs

@@ -93,3 +93,16 @@ impl<S: Stage> SingleAttributeParser<S> for RustcForceInlineParser {
         ))
     }
 }
+
+pub(crate) struct RustcEarlyInlineParser;
+
+impl<S: Stage> SingleAttributeParser<S> for RustcEarlyInlineParser {
+    const PATH: &'static [Symbol] = &[sym::rustc_early_inline];
+    const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepOutermost;
+    const ON_DUPLICATE: OnDuplicate<S> = OnDuplicate::WarnButFutureError;
+    const TEMPLATE: AttributeTemplate = template!(Word);
+
+    fn convert(cx: &mut AcceptContext<'_, '_, S>, _args: &ArgParser<'_>) -> Option<AttributeKind> {
+        Some(AttributeKind::Inline(InlineAttr::Early, cx.attr_span))
+    }
+}

compiler/rustc_attr_parsing/src/context.rs

@@ -24,7 +24,7 @@ use crate::attributes::codegen_attrs::{
 use crate::attributes::confusables::ConfusablesParser;
 use crate::attributes::deprecation::DeprecationParser;
 use crate::attributes::dummy::DummyParser;
-use crate::attributes::inline::{InlineParser, RustcForceInlineParser};
+use crate::attributes::inline::{InlineParser, RustcEarlyInlineParser, RustcForceInlineParser};
 use crate::attributes::link_attrs::{
     ExportStableParser, FfiConstParser, FfiPureParser, LinkNameParser, LinkOrdinalParser,
     LinkSectionParser, StdInternalSymbolParser,
@@ -154,6 +154,7 @@ attribute_parsers!(
         Single<MustUseParser>,
         Single<OptimizeParser>,
         Single<PathAttributeParser>,
+        Single<RustcEarlyInlineParser>,
         Single<RustcForceInlineParser>,
         Single<RustcLayoutScalarValidRangeEnd>,
         Single<RustcLayoutScalarValidRangeStart>,

compiler/rustc_codegen_gcc/src/attributes.rs

@@ -67,7 +67,7 @@ fn inline_attr<'gcc, 'tcx>(
                 Some(FnAttribute::AlwaysInline)
             }
         }
-        InlineAttr::Hint => Some(FnAttribute::Inline),
+        InlineAttr::Hint | InlineAttr::Early => Some(FnAttribute::Inline),
         InlineAttr::Force { .. } => Some(FnAttribute::AlwaysInline),
         InlineAttr::Never => {
             if cx.sess().target.arch != "amdgpu" {

compiler/rustc_codegen_llvm/src/attributes.rs

@@ -52,7 +52,9 @@ fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll
         return Some(AttributeKind::NoInline.create_attr(cx.llcx));
     }
     match inline {
-        InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)),
+        InlineAttr::Hint | InlineAttr::Early => {
+            Some(AttributeKind::InlineHint.create_attr(cx.llcx))
+        }
         InlineAttr::Always | InlineAttr::Force { .. } => {
             Some(AttributeKind::AlwaysInline.create_attr(cx.llcx))
         }

compiler/rustc_codegen_ssa/src/codegen_attrs.rs

@@ -452,7 +452,7 @@ fn codegen_fn_attrs(tcx: TyCtxt<'_>, did: LocalDefId) -> CodegenFnAttrs {
     }
 
     if !codegen_fn_attrs.no_sanitize.is_empty()
-        && codegen_fn_attrs.inline.always()
+        && codegen_fn_attrs.inline.always_in_codegen()
         && let (Some(no_sanitize_span), Some(inline_span)) = (no_sanitize_span, inline_span)
     {
         let hir_id = tcx.local_def_id_to_hir_id(did);

compiler/rustc_feature/src/builtin_attrs.rs

@@ -1123,6 +1123,11 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[
         rustc_no_mir_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_no_mir_inline]` prevents the MIR inliner from inlining a function while not affecting codegen"
     ),
+    rustc_attr!(
+        rustc_early_inline, Normal, template!(Word), WarnFollowing, EncodeCrossCrate::Yes,
+        "`#[rustc_early_inline]` inlines non-generic calls to trivial methods even in debug, \
+         while still allowing them to be codegen'd for generic calls"
+    ),
     rustc_attr!(
         rustc_force_inline, Normal, template!(Word, NameValueStr: "reason"), WarnFollowing, EncodeCrossCrate::Yes,
         "`#[rustc_force_inline]` forces a free function to be inlined"

compiler/rustc_middle/src/mir/mono.rs

@@ -205,7 +205,7 @@ impl<'tcx> MonoItem<'tcx> {
 
         // To ensure that #[inline(always)] can be inlined as much as possible, especially in unoptimized
         // builds, we always select LocalCopy.
-        if codegen_fn_attrs.inline.always() {
+        if codegen_fn_attrs.inline.always_in_codegen() {
             return InstantiationMode::LocalCopy;
         }
 

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -46,8 +46,10 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // #[inline(never)] to force code generation.
     match codegen_fn_attrs.inline {
         InlineAttr::Never => return false,
-        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Force { .. } => return true,
-        _ => {}
+        InlineAttr::Hint | InlineAttr::Always | InlineAttr::Early | InlineAttr::Force { .. } => {
+            return true;
+        }
+        InlineAttr::None => {}
     }
 
     // If the crate is likely to be mostly unused, use cross-crate inlining to defer codegen until
@@ -77,7 +79,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
     // enabled. This ensures that we do inference even if someone only passes -Zinline-mir,
     // which is less confusing than having to also enable -Copt-level=1.
     let inliner_will_run = pm::should_run_pass(tcx, &inline::Inline, pm::Optimizations::Allowed)
-        || inline::ForceInline::should_run_pass_for_callee(tcx, def_id.to_def_id());
+        || inline::ForceInline::applies_for_resolved_callee(tcx, def_id.to_def_id());
     if matches!(tcx.sess.opts.optimize, OptLevel::No) && !inliner_will_run {
         return false;
     }

compiler/rustc_mir_transform/src/inline.rs

@@ -76,9 +76,13 @@ impl<'tcx> crate::MirPass<'tcx> for Inline {
 pub struct ForceInline;
 
 impl ForceInline {
-    pub fn should_run_pass_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
+    pub fn needs_callgraph_for_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
         matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. })
     }
+
+    pub fn applies_for_resolved_callee<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> bool {
+        matches!(tcx.codegen_fn_attrs(def_id).inline, InlineAttr::Force { .. } | InlineAttr::Early)
+    }
 }
 
 impl<'tcx> crate::MirPass<'tcx> for ForceInline {
@@ -117,7 +121,7 @@ trait Inliner<'tcx> {
     fn changed(self) -> bool;
 
     /// Should inlining happen for a given callee?
-    fn should_inline_for_callee(&self, def_id: DefId) -> bool;
+    fn should_inline_for_callee(&self, instance_kind: &InstanceKind<'tcx>) -> bool;
 
     fn check_codegen_attributes_extra(
         &self,
@@ -187,15 +191,22 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
         self.changed
     }
 
-    fn should_inline_for_callee(&self, def_id: DefId) -> bool {
-        ForceInline::should_run_pass_for_callee(self.tcx(), def_id)
+    fn should_inline_for_callee(&self, instance_kind: &InstanceKind<'tcx>) -> bool {
+        if let InstanceKind::Item(def_id) = instance_kind
+            && let InlineAttr::Force { .. } | InlineAttr::Early =
+                self.tcx().codegen_fn_attrs(def_id).inline
+        {
+            true
+        } else {
+            false
+        }
     }
 
     fn check_codegen_attributes_extra(
         &self,
         callee_attrs: &CodegenFnAttrs,
     ) -> Result<(), &'static str> {
-        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. });
+        debug_assert_matches!(callee_attrs.inline, InlineAttr::Force { .. } | InlineAttr::Early);
         Ok(())
     }
 
@@ -247,23 +258,27 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
 
     fn on_inline_failure(&self, callsite: &CallSite<'tcx>, reason: &'static str) {
         let tcx = self.tcx();
-        let InlineAttr::Force { attr_span, reason: justification } =
-            tcx.codegen_fn_attrs(callsite.callee.def_id()).inline
-        else {
-            bug!("called on item without required inlining");
-        };
-
-        let call_span = callsite.source_info.span;
-        tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
-            call_span,
-            attr_span,
-            caller_span: tcx.def_span(self.def_id),
-            caller: tcx.def_path_str(self.def_id),
-            callee_span: tcx.def_span(callsite.callee.def_id()),
-            callee: tcx.def_path_str(callsite.callee.def_id()),
-            reason,
-            justification: justification.map(|sym| crate::errors::ForceInlineJustification { sym }),
-        });
+        match tcx.codegen_fn_attrs(callsite.callee.def_id()).inline {
+            InlineAttr::Early => {
+                // Ok, we don't actually mind if this fails.
+                bug!("early failed for callsite {callsite:?} because {reason}");
+            }
+            InlineAttr::Force { attr_span, reason: justification } => {
+                let call_span = callsite.source_info.span;
+                tcx.dcx().emit_err(crate::errors::ForceInlineFailure {
+                    call_span,
+                    attr_span,
+                    caller_span: tcx.def_span(self.def_id),
+                    caller: tcx.def_path_str(self.def_id),
+                    callee_span: tcx.def_span(callsite.callee.def_id()),
+                    callee: tcx.def_path_str(callsite.callee.def_id()),
+                    reason,
+                    justification: justification
+                        .map(|sym| crate::errors::ForceInlineJustification { sym }),
+                });
+            }
+            _ => bug!("called on item without required inlining"),
+        }
     }
 }
 
@@ -334,7 +349,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
         self.changed
     }
 
-    fn should_inline_for_callee(&self, _: DefId) -> bool {
+    fn should_inline_for_callee(&self, _: &InstanceKind<'tcx>) -> bool {
         true
     }
 
@@ -556,16 +571,16 @@ fn resolve_callsite<'tcx, I: Inliner<'tcx>>(
     if let TerminatorKind::Call { ref func, fn_span, .. } = terminator.kind {
         let func_ty = func.ty(caller_body, tcx);
         if let ty::FnDef(def_id, args) = *func_ty.kind() {
-            if !inliner.should_inline_for_callee(def_id) {
-                debug!("not enabled");
-                return None;
-            }
-
             // To resolve an instance its args have to be fully normalized.
             let args = tcx.try_normalize_erasing_regions(inliner.typing_env(), args).ok()?;
             let callee =
                 Instance::try_resolve(tcx, inliner.typing_env(), def_id, args).ok().flatten()?;
 
+            if !inliner.should_inline_for_callee(&callee.def) {
+                debug!("not enabled");
+                return None;
+            }
+
             if let InstanceKind::Virtual(..) | InstanceKind::Intrinsic(_) = callee.def {
                 return None;
             }