Unrolled build for #144027

Rollup merge of #144027 - RalfJung:clippy, r=Mark-Simulacrum

clippy: make tests work in stage 1

This finally fixes #78717 :)

Similar to what Miri already does, the clippy test step needs to carefully consider  which compiler is used to build clippy and which compiler is linked into clippy (and thus must be used to build the test dependencies). On top of that we have some extra complications that Miri avoided by using `cargo-miri` for building its test dependencies: we need cargo to use the right rustc and the right sysroot, but rust-lang/cargo#4423 makes this quite hard to do. See the long comment in `src/tools/clippy/tests/compile-test.rs` for details.

Some clippy tests tried to import rustc crates; that fundamentally requires a full bootstrap loop so it cannot work in stage 1. I had to kind of guess what those tests were doing so I don't know if my changes there make any sense.

Cc ```@flip1995``` ```@Kobzol```

Author: rust-timer

src/bootstrap/src/core/build_steps/test.rs

@@ -739,7 +739,6 @@ impl Step for CompiletestTest {
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct Clippy {
-    stage: u32,
     host: TargetSelection,
 }
 
@@ -753,33 +752,23 @@ impl Step for Clippy {
     }
 
     fn make_run(run: RunConfig<'_>) {
-        // If stage is explicitly set or not lower than 2, keep it. Otherwise, make sure it's at least 2
-        // as tests for this step don't work with a lower stage.
-        let stage = if run.builder.config.is_explicit_stage() || run.builder.top_stage >= 2 {
-            run.builder.top_stage
-        } else {
-            2
-        };
-
-        run.builder.ensure(Clippy { stage, host: run.target });
+        run.builder.ensure(Clippy { host: run.target });
     }
 
     /// Runs `cargo test` for clippy.
     fn run(self, builder: &Builder<'_>) {
-        let stage = self.stage;
+        let stage = builder.top_stage;
         let host = self.host;
-        let compiler = builder.compiler(stage, host);
-
-        if stage < 2 {
-            eprintln!("WARNING: clippy tests on stage {stage} may not behave well.");
-            eprintln!("HELP: consider using stage 2");
-        }
+        // We need to carefully distinguish the compiler that builds clippy, and the compiler
+        // that is linked into the clippy being tested. `target_compiler` is the latter,
+        // and it must also be used by clippy's test runner to build tests and their dependencies.
+        let target_compiler = builder.compiler(stage, host);
 
-        let tool_result = builder.ensure(tool::Clippy { compiler, target: self.host });
-        let compiler = tool_result.build_compiler;
+        let tool_result = builder.ensure(tool::Clippy { compiler: target_compiler, target: host });
+        let tool_compiler = tool_result.build_compiler;
         let mut cargo = tool::prepare_tool_cargo(
             builder,
-            compiler,
+            tool_compiler,
             Mode::ToolRustc,
             host,
             Kind::Test,
@@ -788,11 +777,17 @@ impl Step for Clippy {
             &[],
         );
 
-        cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler));
-        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler));
-        let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir());
+        cargo.env("RUSTC_TEST_SUITE", builder.rustc(tool_compiler));
+        cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(tool_compiler));
+        let host_libs = builder.stage_out(tool_compiler, Mode::ToolRustc).join(builder.cargo_dir());
         cargo.env("HOST_LIBS", host_libs);
 
+        // Build the standard library that the tests can use.
+        builder.std(target_compiler, host);
+        cargo.env("TEST_SYSROOT", builder.sysroot(target_compiler));
+        cargo.env("TEST_RUSTC", builder.rustc(target_compiler));
+        cargo.env("TEST_RUSTC_LIB", builder.rustc_libdir(target_compiler));
+
         // Collect paths of tests to run
         'partially_test: {
             let paths = &builder.config.paths[..];
@@ -813,7 +808,8 @@ impl Step for Clippy {
         cargo.add_rustc_lib_path(builder);
         let cargo = prepare_cargo_test(cargo, &[], &[], host, builder);
 
-        let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host);
+        let _guard =
+            builder.msg_sysroot_tool(Kind::Test, tool_compiler.stage, "clippy", host, host);
 
         // Clippy reports errors if it blessed the outputs
         if cargo.allow_failure().run(builder) {

src/tools/clippy/clippy_test_deps/Cargo.lock

@@ -72,6 +72,7 @@ dependencies = [
  "futures",
  "if_chain",
  "itertools",
+ "libc",
  "parking_lot",
  "quote",
  "regex",

src/tools/clippy/clippy_test_deps/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 # Add dependencies here to make them available in ui tests.
 
 [dependencies]
+libc = "0.2"
 regex = "1.5.5"
 serde = { version = "1.0.145", features = ["derive"] }
 if_chain = "1.0"

src/tools/clippy/tests/compile-test.rs

@@ -151,7 +151,31 @@ impl TestContext {
         defaults.set_custom(
             "dependencies",
             DependencyBuilder {
-                program: CommandBuilder::cargo(),
+                program: {
+                    let mut p = CommandBuilder::cargo();
+                    // If we run in bootstrap, we need to use the right compiler for building the
+                    // tests -- not the compiler that built clippy, but the compiler that got linked
+                    // into clippy. Just invoking TEST_RUSTC does not work because LD_LIBRARY_PATH
+                    // is set in a way that makes it pick the wrong sysroot. Sadly due to
+                    // <https://github.com/rust-lang/cargo/issues/4423> we cannot use RUSTFLAGS to
+                    // set `--sysroot`, so we need to use bootstrap's rustc wrapper. That wrapper
+                    // however has some staging logic that is hurting us here, so to work around
+                    // that we set both the "real" and "staging" rustc to TEST_RUSTC, including the
+                    // associated library paths.
+                    if let Some(rustc) = option_env!("TEST_RUSTC") {
+                        let libdir = option_env!("TEST_RUSTC_LIB").unwrap();
+                        let sysroot = option_env!("TEST_SYSROOT").unwrap();
+                        p.envs.push(("RUSTC_REAL".into(), Some(rustc.into())));
+                        p.envs.push(("RUSTC_REAL_LIBDIR".into(), Some(libdir.into())));
+                        p.envs.push(("RUSTC_SNAPSHOT".into(), Some(rustc.into())));
+                        p.envs.push(("RUSTC_SNAPSHOT_LIBDIR".into(), Some(libdir.into())));
+                        p.envs.push((
+                            "RUSTC_SYSROOT".into(),
+                            Some(sysroot.into()),
+                        ));
+                    }
+                    p
+                },
                 crate_manifest_path: Path::new("clippy_test_deps").join("Cargo.toml"),
                 build_std: None,
                 bless_lockfile: self.args.bless,
@@ -192,6 +216,9 @@ impl TestContext {
             let dep = format!("-Ldependency={}", Path::new(host_libs).join("deps").display());
             config.program.args.push(dep.into());
         }
+        if let Some(sysroot) = option_env!("TEST_SYSROOT") {
+            config.program.args.push(format!("--sysroot={sysroot}").into());
+        }
 
         config.program.program = profile_path.join(if cfg!(windows) {
             "clippy-driver.exe"

src/tools/clippy/tests/ui/auxiliary/proc_macro_derive.rs

@@ -16,7 +16,7 @@ pub fn derive(_: TokenStream) -> TokenStream {
     let output = quote! {
         // Should not trigger `useless_attribute`
         #[allow(dead_code)]
-        extern crate rustc_middle;
+        extern crate core;
     };
     output
 }

src/tools/clippy/tests/ui/cast_alignment.rs

@@ -1,6 +1,5 @@
 //! Test casts for alignment issues
 
-#![feature(rustc_private)]
 #![feature(core_intrinsics)]
 #![warn(clippy::cast_ptr_alignment)]
 #![allow(
@@ -10,8 +9,6 @@
     clippy::borrow_as_ptr
 )]
 
-extern crate libc;
-
 fn main() {
     /* These should be warned against */
 

src/tools/clippy/tests/ui/cast_alignment.stderr

@@ -1,5 +1,5 @@
 error: casting from `*const u8` to a more-strictly-aligned pointer (`*const u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:19:5
+  --> tests/ui/cast_alignment.rs:16:5
    |
 LL |     (&1u8 as *const u8) as *const u16;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -8,19 +8,19 @@ LL |     (&1u8 as *const u8) as *const u16;
    = help: to override `-D warnings` add `#[allow(clippy::cast_ptr_alignment)]`
 
 error: casting from `*mut u8` to a more-strictly-aligned pointer (`*mut u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:22:5
+  --> tests/ui/cast_alignment.rs:19:5
    |
 LL |     (&mut 1u8 as *mut u8) as *mut u16;
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: casting from `*const u8` to a more-strictly-aligned pointer (`*const u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:26:5
+  --> tests/ui/cast_alignment.rs:23:5
    |
 LL |     (&1u8 as *const u8).cast::<u16>();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 error: casting from `*mut u8` to a more-strictly-aligned pointer (`*mut u16`) (1 < 2 bytes)
-  --> tests/ui/cast_alignment.rs:29:5
+  --> tests/ui/cast_alignment.rs:26:5
    |
 LL |     (&mut 1u8 as *mut u8).cast::<u16>();
    |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

src/tools/clippy/tests/ui/iter_over_hash_type.rs

@@ -3,15 +3,18 @@
 #![warn(clippy::iter_over_hash_type)]
 use std::collections::{HashMap, HashSet};
 
-extern crate rustc_data_structures;
-
 extern crate proc_macros;
 
+// Ensure it also works via type aliases (this isn't really the Fx hasher but that does not matter).
+type FxBuildHasher = std::collections::hash_map::RandomState;
+type FxHashMap<K, V> = HashMap<K, V, FxBuildHasher>;
+type FxHashSet<K> = HashSet<K, FxBuildHasher>;
+
 fn main() {
     let mut hash_set = HashSet::<i32>::new();
     let mut hash_map = HashMap::<i32, i32>::new();
-    let mut fx_hash_map = rustc_data_structures::fx::FxHashMap::<i32, i32>::default();
-    let mut fx_hash_set = rustc_data_structures::fx::FxHashMap::<i32, i32>::default();
+    let mut fx_hash_map = FxHashMap::<i32, i32>::default();
+    let mut fx_hash_set = FxHashSet::<i32>::default();
     let vec = Vec::<i32>::new();
 
     // test hashset

src/tools/clippy/tests/ui/iter_over_hash_type.stderr

@@ -1,5 +1,5 @@
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:18:5
+  --> tests/ui/iter_over_hash_type.rs:21:5
    |
 LL | /     for x in &hash_set {
 LL | |
@@ -11,7 +11,7 @@ LL | |     }
    = help: to override `-D warnings` add `#[allow(clippy::iter_over_hash_type)]`
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:22:5
+  --> tests/ui/iter_over_hash_type.rs:25:5
    |
 LL | /     for x in hash_set.iter() {
 LL | |
@@ -20,7 +20,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:26:5
+  --> tests/ui/iter_over_hash_type.rs:29:5
    |
 LL | /     for x in hash_set.clone() {
 LL | |
@@ -29,7 +29,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:30:5
+  --> tests/ui/iter_over_hash_type.rs:33:5
    |
 LL | /     for x in hash_set.drain() {
 LL | |
@@ -38,7 +38,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:36:5
+  --> tests/ui/iter_over_hash_type.rs:39:5
    |
 LL | /     for (x, y) in &hash_map {
 LL | |
@@ -47,7 +47,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:40:5
+  --> tests/ui/iter_over_hash_type.rs:43:5
    |
 LL | /     for x in hash_map.keys() {
 LL | |
@@ -56,7 +56,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:44:5
+  --> tests/ui/iter_over_hash_type.rs:47:5
    |
 LL | /     for x in hash_map.values() {
 LL | |
@@ -65,7 +65,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:48:5
+  --> tests/ui/iter_over_hash_type.rs:51:5
    |
 LL | /     for x in hash_map.values_mut() {
 LL | |
@@ -74,7 +74,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:52:5
+  --> tests/ui/iter_over_hash_type.rs:55:5
    |
 LL | /     for x in hash_map.iter() {
 LL | |
@@ -83,7 +83,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:56:5
+  --> tests/ui/iter_over_hash_type.rs:59:5
    |
 LL | /     for x in hash_map.clone() {
 LL | |
@@ -92,7 +92,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:60:5
+  --> tests/ui/iter_over_hash_type.rs:63:5
    |
 LL | /     for x in hash_map.drain() {
 LL | |
@@ -101,7 +101,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:66:5
+  --> tests/ui/iter_over_hash_type.rs:69:5
    |
 LL | /     for x in fx_hash_set {
 LL | |
@@ -110,7 +110,7 @@ LL | |     }
    | |_____^
 
 error: iteration over unordered hash-based type
-  --> tests/ui/iter_over_hash_type.rs:70:5
+  --> tests/ui/iter_over_hash_type.rs:73:5
    |
 LL | /     for x in fx_hash_map {
 LL | |

src/tools/clippy/tests/ui/strlen_on_c_strings.fixed

@@ -1,7 +1,5 @@
 #![warn(clippy::strlen_on_c_strings)]
 #![allow(dead_code, clippy::manual_c_str_literals)]
-#![feature(rustc_private)]
-extern crate libc;
 
 #[allow(unused)]
 use libc::strlen;