Auto merge of #145408 - Kobzol:deduplicate-search-paths, r=<try>

Deduplicate -L search paths

Author: rust-bors[bot]

compiler/rustc_session/src/config.rs

@@ -2847,16 +2847,27 @@ pub fn build_session_options(early_dcx: &mut EarlyDiagCtxt, matches: &getopts::M
         // This is the location used by the `rustc-dev` `rustup` component.
         real_source_base_dir("lib/rustlib/rustc-src/rust", "compiler/rustc/src/main.rs");
 
-    let mut search_paths = vec![];
-    for s in &matches.opt_strs("L") {
-        search_paths.push(SearchPath::from_cli_opt(
-            sysroot.path(),
-            &target_triple,
-            early_dcx,
-            s,
-            unstable_opts.unstable_options,
-        ));
-    }
+    // We eagerly scan all files in each passed -L path. If the same directory is passed multiple
+    // times, and the directory contains a lot of files, this can take a lot of time.
+    // So we remove -L paths that were passed multiple times, and keep only the first occurrence.
+    // We still have to keep the original order of the -L arguments.
+    let search_paths: Vec<SearchPath> = {
+        let mut seen_search_paths = FxHashSet::default();
+        let search_path_matches: Vec<String> = matches.opt_strs("L");
+        search_path_matches
+            .iter()
+            .filter(|p| seen_search_paths.insert(*p))
+            .map(|path| {
+                SearchPath::from_cli_opt(
+                    sysroot.path(),
+                    &target_triple,
+                    early_dcx,
+                    &path,
+                    unstable_opts.unstable_options,
+                )
+            })
+            .collect()
+    };
 
     let working_dir = std::env::current_dir().unwrap_or_else(|e| {
         early_dcx.early_fatal(format!("Current directory is invalid: {e}"));

src/librustdoc/doctest.rs

@@ -16,7 +16,7 @@ use std::{fmt, panic, str};
 
 pub(crate) use make::{BuildDocTestBuilder, DocTestBuilder};
 pub(crate) use markdown::test as test_markdown;
-use rustc_data_structures::fx::{FxHashMap, FxHasher, FxIndexMap, FxIndexSet};
+use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxHasher, FxIndexMap, FxIndexSet};
 use rustc_errors::emitter::HumanReadableErrorType;
 use rustc_errors::{ColorConfig, DiagCtxtHandle};
 use rustc_hir as hir;
@@ -689,6 +689,10 @@ fn run_test(
             "--extern=doctest_bundle_{edition}=",
             edition = doctest.edition
         ));
+
+        // Deduplicate passed -L directory paths, since usually all dependencies will be in the
+        // same directory (e.g. target/debug/deps from Cargo).
+        let mut seen_search_dirs = FxHashSet::default();
         for extern_str in &rustdoc_options.extern_strs {
             if let Some((_cratename, path)) = extern_str.split_once('=') {
                 // Direct dependencies of the tests themselves are
@@ -698,7 +702,9 @@ fn run_test(
                     .parent()
                     .filter(|x| x.components().count() > 0)
                     .unwrap_or(Path::new("."));
-                runner_compiler.arg("-L").arg(dir);
+                if seen_search_dirs.insert(dir) {
+                    runner_compiler.arg("-L").arg(dir);
+                }
             }
         }
         let output_bundle_file = doctest