diff --git a/compiler/rustc_llvm/build.rs b/compiler/rustc_llvm/build.rs
index 28e092c1eb72c..a472dd122f259 100644
--- a/compiler/rustc_llvm/build.rs
+++ b/compiler/rustc_llvm/build.rs
@@ -46,11 +46,12 @@ fn detect_llvm_link() -> (&'static str, &'static str) {
 // perfect -- we might actually want to see something from Cargo's added library paths -- but
 // for now it works.
 fn restore_library_path() {
-    let key = tracked_env_var_os("REAL_LIBRARY_PATH_VAR").expect("REAL_LIBRARY_PATH_VAR");
-    if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") {
-        env::set_var(&key, &env);
-    } else {
-        env::remove_var(&key);
+    if let Some(key) = tracked_env_var_os("REAL_LIBRARY_PATH_VAR") {
+        if let Some(env) = tracked_env_var_os("REAL_LIBRARY_PATH") {
+            env::set_var(&key, &env);
+        } else {
+            env::remove_var(&key);
+        }
     }
 }
 
diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs
index 31158870f394e..809be5428c9b6 100644
--- a/src/bootstrap/builder.rs
+++ b/src/bootstrap/builder.rs
@@ -1361,9 +1361,11 @@ impl<'a> Builder<'a> {
 
         // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config
         // needs to not accidentally link to libLLVM in stage0/lib.
-        cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var());
-        if let Some(e) = env::var_os(util::dylib_path_var()) {
-            cargo.env("REAL_LIBRARY_PATH", e);
+        if !self.config.llvm_from_ci {
+            cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var());
+            if let Some(e) = env::var_os(util::dylib_path_var()) {
+                cargo.env("REAL_LIBRARY_PATH", e);
+            }
         }
 
         // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger`