diff --git a/include/swift/SIL/SILModule.h b/include/swift/SIL/SILModule.h
index d32f283e3df64..c21b9368a301d 100644
--- a/include/swift/SIL/SILModule.h
+++ b/include/swift/SIL/SILModule.h
@@ -393,6 +393,10 @@ class SILModule {
   /// This gets set in OwnershipModelEliminator pass.
   bool regDeserializationNotificationHandlerForAllFuncOME;
 
+  // True if a DeserializationNotificationHandler is set for
+  // AccessMarkerElimination.
+  bool hasAccessMarkerHandler;
+
   bool prespecializedFunctionDeclsImported;
 
   /// Action to be executed for serializing the SILModule.
@@ -445,6 +449,13 @@ class SILModule {
     regDeserializationNotificationHandlerForAllFuncOME = true;
   }
 
+  bool checkHasAccessMarkerHandler() {
+    return hasAccessMarkerHandler;
+  }
+  void setHasAccessMarkerHandler() {
+    hasAccessMarkerHandler = true;
+  }
+
   /// Returns the instruction which defines the given root local archetype,
   /// e.g. an open_existential_addr.
   ///
diff --git a/lib/SIL/IR/SILModule.cpp b/lib/SIL/IR/SILModule.cpp
index 21b39a2407c11..f9ac59592c2c0 100644
--- a/lib/SIL/IR/SILModule.cpp
+++ b/lib/SIL/IR/SILModule.cpp
@@ -100,6 +100,7 @@ SILModule::SILModule(llvm::PointerUnion<FileUnit *, ModuleDecl *> context,
       irgenOptions(irgenOptions), serialized(false),
       regDeserializationNotificationHandlerForNonTransparentFuncOME(false),
       regDeserializationNotificationHandlerForAllFuncOME(false),
+      hasAccessMarkerHandler(false),
       prespecializedFunctionDeclsImported(false), SerializeSILAction(),
       Types(TC) {
   assert(!context.isNull());
diff --git a/lib/SILOptimizer/Mandatory/AccessMarkerElimination.cpp b/lib/SILOptimizer/Mandatory/AccessMarkerElimination.cpp
index ccf2b3e46a933..9ded733b8c9f1 100644
--- a/lib/SILOptimizer/Mandatory/AccessMarkerElimination.cpp
+++ b/lib/SILOptimizer/Mandatory/AccessMarkerElimination.cpp
@@ -183,16 +183,16 @@ struct AccessMarkerEliminationPass : SILModuleTransform {
         auto InvalidKind = SILAnalysis::InvalidationKind::Instructions;
         invalidateAnalysis(&F, InvalidKind);
       }
-
-      // Markers from all current SIL functions are stripped. Register a
-      // callback to strip an subsequently loaded functions on-the-fly.
-      if (!EnableOptimizedAccessMarkers) {
-        using NotificationHandlerTy =
-            FunctionBodyDeserializationNotificationHandler;
-        auto *n = new NotificationHandlerTy(prepareSILFunctionForOptimization);
-        std::unique_ptr<DeserializationNotificationHandler> ptr(n);
-        M.registerDeserializationNotificationHandler(std::move(ptr));
-      }
+    }
+    // Markers from all current SIL functions are stripped. Register a
+    // callback to strip an subsequently loaded functions on-the-fly.
+    if (!EnableOptimizedAccessMarkers && !M.checkHasAccessMarkerHandler()) {
+      using NotificationHandlerTy =
+        FunctionBodyDeserializationNotificationHandler;
+      auto *n = new NotificationHandlerTy(prepareSILFunctionForOptimization);
+      std::unique_ptr<DeserializationNotificationHandler> ptr(n);
+      M.registerDeserializationNotificationHandler(std::move(ptr));
+      M.setHasAccessMarkerHandler();
     }
   }
 };