Make Global closeable

Close Global-scoped JARs and ClassLoaders.

Entries of FileBasedCache that have a lifetime that exceeds any
single Global instance are reference counted. When the count
hits zero, it is closed if no references appear after a delay.

Author: retronym

src/compiler/scala/tools/nsc/CloseableRegistry.scala

@@ -0,0 +1,34 @@
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc.
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+
+package scala.tools.nsc
+
+import scala.util.control.NonFatal
+
+/** Registry for resources to close when `Global` is closed */
+final class CloseableRegistry {
+  private[this] var closeables: List[java.io.Closeable] = Nil
+  final def registerClosable(c: java.io.Closeable): Unit = {
+    closeables ::= c
+  }
+
+  def close(): Unit = {
+    for (c <- closeables) {
+      try {
+        c.close()
+      } catch {
+        case NonFatal(_) =>
+      }
+    }
+    closeables = Nil
+  }
+}

src/compiler/scala/tools/nsc/GenericRunnerSettings.scala

@@ -16,7 +16,14 @@ import java.net.URL
 import scala.tools.util.PathResolver
 
 class GenericRunnerSettings(error: String => Unit) extends Settings(error) {
-  lazy val classpathURLs: Seq[URL] = new PathResolver(this).resultAsURLs
+  lazy val classpathURLs: Seq[URL] = {
+    val registry = new CloseableRegistry
+    try {
+      new PathResolver(this, new CloseableRegistry).resultAsURLs
+    } finally {
+      registry.close()
+    }
+  }
 
   val howtorun =
     ChoiceSetting(

src/compiler/scala/tools/nsc/Global.scala

@@ -40,9 +40,12 @@ import scala.language.postfixOps
 import scala.tools.nsc.ast.{TreeGen => AstTreeGen}
 import scala.tools.nsc.classpath._
 import scala.tools.nsc.profile.Profiler
+import scala.util.control.NonFatal
+import java.io.Closeable
 
 class Global(var currentSettings: Settings, reporter0: LegacyReporter)
     extends SymbolTable
+    with Closeable
     with CompilationUnits
     with Plugins
     with PhaseAssembly
@@ -814,7 +817,7 @@ class Global(var currentSettings: Settings, reporter0: LegacyReporter)
 
   /** Extend classpath of `platform` and rescan updated packages. */
   def extendCompilerClassPath(urls: URL*): Unit = {
-    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings))
+    val urlClasspaths = urls.map(u => ClassPathFactory.newClassPath(AbstractFile.getURL(u), settings, closeableRegistry))
     val newClassPath = AggregateClassPath.createAggregate(platform.classPath +: urlClasspaths : _*)
     platform.currentClassPath = Some(newClassPath)
     invalidateClassPathEntries(urls.map(_.getPath): _*)
@@ -876,7 +879,7 @@ class Global(var currentSettings: Settings, reporter0: LegacyReporter)
       }
       entries(classPath) find matchesCanonical match {
         case Some(oldEntry) =>
-          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings))
+          Some(oldEntry -> ClassPathFactory.newClassPath(dir, settings, closeableRegistry))
         case None =>
           error(s"Error adding entry to classpath. During invalidation, no entry named $path in classpath $classPath")
           None
@@ -1685,6 +1688,13 @@ class Global(var currentSettings: Settings, reporter0: LegacyReporter)
   }
 
   def createJavadoc    = false
+
+  final val closeableRegistry: CloseableRegistry = new CloseableRegistry
+
+  def close(): Unit = {
+    perRunCaches.clearAll()
+    closeableRegistry.close()
+  }
 }
 
 object Global {

src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

@@ -27,7 +27,7 @@ trait JavaPlatform extends Platform {
   private[nsc] var currentClassPath: Option[ClassPath] = None
 
   private[nsc] def classPath: ClassPath = {
-    if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result)
+    if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings, global.closeableRegistry).result)
     currentClassPath.get
   }
 

src/compiler/scala/tools/nsc/classpath/AggregateClassPath.scala

@@ -65,7 +65,6 @@ case class AggregateClassPath(aggregates: Seq[ClassPath]) extends ClassPath {
   override def asClassPathStrings: Seq[String] = aggregates.map(_.asClassPathString).distinct
 
   override def asSourcePathString: String = ClassPath.join(aggregates map (_.asSourcePathString): _*)
-
   override private[nsc] def packages(inPackage: String): Seq[PackageEntry] = {
     val aggregatedPackages = aggregates.flatMap(_.packages(inPackage)).distinct
     aggregatedPackages

src/compiler/scala/tools/nsc/classpath/ClassPathFactory.scala

@@ -14,19 +14,19 @@ package scala.tools.nsc.classpath
 
 import scala.reflect.io.{AbstractFile, VirtualDirectory}
 import scala.reflect.io.Path.string2path
-import scala.tools.nsc.Settings
+import scala.tools.nsc.{CloseableRegistry, Settings}
 import FileUtils.AbstractFileOps
 import scala.tools.nsc.util.ClassPath
 
 /**
  * Provides factory methods for classpath. When creating classpath instances for a given path,
  * it uses proper type of classpath depending on a types of particular files containing sources or classes.
  */
-class ClassPathFactory(settings: Settings) {
+class ClassPathFactory(settings: Settings, closeableRegistry: CloseableRegistry) {
   /**
     * Create a new classpath based on the abstract file.
     */
-  def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings)
+  def newClassPath(file: AbstractFile): ClassPath = ClassPathFactory.newClassPath(file, settings, closeableRegistry)
 
   /**
     * Creators for sub classpaths which preserve this context.
@@ -70,19 +70,19 @@ class ClassPathFactory(settings: Settings) {
 
   private def createSourcePath(file: AbstractFile): ClassPath =
     if (file.isJarOrZip)
-      ZipAndJarSourcePathFactory.create(file, settings)
+      ZipAndJarSourcePathFactory.create(file, settings, closeableRegistry)
     else if (file.isDirectory)
       DirectorySourcePath(file.file)
     else
       throw new IllegalArgumentException(s"Unsupported sourcepath element: $file")
 }
 
 object ClassPathFactory {
-  def newClassPath(file: AbstractFile, settings: Settings): ClassPath = file match {
+  def newClassPath(file: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = file match {
     case vd: VirtualDirectory => VirtualDirectoryClassPath(vd)
     case _ =>
       if (file.isJarOrZip)
-        ZipAndJarClassPathFactory.create(file, settings)
+        ZipAndJarClassPathFactory.create(file, settings, closeableRegistry)
       else if (file.isDirectory)
         DirectoryClassPath(file.file)
       else

src/compiler/scala/tools/nsc/classpath/DirectoryClassPath.scala

@@ -12,7 +12,7 @@
 
 package scala.tools.nsc.classpath
 
-import java.io.File
+import java.io.{Closeable, File}
 import java.net.URL
 import java.nio.file.{FileSystems, Files}
 import java.util
@@ -22,6 +22,7 @@ import scala.tools.nsc.util.{ClassPath, ClassRepresentation}
 import FileUtils._
 import scala.collection.JavaConverters._
 import scala.reflect.internal.JDK9Reflectors
+import scala.tools.nsc.CloseableRegistry
 import scala.tools.nsc.classpath.PackageNameUtils.{packageContains, separatePkgAndClassNames}
 
 /**
@@ -58,6 +59,7 @@ trait DirectoryLookup[FileEntryType <: ClassRepresentation] extends ClassPath {
 
   private[nsc] def packages(inPackage: String): Seq[PackageEntry] = {
     val dirForPackage = getDirectory(inPackage)
+
     val nestedDirs: Array[F] = dirForPackage match {
       case None => emptyFiles
       case Some(directory) => listChildren(directory, Some(isPackage))
@@ -134,7 +136,7 @@ trait JFileDirectoryLookup[FileEntryType <: ClassRepresentation] extends Directo
 
 object JrtClassPath {
   import java.nio.file._, java.net.URI
-  def apply(release: Option[String]): Option[ClassPath] = {
+  def apply(release: Option[String], closeableRegistry: CloseableRegistry): Option[ClassPath] = {
     import scala.util.Properties._
     if (!isJavaAtLeast("9")) None
     else {
@@ -151,7 +153,11 @@ object JrtClassPath {
           try {
             val ctSym = Paths.get(javaHome).resolve("lib").resolve("ct.sym")
             if (Files.notExists(ctSym)) None
-            else Some(new CtSymClassPath(ctSym, v.toInt))
+            else {
+              val classPath = new CtSymClassPath(ctSym, v.toInt)
+              closeableRegistry.registerClosable(classPath)
+              Some(classPath)
+            }
           } catch {
             case _: Throwable => None
           }
@@ -227,7 +233,7 @@ final class JrtClassPath(fs: java.nio.file.FileSystem) extends ClassPath with No
 /**
   * Implementation `ClassPath` based on the $JAVA_HOME/lib/ct.sym backing http://openjdk.java.net/jeps/247
   */
-final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths {
+final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends ClassPath with NoSourcePaths with Closeable {
   import java.nio.file.Path, java.nio.file._
 
   private val fileSystem: FileSystem = FileSystems.newFileSystem(ctSym, null)
@@ -273,7 +279,7 @@ final class CtSymClassPath(ctSym: java.nio.file.Path, release: Int) extends Clas
 
   def asURLs: Seq[URL] = Nil
   def asClassPathStrings: Seq[String] = Nil
-
+  override def close(): Unit = fileSystem.close()
   def findClassFile(className: String): Option[AbstractFile] = {
     if (!className.contains(".")) None
     else {

src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala

@@ -37,7 +37,6 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
   // mimic the behavior of the old nsc.util.DirectoryClassPath
   def asURLs: Seq[URL] = Seq(new URL(dir.name))
   def asClassPathStrings: Seq[String] = Seq(dir.path)
-
   override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl
 
   def findClassFile(className: String): Option[AbstractFile] = {

src/compiler/scala/tools/nsc/classpath/ZipAndJarFileLookupFactory.scala

@@ -12,15 +12,17 @@
 
 package scala.tools.nsc.classpath
 
-import java.io.File
+import java.io.{Closeable, File}
 import java.net.URL
 import java.nio.file.Files
 import java.nio.file.attribute.{BasicFileAttributes, FileTime}
+import java.util.{Timer, TimerTask}
+import java.util.concurrent.atomic.AtomicInteger
 
 import scala.annotation.tailrec
 import scala.reflect.io.{AbstractFile, FileZipArchive, ManifestResources}
 import scala.tools.nsc.util.{ClassPath, ClassRepresentation}
-import scala.tools.nsc.Settings
+import scala.tools.nsc.{CloseableRegistry, Settings}
 import FileUtils._
 
 /**
@@ -29,18 +31,19 @@ import FileUtils._
  * when there are a lot of projects having a lot of common dependencies.
  */
 sealed trait ZipAndJarFileLookupFactory {
-  private val cache = new FileBasedCache[ClassPath]
-
-  def create(zipFile: AbstractFile, settings: Settings): ClassPath = {
-    if (settings.YdisableFlatCpCaching || zipFile.file == null) createForZipFile(zipFile, settings.releaseValue)
-    else createUsingCache(zipFile, settings)
+  private val cache = new FileBasedCache[ClassPath with Closeable]
+
+  def create(zipFile: AbstractFile, settings: Settings, closeableRegistry: CloseableRegistry): ClassPath = {
+    if (settings.YdisableFlatCpCaching || zipFile.file == null) {
+      val result: ClassPath with Closeable = createForZipFile(zipFile, settings.releaseValue)
+      closeableRegistry.registerClosable(result)
+      result
+    } else {
+      cache.getOrCreate(List(zipFile.file.toPath), () => createForZipFile(zipFile, settings.releaseValue), closeableRegistry)
+    }
   }
 
-  protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath
-
-  private def createUsingCache(zipFile: AbstractFile, settings: Settings): ClassPath = {
-    cache.getOrCreate(List(zipFile.file.toPath), () => createForZipFile(zipFile, settings.releaseValue))
-  }
+  protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable
 }
 
 /**
@@ -75,7 +78,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
    * with a particularly prepared scala-library.jar. It should have all classes listed in the manifest like e.g. this entry:
    * Name: scala/Function2$mcFJD$sp.class
    */
-  private case class ManifestResourcesClassPath(file: ManifestResources) extends ClassPath with NoSourcePaths {
+  private case class ManifestResourcesClassPath(file: ManifestResources) extends ClassPath with NoSourcePaths with Closeable {
     override def findClassFile(className: String): Option[AbstractFile] = {
       val (pkg, simpleClassName) = PackageNameUtils.separatePkgAndClassNames(className)
       classes(pkg).find(_.name == simpleClassName).map(_.file)
@@ -84,6 +87,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
     override def asClassPathStrings: Seq[String] = Seq(file.path)
 
     override def asURLs: Seq[URL] = file.toURLs()
+    override def close(): Unit = file.close()
 
     import ManifestResourcesClassPath.PackageFileInfo
     import ManifestResourcesClassPath.PackageInfo
@@ -152,7 +156,7 @@ object ZipAndJarClassPathFactory extends ZipAndJarFileLookupFactory {
     case class PackageInfo(packageName: String, subpackages: List[AbstractFile])
   }
 
-  override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath =
+  override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable =
     if (zipFile.file == null) createWithoutUnderlyingFile(zipFile)
     else ZipArchiveClassPath(zipFile.file, release)
 
@@ -183,15 +187,33 @@ object ZipAndJarSourcePathFactory extends ZipAndJarFileLookupFactory {
     override protected def isRequiredFileType(file: AbstractFile): Boolean = file.isScalaOrJavaSource
   }
 
-  override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath = ZipArchiveSourcePath(zipFile.file)
+  override protected def createForZipFile(zipFile: AbstractFile, release: Option[String]): ClassPath with Closeable = ZipArchiveSourcePath(zipFile.file)
 }
 
 final class FileBasedCache[T] {
   import java.nio.file.Path
   private case class Stamp(lastModified: FileTime, fileKey: Object)
-  private val cache = collection.mutable.Map.empty[Seq[Path], (Seq[Stamp], T)]
+  private case class Entry(stamps: Seq[Stamp], t: T) {
+    val referenceCount: AtomicInteger = new AtomicInteger(1)
+    def referenceCountDecrementer: Closeable = new Closeable {
+      var closed = false
+      override def close(): Unit = {
+        if (!closed) {
+          closed = true
+          val count = referenceCount.decrementAndGet()
+          if (count == 0) {
+            t match {
+              case cl: Closeable => FileBasedCache.deferredClose(referenceCount, cl)
+              case _ =>
+            }
+          }
+        }
+      }
+    }
+  }
+  private val cache = collection.mutable.Map.empty[Seq[Path], Entry]
 
-  def getOrCreate(paths: Seq[Path], create: () => T): T = cache.synchronized {
+  def getOrCreate(paths: Seq[Path], create: () => T, closeableRegistry: CloseableRegistry): T = cache.synchronized {
     val stamps = paths.map { path =>
       val attrs = Files.readAttributes(path, classOf[BasicFileAttributes])
       val lastModified = attrs.lastModifiedTime()
@@ -201,10 +223,15 @@ final class FileBasedCache[T] {
     }
 
     cache.get(paths) match {
-      case Some((cachedStamps, cached)) if cachedStamps == stamps => cached
+      case Some(e@Entry(cachedStamps, cached)) if cachedStamps == stamps =>
+        e.referenceCount.incrementAndGet()
+        closeableRegistry.registerClosable(e.referenceCountDecrementer)
+        cached
       case _ =>
         val value = create()
-        cache.put(paths, (stamps, value))
+        val entry = Entry(stamps, value)
+        cache.put(paths, entry)
+        closeableRegistry.registerClosable(entry.referenceCountDecrementer)
         value
     }
   }
@@ -215,3 +242,26 @@ final class FileBasedCache[T] {
     cache.clear()
   }
 }
+
+object FileBasedCache {
+  private val deferCloseMs = Integer.getInteger("scalac.filebasedcache.defer.close.ms", 1000)
+  private val timer: Option[Timer] = {
+    if (deferCloseMs > 0)
+      Some(new java.util.Timer(true))
+    else None
+  }
+  private def deferredClose(referenceCount: AtomicInteger, closable: Closeable): Unit = {
+    timer match {
+      case Some(timer) =>
+        val task = new TimerTask {
+          override def run(): Unit = {
+            if (referenceCount.get == 0)
+              closable.close()
+          }
+        }
+        timer.schedule(task, FileBasedCache.deferCloseMs.toLong)
+      case None =>
+        closable.close()
+    }
+  }
+}