Classpath infrastracture to support pipelining, caching.

Author: retronym

src/compiler/scala/tools/nsc/PipelineMain.scala

@@ -27,7 +27,7 @@ trait JavaPlatform extends Platform {
   private[nsc] var currentClassPath: Option[ClassPath] = None
 
   private[nsc] def classPath: ClassPath = {
-    if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result)
+    if (currentClassPath.isEmpty) currentClassPath = Some(applyClassPathPlugins(new PathResolver(settings).result))
     currentClassPath.get
   }
 

src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

@@ -13,7 +13,10 @@
 package scala.tools.nsc
 package backend
 
+import java.nio.ByteBuffer
+
 import io.AbstractFile
+import scala.tools.nsc.classpath.AggregateClassPath
 import scala.tools.nsc.util.ClassPath
 
 /** The platform dependent pieces of Global.
@@ -44,5 +47,57 @@ trait Platform {
    * a re-compile is triggered. On .NET by contrast classfiles always take precedence.
    */
   def needCompile(bin: AbstractFile, src: AbstractFile): Boolean
+
+  /**
+    * A class path plugin can modify the classpath before it is used by the compiler, and can
+    * customize the way that the compiler reads the contents of class files.
+    *
+    * Applications could include:
+    *
+    *   - Caching the ScalaSignature annotation contents, to avoid the cost of decompressing
+    *     and parsing the classfile, akin to the OpenJDK's .sig format for stripped class files.
+    *   - Starting a downstream compilation job immediately after the upstream job has completed
+    *     the pickler phase ("Build Pipelineing")
+    */
+  abstract class ClassPathPlugin {
+    def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo]
+    def parsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = ()
+    def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = classPath
+  }
+
+  /** A list of registered classpath plugins */
+  private var classPathPlugins: List[ClassPathPlugin] = Nil
+
+  protected final def applyClassPathPlugins(original: ClassPath): ClassPath = {
+    val entries = original match {
+      case AggregateClassPath(entries) => entries
+      case single => single :: Nil
+    }
+    val entries1 = classPathPlugins.foldLeft(entries) {
+      (entries, plugin) => plugin.modifyClassPath(entries)
+    }
+    AggregateClassPath(entries1)
+  }
+
+
+  /** Registers a new classpath plugin */
+  final def addClassPathPlugin(plugin: ClassPathPlugin): Unit = {
+    if (!classPathPlugins.contains(plugin))
+      classPathPlugins = plugin :: classPathPlugins
+  }
+  final def classFileInfo(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] = if (classPathPlugins eq Nil) None else {
+    classPathPlugins.foldLeft(Option.empty[ClassfileInfo]) {
+      case (Some(info), _) => Some(info)
+      case (None, plugin) => plugin.info(file, clazz)
+    }
+  }
+  final def classFileInfoParsed(file: AbstractFile, clazz: ClassSymbol, info: ClassfileInfo): Unit = if (classPathPlugins eq Nil) None else {
+    classPathPlugins.foreach(_.parsed(file, clazz, info))
+  }
 }
 
+sealed abstract class ClassfileInfo {}
+final case class ClassBytes(data: ByteBuffer) extends ClassfileInfo
+final case class ScalaRawClass(className: String) extends ClassfileInfo
+final case class ScalaClass(className: String, pickle: ByteBuffer) extends ClassfileInfo
+

src/compiler/scala/tools/nsc/backend/Platform.scala

@@ -35,7 +35,7 @@ case class VirtualDirectoryClassPath(dir: VirtualDirectory) extends ClassPath wi
   def isPackage(f: AbstractFile): Boolean = f.isPackage
 
   // mimic the behavior of the old nsc.util.DirectoryClassPath
-  def asURLs: Seq[URL] = Seq(new URL(dir.name))
+  def asURLs: Seq[URL] = Seq(new URL("file://_VIRTUAL_/" + dir.name))
   def asClassPathStrings: Seq[String] = Seq(dir.path)
 
   override def findClass(className: String): Option[ClassRepresentation] = findClassFile(className) map ClassFileEntryImpl

src/compiler/scala/tools/nsc/classpath/VirtualDirectoryClassPath.scala

@@ -25,7 +25,7 @@ trait AbsSettings extends scala.reflect.internal.settings.AbsSettings {
   protected def allSettings: scala.collection.Set[Setting]
 
   // settings minus internal usage settings
-  def visibleSettings = allSettings filterNot (_.isInternalOnly)
+  def visibleSettings = allSettings.iterator filterNot (_.isInternalOnly)
 
   // only settings which differ from default
   def userSetSettings = visibleSettings filterNot (_.isDefault)

src/compiler/scala/tools/nsc/settings/AbsSettings.scala

@@ -243,6 +243,7 @@ trait ScalaSettings extends AbsScalaSettings
   val YcacheMacroClassLoader   = CachePolicy.setting("macro", "macros")
   val YpartialUnification = BooleanSetting ("-Ypartial-unification", "Enable partial unification in type constructor inference")
   val Yvirtpatmat     = BooleanSetting    ("-Yvirtpatmat", "Enable pattern matcher virtualization")
+  val Youtline        = BooleanSetting    ("-Youtline", "Don't compile method bodies. Use together with `-Ystop-afer:pickler to generate the pickled signatures for all source files.")
 
   val exposeEmptyPackage = BooleanSetting ("-Yexpose-empty-package", "Internal only: expose the empty package.").internalOnly()
   val Ydelambdafy        = ChoiceSetting  ("-Ydelambdafy", "strategy", "Strategy used for translating lambdas into JVM code.", List("inline", "method"), "method")

src/compiler/scala/tools/nsc/settings/ScalaSettings.scala

@@ -277,6 +277,8 @@ abstract class SymbolLoaders {
 
       val classPathEntries = classPath.list(packageName)
 
+      if (root.name.string_==("immutable"))
+        getClass
       if (!root.isRoot)
         for (entry <- classPathEntries.classesAndSources) initializeFromClassPath(root, entry)
       if (!root.isEmptyPackageClass) {

src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala

@@ -25,11 +25,8 @@ import scala.tools.nsc.io.AbstractFile
  * @author Philippe Altherr
  * @version 1.0, 23/03/2004
  */
-class AbstractFileReader(val file: AbstractFile) {
-
-  /** the buffer containing the file
-   */
-  val buf: Array[Byte] = file.toByteArray
+class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) {
+  def this(file: AbstractFile) = this(file, file.toByteArray)
 
   /** the current input pointer
    */

src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala

@@ -15,16 +15,18 @@ package tools.nsc
 package symtab
 package classfile
 
-import java.io.{ByteArrayInputStream, DataInputStream, File, IOException}
+import java.io._
 import java.lang.Integer.toHexString
+import java.nio.ByteBuffer
 
 import scala.collection.{immutable, mutable}
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 import scala.annotation.switch
 import scala.reflect.internal.JavaAccFlags
 import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer}
-import scala.reflect.io.NoAbstractFile
+import scala.reflect.io.{NoAbstractFile, VirtualFile}
 import scala.reflect.internal.util.Collections._
+import scala.tools.nsc.backend.{ClassBytes, ScalaClass, ScalaRawClass}
 import scala.tools.nsc.util.ClassPath
 import scala.tools.nsc.io.AbstractFile
 import scala.util.control.NonFatal
@@ -152,14 +154,56 @@ abstract class ClassfileParser {
   def parse(file: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol): Unit = {
     this.file = file
     pushBusy(clazz) {
-      this.in           = new AbstractFileReader(file)
       this.clazz        = clazz
       this.staticModule = module
       this.isScala      = false
 
-      parseHeader()
-      this.pool = newConstantPool
-      parseClass()
+      import loaders.platform._
+      classFileInfo(file, clazz) match {
+        case Some(info) =>
+          info match {
+            case ScalaRawClass(className) =>
+              isScalaRaw = true
+              currentClass = TermName(className)
+            case ScalaClass(className, pickle) =>
+              val pickle1 = pickle
+              isScala = true
+              currentClass = TermName(className)
+              if (pickle1.hasArray) {
+                unpickler.unpickle(pickle1.array, pickle1.arrayOffset + pickle1.position(), clazz, staticModule, file.name)
+              } else {
+                val array = new Array[Byte](pickle1.remaining)
+                pickle1.get(array)
+                unpickler.unpickle(array, 0, clazz, staticModule, file.name)
+              }
+            case ClassBytes(data) =>
+              val data1 = data.duplicate()
+              val array = new Array[Byte](data1.remaining)
+              data1.get(array)
+              this.in = new AbstractFileReader(file, array)
+              parseHeader()
+              this.pool = newConstantPool
+              parseClass()
+          }
+        case None =>
+          this.in = new AbstractFileReader(file)
+          parseHeader()
+          this.pool = newConstantPool
+          parseClass()
+          if (!(isScala || isScalaRaw))
+            loaders.platform.classFileInfoParsed(file, clazz, ClassBytes(ByteBuffer.wrap(in.buf)))
+      }
+      if (isScalaRaw && !isNothingOrNull) {
+        unlinkRaw()
+      }
+    }
+  }
+
+  private def unlinkRaw(): Unit = {
+    val decls = clazz.enclosingPackage.info.decls
+    for (c <- List(clazz, staticModule, staticModule.moduleClass)) {
+      c.setInfo(NoType)
+      decls.unlink(c)
     }
   }
 
@@ -441,6 +485,15 @@ abstract class ClassfileParser {
       lookupClass(name)
   }
 
+  // TODO: remove after the next 2.13 milestone
+  // A bug in the backend caused classes ending in `$` do get only a Scala marker attribute
+  // instead of a ScalaSig and a Signature annotaiton. This went unnoticed because isScalaRaw
+  // classes were parsed like Java classes. The below covers the cases in the std lib.
+  private def isNothingOrNull = {
+    val n = clazz.fullName.toString
+    n == "scala.runtime.Nothing$" || n == "scala.runtime.Null$"
+  }
+
   def parseClass() {
     val jflags   = readClassFlags()
     val sflags   = jflags.toScalaFlags
@@ -890,8 +943,8 @@ abstract class ClassfileParser {
               case Some(san: AnnotationInfo) =>
                 val bytes =
                   san.assocs.find({ _._1 == nme.bytes }).get._2.asInstanceOf[ScalaSigBytes].bytes
-
                 unpickler.unpickle(bytes, 0, clazz, staticModule, in.file.name)
+                loaders.platform.classFileInfoParsed(file, clazz, ScalaClass(this.currentClass.toString, ByteBuffer.wrap(bytes)))
               case None =>
                 throw new RuntimeException("Scala class file does not contain Scala annotation")
             }
@@ -1216,6 +1269,7 @@ abstract class ClassfileParser {
           in.skip(attrLen)
         case tpnme.ScalaATTR =>
           isScalaRaw = true
+          loaders.platform.classFileInfoParsed(file, clazz, ScalaRawClass(this.currentClass.toString))
         case tpnme.InnerClassesATTR if !isScala =>
           val entries = u2
           for (i <- 0 until entries) {

src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala

@@ -112,11 +112,13 @@ trait Analyzer extends AnyRef
         try {
           val typer = newTyper(rootContext(unit))
           unit.body = typer.typed(unit.body)
-          for (workItem <- unit.toCheck) workItem()
-          if (settings.warnUnusedImport)
-            warnUnusedImports(unit)
-          if (settings.warnUnused.isSetByUser)
-            new checkUnused(typer).apply(unit)
+          if (!settings.Youtline.value) {
+            for (workItem <- unit.toCheck) workItem()
+            if (settings.warnUnusedImport)
+              warnUnusedImports(unit)
+            if (settings.warnUnused.isSetByUser)
+              new checkUnused(typer).apply(unit)
+          }
         }
         finally {
           unit.toCheck.clear()