Introduce Best Effort compilation options

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -21,6 +21,8 @@ class GenBCode extends Phase { self =>
 
  override def description: String = GenBCode.description
 
+ override def isRunnable(using Context) = super.isRunnable && !ctx.usedBestEffortTasty
+
  private val superCallsMap = new MutableSymbolMap[Set[ClassSymbol]]
  def registerSuperCall(sym: Symbol, calls: ClassSymbol): Unit = {
  val old = superCallsMap.getOrElse(sym, Set.empty)

compiler/src/dotty/tools/backend/sjs/GenSJSIR.scala

@@ -12,7 +12,7 @@ class GenSJSIR extends Phase {
  override def description: String = GenSJSIR.description
 
  override def isRunnable(using Context): Boolean =
- super.isRunnable && ctx.settings.scalajs.value
+ super.isRunnable && ctx.settings.scalajs.value && !ctx.usedBestEffortTasty
 
  def run(using Context): Unit =
  new JSCodeGen().run()

compiler/src/dotty/tools/dotc/Driver.scala

@@ -39,6 +39,9 @@ class Driver {
  catch
  case ex: FatalError =>
  report.error(ex.getMessage.nn) // signals that we should fail compilation.
+ case ex: Throwable if ctx.usedBestEffortTasty =>
+ report.bestEffortError(ex, "Some best-effort tasty files were not able to be read.")
+ throw ex
  case ex: TypeError if !runOrNull.enrichedErrorMessage =>
  println(runOrNull.enrichErrorMessage(s"${ex.toMessage} while compiling ${files.map(_.path).mkString(", ")}"))
  throw ex
@@ -102,8 +105,8 @@ class Driver {
  None
  else file.ext match
  case FileExtension.Jar => Some(file.path)
- case FileExtension.Tasty =>
- TastyFileUtil.getClassPath(file) match
+ case FileExtension.Tasty | FileExtension.Betasty =>
+ TastyFileUtil.getClassPath(file, ctx.withBestEffortTasty) match
  case Some(classpath) => Some(classpath)
  case _ =>
  report.error(em"Could not load classname from: ${file.path}")

compiler/src/dotty/tools/dotc/Run.scala

@@ -329,9 +329,13 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
  val profiler = ctx.profiler
  var phasesWereAdjusted = false
 
+ var forceReachPhaseMaybe =
+ if (ctx.isBestEffort && phases.exists(_.phaseName == "typer")) Some("typer")
+ else None
+
  for phase <- allPhases do
  doEnterPhase(phase)
- val phaseWillRun = phase.isRunnable
+ val phaseWillRun = phase.isRunnable || forceReachPhaseMaybe.nonEmpty
  if phaseWillRun then
  Stats.trackTime(s"phase time ms/$phase") {
  val start = System.currentTimeMillis
@@ -344,6 +348,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
  def printCtx(unit: CompilationUnit) = phase.printingContext(
  ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
  lastPrintedTree = printTree(lastPrintedTree)(using printCtx(unit))
+
+ if forceReachPhaseMaybe.contains(phase.phaseName) then
+ forceReachPhaseMaybe = None
+
  report.informTime(s"$phase ", start)
  Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
  for (unit <- units)

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -919,12 +919,12 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
  else cpy.PackageDef(tree)(pid, slicedStats) :: Nil
  case tdef: TypeDef =>
  val sym = tdef.symbol
- assert(sym.isClass)
+ assert(sym.isClass || ctx.tolerateErrorsForBestEffort)
  if (cls == sym || cls == sym.linkedClass) tdef :: Nil
  else Nil
  case vdef: ValDef =>
  val sym = vdef.symbol
- assert(sym.is(Module))
+ assert(sym.is(Module) || ctx.tolerateErrorsForBestEffort)
  if (cls == sym.companionClass || cls == sym.moduleClass) vdef :: Nil
  else Nil
  case tree =>

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -47,7 +47,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
  case _: RefTree | _: GenericApply | _: Inlined | _: Hole =>
  ta.assignType(untpd.Apply(fn, args), fn, args)
  case _ =>
- assert(ctx.reporter.errorsReported)
+ assert(ctx.reporter.errorsReported || ctx.tolerateErrorsForBestEffort)
  ta.assignType(untpd.Apply(fn, args), fn, args)
 
  def TypeApply(fn: Tree, args: List[Tree])(using Context): TypeApply = fn match
@@ -56,7 +56,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
  case _: RefTree | _: GenericApply =>
  ta.assignType(untpd.TypeApply(fn, args), fn, args)
  case _ =>
- assert(ctx.reporter.errorsReported, s"unexpected tree for type application: $fn")
+ assert(ctx.reporter.errorsReported || ctx.tolerateErrorsForBestEffort, s"unexpected tree for type application: $fn")
  ta.assignType(untpd.TypeApply(fn, args), fn, args)
 
  def Literal(const: Constant)(using Context): Literal =

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -285,7 +285,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[BinaryFil
  protected def createFileEntry(file: AbstractFile): BinaryFileEntry = BinaryFileEntry(file)
 
  protected def isMatchingFile(f: JFile): Boolean =
- f.isTasty || (f.isClass && !f.hasSiblingTasty)
+ f.isTasty || f.isBestEffortTasty || (f.isClass && !f.hasSiblingTasty)
 
  private[dotty] def classes(inPackage: PackageName): Seq[BinaryFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -23,8 +23,12 @@ object FileUtils {
 
  def hasTastyExtension: Boolean = file.ext.isTasty
 
+ def hasBetastyExtension: Boolean = file.ext.isBetasty
+
  def isTasty: Boolean = !file.isDirectory && hasTastyExtension
 
+ def isBestEffortTasty: Boolean = !file.isDirectory && hasBetastyExtension
+
  def isScalaBinary: Boolean = file.isClass || file.isTasty
 
  def isScalaOrJavaSource: Boolean = !file.isDirectory && file.ext.isScalaOrJava
@@ -55,6 +59,9 @@ object FileUtils {
 
  def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
 
+ def isBestEffortTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_BETASTY)
+
+
  /**
  * Returns if there is an existing sibling `.tasty` file.
  */
@@ -69,6 +76,7 @@ object FileUtils {
  private val SUFFIX_CLASS = ".class"
  private val SUFFIX_SCALA = ".scala"
  private val SUFFIX_TASTY = ".tasty"
+ private val SUFFIX_BETASTY = ".betasty"
  private val SUFFIX_JAVA = ".java"
  private val SUFFIX_SIG = ".sig"
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -414,6 +414,9 @@ private sealed trait YSettings:
  val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting(ForkSetting, "Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
  //.withPostSetHook( _ => YprofileEnabled.value = true )
 
+ val YbestEffort: Setting[Boolean] = BooleanSetting(ForkSetting, "Ybest-effort", "Enable best-effort compilation attempting to produce betasty to the META-INF/best-effort directory, regardless of errors, as part of the pickler phase.")
+ val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Ywith-best-effort-tasty", "Allow to compile using best-effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
+
  // Experimental language features
  val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-kind-polymorphism", "Disable kind polymorphism.")
  val YexplicitNulls: Setting[Boolean] = BooleanSetting(ForkSetting, "Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -474,6 +474,21 @@ object Contexts {
 
  /** Is the flexible types option set? */
  def flexibleTypes: Boolean = base.settings.YexplicitNulls.value && !base.settings.YnoFlexibleTypes.value
+
+ /** Is the best-effort option set? */
+ def isBestEffort: Boolean = base.settings.YbestEffort.value
+
+ /** Is the with-best-effort-tasty option set? */
+ def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value
+
+ /** Were any best effort tasty dependencies used during compilation? */
+ def usedBestEffortTasty: Boolean = base.usedBestEffortTasty
+
+ /** Confirm that a best effort tasty dependency was used during compilation. */
+ def setUsedBestEffortTasty(): Unit = base.usedBestEffortTasty = true
+
+ /** Is either the best-effort option set or .betasty files were used during compilation? */
+ def tolerateErrorsForBestEffort = isBestEffort || usedBestEffortTasty
 
  /** A fresh clone of this context embedded in this context. */
  def fresh: FreshContext = freshOver(this)
@@ -960,6 +975,9 @@ object Contexts {
  val sources: util.HashMap[AbstractFile, SourceFile] = util.HashMap[AbstractFile, SourceFile]()
  val files: util.HashMap[TermName, AbstractFile] = util.HashMap()
 
+ /** Was best effort file used during compilation? */
+ private[core] var usedBestEffortTasty = false
+
  // Types state
  /** A table for hash consing unique types */
  private[core] val uniques: Uniques = Uniques()

compiler/src/dotty/tools/dotc/core/DenotTransformers.scala

@@ -28,6 +28,8 @@ object DenotTransformers {
 
  /** The transformation method */
  def transform(ref: SingleDenotation)(using Context): SingleDenotation
+
+ override def isRunnable(using Context) = super.isRunnable && !ctx.usedBestEffortTasty
  }
 
  /** A transformer that only transforms the info field of denotations */

compiler/src/dotty/tools/dotc/core/Denotations.scala

@@ -719,7 +719,8 @@ object Denotations {
  ctx.runId >= validFor.runId
  || ctx.settings.YtestPickler.value // mixing test pickler with debug printing can travel back in time
  || ctx.mode.is(Mode.Printing) // no use to be picky when printing error messages
- || symbol.isOneOf(ValidForeverFlags),
+ || symbol.isOneOf(ValidForeverFlags)
+ || ctx.tolerateErrorsForBestEffort,
  s"denotation $this invalid in run ${ctx.runId}. ValidFor: $validFor")
  var d: SingleDenotation = this
  while ({

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -720,12 +720,16 @@ object SymDenotations {
  * TODO: Find a more robust way to characterize self symbols, maybe by
  * spending a Flag on them?
  */
- final def isSelfSym(using Context): Boolean = owner.infoOrCompleter match {
- case ClassInfo(_, _, _, _, selfInfo) =>
- selfInfo == symbol ||
- selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
- case _ => false
- }
+ final def isSelfSym(using Context): Boolean =
+ if !ctx.isBestEffort || exists then
+ owner.infoOrCompleter match {
+ case ClassInfo(_, _, _, _, selfInfo) =>
+ selfInfo == symbol ||
+ selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
+ case _ => false
+ }
+ else false
+
 
  /** Is this definition contained in `boundary`?
  * Same as `ownersIterator contains boundary` but more efficient.
@@ -2003,7 +2007,7 @@ object SymDenotations {
  case p :: parents1 =>
  p.classSymbol match {
  case pcls: ClassSymbol => builder.addAll(pcls.baseClasses)
- case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive), s"$this has non-class parent: $p")
+ case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive) || ctx.tolerateErrorsForBestEffort, s"$this has non-class parent: $p")
  }
  traverse(parents1)
  case nil =>

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -7,7 +7,7 @@ import java.nio.channels.ClosedByInterruptException
 
 import scala.util.control.NonFatal
 
-import dotty.tools.dotc.classpath.FileUtils.hasTastyExtension
+import dotty.tools.dotc.classpath.FileUtils.{hasTastyExtension, hasBetastyExtension}
 import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile }
 import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions
 
@@ -26,6 +26,7 @@ import parsing.JavaParsers.OutlineJavaParser
 import parsing.Parsers.OutlineParser
 import dotty.tools.tasty.{TastyHeaderUnpickler, UnpickleException, UnpicklerConfig, TastyVersion}
 import dotty.tools.dotc.core.tasty.TastyUnpickler
+import dotty.tools.tasty.besteffort.BestEffortTastyHeaderUnpickler
 
 object SymbolLoaders {
  import ast.untpd.*
@@ -198,7 +199,7 @@ object SymbolLoaders {
  enterToplevelsFromSource(owner, nameOf(classRep), src)
  case (Some(bin), _) =>
  val completer =
- if bin.hasTastyExtension then ctx.platform.newTastyLoader(bin)
+ if bin.hasTastyExtension || bin.hasBetastyExtension then ctx.platform.newTastyLoader(bin)
  else ctx.platform.newClassLoader(bin)
  enterClassAndModule(owner, nameOf(classRep), completer)
  }
@@ -261,7 +262,8 @@ object SymbolLoaders {
  (idx + str.TOPLEVEL_SUFFIX.length + 1 != name.length || !name.endsWith(str.TOPLEVEL_SUFFIX))
  }
 
- def maybeModuleClass(classRep: ClassRepresentation): Boolean = classRep.name.last == '$'
+ def maybeModuleClass(classRep: ClassRepresentation): Boolean =
+ classRep.name.nonEmpty && classRep.name.last == '$'
 
  private def enterClasses(root: SymDenotation, packageName: String, flat: Boolean)(using Context) = {
  def isAbsent(classRep: ClassRepresentation) =
@@ -416,34 +418,45 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader {
 }
 
 class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
-
+ val isBestEffortTasty = tastyFile.hasBetastyExtension
  private val unpickler: tasty.DottyUnpickler =
  handleUnpicklingExceptions:
  val tastyBytes = tastyFile.toByteArray
- new tasty.DottyUnpickler(tastyFile, tastyBytes) // reads header and name table
+ new tasty.DottyUnpickler(tastyFile, tastyBytes, isBestEffortTasty) // reads header and name table
 
  val compilationUnitInfo: CompilationUnitInfo | Null = unpickler.compilationUnitInfo
 
- def description(using Context): String = "TASTy file " + tastyFile.toString
+ def description(using Context): String =
+ if isBestEffortTasty then "Best Effort TASTy file " + tastyFile.toString
+ else "TASTy file " + tastyFile.toString
 
  override def doComplete(root: SymDenotation)(using Context): Unit =
  handleUnpicklingExceptions:
- checkTastyUUID()
  val (classRoot, moduleRoot) = rootDenots(root.asClass)
- unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
- if mayLoadTreesFromTasty then
- classRoot.classSymbol.rootTreeOrProvider = unpickler
- moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+ if (!isBestEffortTasty || ctx.withBestEffortTasty) then
+ val tastyBytes = tastyFile.toByteArray
+ unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
+ if mayLoadTreesFromTasty || isBestEffortTasty then
+ classRoot.classSymbol.rootTreeOrProvider = unpickler
+ moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+ if isBestEffortTasty then
+ checkBeTastyUUID(tastyFile, tastyBytes)
+ ctx.setUsedBestEffortTasty()
+ else
+ checkTastyUUID()
+ else
+ report.error(em"Cannot read Best Effort TASTy $tastyFile without the ${ctx.settings.YwithBestEffortTasty.name} option")
 
  private def handleUnpicklingExceptions[T](thunk: =>T): T =
  try thunk
  catch case e: RuntimeException =>
+ val tastyType = if (isBestEffortTasty) "Best Effort TASTy" else "TASTy"
  val message = e match
  case e: UnpickleException =>
- s"""TASTy file ${tastyFile.canonicalPath} could not be read, failing with:
+ s"""$tastyType file ${tastyFile.canonicalPath} could not be read, failing with:
  | ${Option(e.getMessage).getOrElse("")}""".stripMargin
  case _ =>
- s"""TASTy file ${tastyFile.canonicalPath} is broken, reading aborted with ${e.getClass}
+ s"""$tastyFile file ${tastyFile.canonicalPath} is broken, reading aborted with ${e.getClass}
  | ${Option(e.getMessage).getOrElse("")}""".stripMargin
  throw IOException(message, e)
 
@@ -460,6 +473,9 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
  // tasty file compiled by `-Yearly-tasty-output-write` comes from an early output jar.
  report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
+ private def checkBeTastyUUID(tastyFile: AbstractFile, tastyBytes: Array[Byte])(using Context): Unit =
+ new BestEffortTastyHeaderUnpickler(tastyBytes).readHeader()
+
  private def mayLoadTreesFromTasty(using Context): Boolean =
  ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value
 }

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -747,7 +747,9 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
  assert(!etp.isInstanceOf[WildcardType] || inSigName, i"Unexpected WildcardType erasure for $tp")
  etp
 
- /** Like translucentSuperType, but issue a fatal error if it does not exist. */
+ /** Like translucentSuperType, but issue a fatal error if it does not exist.
+ * If using the best-effort option, the fatal error will not be issued.
+ */
  private def checkedSuperType(tp: TypeProxy)(using Context): Type =
  val tp1 = tp.translucentSuperType
  if !tp1.exists then
@@ -756,7 +758,8 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
  MissingType(tycon.prefix, tycon.name)
  case _ =>
  TypeError(em"Cannot resolve reference to $tp")
- throw typeErr
+ if ctx.isBestEffort then report.error(typeErr.toMessage)
+ else throw typeErr
  tp1
 
  /** Widen term ref, skipping any `()` parameter of an eventual getter. Used to erase a TermRef.

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -3149,7 +3149,8 @@ object Types extends TypeUtils {
  if (ctx.erasedTypes) tref
  else cls.info match {
  case cinfo: ClassInfo => cinfo.selfType
- case _: ErrorType | NoType if ctx.mode.is(Mode.Interactive) => cls.info
+ case _: ErrorType | NoType
+ if ctx.mode.is(Mode.Interactive) || ctx.tolerateErrorsForBestEffort => cls.info
  // can happen in IDE if `cls` is stale
  }
 
@@ -3719,8 +3720,9 @@ object Types extends TypeUtils {
 
  def apply(tp1: Type, tp2: Type, soft: Boolean)(using Context): OrType = {
  def where = i"in union $tp1 | $tp2"
- expectValueTypeOrWildcard(tp1, where)
- expectValueTypeOrWildcard(tp2, where)
+ if !ctx.usedBestEffortTasty then
+ expectValueTypeOrWildcard(tp1, where)
+ expectValueTypeOrWildcard(tp2, where)
  assertUnerased()
  unique(new CachedOrType(tp1, tp2, soft))
  }