Merge pull request #334 from davidungar/incr-10-27c

[Incremental] Redoing do incremental builds

Author: David Ungar

Sources/SwiftDriver/CMakeLists.txt

@@ -44,6 +44,7 @@ add_library(SwiftDriver
  "Incremental Compilation/ModuleDependencyGraph.swift"
  "Incremental Compilation/Multidictionary.swift"
  "Incremental Compilation/SourceFileDependencyGraph.swift"
+ "Incremental Compilation/SynchronizedQueue.swift"
  "Incremental Compilation/TwoDMap.swift"
 
  Jobs/AutolinkExtractJob.swift

Sources/SwiftDriver/Driver/CompilerMode.swift

@@ -71,6 +71,15 @@ extension CompilerMode {
  }
  }
 
+ public var batchModeInfo: BatchModeInfo? {
+ switch self {
+ case let .batchCompile(info):
+ return info
+ default:
+ return nil
+ }
+ }
+
  // Whether this compilation mode supports the use of bridging pre-compiled
  // headers.
  public var supportsBridgingPCH: Bool {

Sources/SwiftDriver/Driver/Driver.swift

@@ -825,8 +825,9 @@ extension Driver {
  }
 
  if jobs.contains(where: { $0.requiresInPlaceExecution })
- // Only one job and no cleanup required
- || (jobs.count == 1 && !parsedOptions.hasArgument(.parseableOutput)) {
+ // Only one job and no cleanup required, e.g. not writing build record
+ || (jobs.count == 1 && !parsedOptions.hasArgument(.parseableOutput) &&
+ buildRecordInfo == nil) {
  assert(jobs.count == 1, "Cannot execute in place for multi-job build plans")
  var job = jobs[0]
  // Require in-place execution for all single job plans.
@@ -837,17 +838,11 @@ extension Driver {
  return
  }
 
- // Create and use the tool execution delegate if one is not provided explicitly.
- let executorDelegate = createToolExecutionDelegate()
-
- // Perform the build
- try executor.execute(jobs: jobs,
- delegate: executorDelegate,
- numParallelJobs: numParallelJobs ?? 1,
- forceResponseFiles: forceResponseFiles,
- recordedInputModificationDates: recordedInputModificationDates)
+ try performTheBuild(allJobs: jobs, forceResponseFiles: forceResponseFiles)
 
- buildRecordInfo?.writeBuildRecord( jobs, nil)
+ buildRecordInfo?.writeBuildRecord(
+ jobs,
+ incrementalCompilationState?.skippedCompilationInputs)
 
  // If requested, warn for options that weren't used by the driver after the build is finished.
  if parsedOptions.hasArgument(.driverWarnUnusedOptions) {
@@ -875,6 +870,35 @@ extension Driver {
  diagnosticEngine: diagnosticEngine)
  }
 
+ private mutating func performTheBuild(
+ allJobs: [Job],
+ forceResponseFiles: Bool
+ ) throws {
+ // Create and use the tool execution delegate if one is not provided explicitly.
+ let executorDelegate = createToolExecutionDelegate()
+
+ func execute(jobs: [Job]) throws {
+ try executor.execute(jobs: jobs,
+ delegate: executorDelegate,
+ numParallelJobs: numParallelJobs ?? 1,
+ forceResponseFiles: forceResponseFiles,
+ recordedInputModificationDates: recordedInputModificationDates)
+ }
+
+ guard let incrementalCompilationState = incrementalCompilationState else {
+ try execute(jobs: allJobs)
+ return
+ }
+ while let jobs = incrementalCompilationState.preOrCompileJobs.removeAll() {
+ try execute(jobs: formBatchedJobs(jobs, forIncremental: true))
+ }
+ guard let postCompileJobs = incrementalCompilationState.postCompileJobs
+ else {
+ fatalError("planning must have finished by now")
+ }
+ try execute(jobs: postCompileJobs)
+ }
+
  private func printBindings(_ job: Job) {
  stdoutStream <<< #"# ""# <<< targetTriple.triple
  stdoutStream <<< #"" - ""# <<< job.tool.basename

Sources/SwiftDriver/Driver/OutputFileMap.swift

@@ -77,6 +77,17 @@ public struct OutputFileMap: Hashable, Codable {
  }))
  }
 
+ /// Slow, but only for debugging output
+ public func getInput(outputFile: VirtualPath) -> VirtualPath? {
+ entries
+ .compactMap {
+ $0.value.values.contains(outputFile)
+ ? $0.key
+ : nil
+ }
+ .first
+ }
+
  /// Load the output file map at the given path.
  @_spi(Testing) public static func load(
  fileSystem: FileSystem,

Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift

@@ -68,7 +68,7 @@ struct ToolExecutionDelegate: JobExecutionDelegate {
  }
 
  public func jobFinished(job: Job, result: ProcessResult, pid: Int) {
- if showJobLifecycle {
+  if showJobLifecycle {
  diagnosticEngine.emit(.remark_job_lifecycle("Finished", job))
  }
 
@@ -116,6 +116,6 @@ struct ToolExecutionDelegate: JobExecutionDelegate {
 fileprivate extension Diagnostic.Message {
  static func remark_job_lifecycle(_ what: String, _ job: Job
  ) -> Diagnostic.Message {
- .remark("\(what) \(job)")
+ .remark("\(what) \(job.descriptionForLifecycle)")
  }
 }

Sources/SwiftDriver/Incremental Compilation/BuildRecord.swift

@@ -129,22 +129,6 @@ public extension BuildRecord {
  }
  )
  }
-
- /// Returns why it did not match
- @_spi(Testing) func mismatchReason(buildRecordInfo: BuildRecordInfo, inputFiles: [TypedVirtualPath]) -> String? {
- guard buildRecordInfo.actualSwiftVersion == self.swiftVersion else {
- return "the compiler version has changed from \(self.swiftVersion) to \(buildRecordInfo.actualSwiftVersion)"
- }
- guard buildRecordInfo.argsHash == self.argsHash else {
- return "different arguments were passed to the compiler"
- }
- let missingInputs = Set(self.inputInfos.keys).subtracting(inputFiles.map {$0.file})
- guard missingInputs.isEmpty else {
- return "the following inputs were used in the previous compilation but not in this one: "
- + missingInputs.map {$0.name} .joined(separator: ", ")
- }
- return nil
- }
 }
 
 // MARK: - Creating and writing a new map
@@ -235,11 +219,3 @@ extension Diagnostic.Message {
  .warning("next compile won't be incremental; could not write build record to \(path)")
  }
 }
-
-extension Diagnostic.Message {
- static func remark_could_not_read_build_record(_ path: VirtualPath,
- _ error: Error
- ) -> Diagnostic.Message {
- .remark("incremental compilation could not read build record at \(path): \(error.localizedDescription).")
- }
-}

Sources/SwiftDriver/Incremental Compilation/BuildRecordInfo.swift

@@ -128,15 +128,36 @@ import SwiftOptions
  }
 
 // TODO: Incremental too many names, buildRecord BuildRecord outofdatemap
- func populateOutOfDateBuildRecord() -> BuildRecord? {
+ func populateOutOfDateBuildRecord(
+ inputFiles: [TypedVirtualPath],
+ failed: (String) -> Void
+ ) -> BuildRecord? {
+ let outOfDateBuildRecord: BuildRecord
  do {
  let contents = try fileSystem.readFileContents(buildRecordPath).cString
- return try BuildRecord(contents: contents)
+ outOfDateBuildRecord = try BuildRecord(contents: contents)
  }
  catch {
- diagnosticEngine.emit(.remark_could_not_read_build_record(buildRecordPath, error))
+ failed("incremental compilation could not read build record at \(buildRecordPath): \(error.localizedDescription).")
  return nil
  }
+ guard actualSwiftVersion == outOfDateBuildRecord.swiftVersion else {
+ failed(
+ "the compiler version has changed from \(outOfDateBuildRecord.swiftVersion) to \(actualSwiftVersion)"
+ )
+ return nil
+ }
+ guard argsHash == outOfDateBuildRecord.argsHash else {
+ failed( "different arguments were passed to the compiler" )
+ return nil
+ }
+ let missingInputs = Set(outOfDateBuildRecord.inputInfos.keys).subtracting(inputFiles.map {$0.file})
+ guard missingInputs.isEmpty else {
+ failed( "the following inputs were used in the previous compilation but not in this one: "
+ + missingInputs.map {$0.basename} .joined(separator: ", "))
+ return nil
+ }
+ return outOfDateBuildRecord
  }
 
  func jobFinished(job: Job, result: ProcessResult) {