Handle -continue-building-after-errors and add mechanism for stopping build on failure

Sources/SwiftDriver/Driver/CompilerMode.swift

@@ -80,6 +80,15 @@ extension CompilerMode {
  }
  }
 
+ var isBatchCompile: Bool {
+ switch self {
+ case .batchCompile:
+ return true
+ case .singleCompile, .standardCompile, .compilePCM, .immediate, .repl:
+ return false
+ }
+ }
+
  // Whether this compilation mode supports the use of bridging pre-compiled
  // headers.
  public var supportsBridgingPCH: Bool {

Sources/SwiftDriver/Driver/Driver.swift

@@ -183,6 +183,9 @@ public struct Driver {
  /// The specified maximum number of parallel jobs to execute.
  @_spi(Testing) public let numParallelJobs: Int?
 
+ /// Whether jobs should continue to be executed after a failure.
+ @_spi(Testing) public let continueBuildingAfterErrors: Bool
+
  /// The set of sanitizers that were requested
  let enabledSanitizers: Set<Sanitizer>
 
@@ -408,6 +411,8 @@ public struct Driver {
  self.numThreads = Self.determineNumThreads(&parsedOptions, compilerMode: compilerMode, diagnosticsEngine: diagnosticEngine)
  self.numParallelJobs = Self.determineNumParallelJobs(&parsedOptions, diagnosticsEngine: diagnosticEngine, env: env)
 
+ self.continueBuildingAfterErrors = Self.computeContinueBuildingAfterErrors(&parsedOptions, compilerMode: compilerMode)
+
  Self.validateWarningControlArgs(&parsedOptions, diagnosticEngine: diagnosticEngine)
  Self.validateProfilingArgs(&parsedOptions,
  fileSystem: fileSystem,
@@ -896,6 +901,7 @@ extension Driver {
  workload: .init(allJobs, incrementalCompilationState),
  delegate: createToolExecutionDelegate(),
  numParallelJobs: numParallelJobs ?? 1,
+ continueBuildingAfterErrors: continueBuildingAfterErrors,
  forceResponseFiles: forceResponseFiles,
  recordedInputModificationDates: recordedInputModificationDates)
  }
@@ -1357,6 +1363,25 @@ extension Driver {
 
  return numJobs
  }
+
+ static func computeContinueBuildingAfterErrors(
+ _ parsedOptions: inout ParsedOptions,
+ compilerMode: CompilerMode
+ ) -> Bool {
+ // Note: Batch mode handling of serialized diagnostics requires that all
+ // batches get to run, in order to make sure that all diagnostics emitted
+ // during the compilation end up in at least one serialized diagnostic file.
+ // Therefore, treat batch mode as implying -continue-building-after-errors.
+ // (This behavior could be limited to only when serialized diagnostics are
+ // being emitted, but this seems more consistent and less surprising for
+ // users.)
+ // FIXME: We don't really need (or want) a full ContinueBuildingAfterErrors.
+ // If we fail to precompile a bridging header, for example, there's no need
+ // to go on to compilation of source files, and if compilation of source files
+ // fails, we shouldn't try to link. Instead, we'd want to let all jobs finish
+ // but not schedule any new ones.
+ return compilerMode.isBatchCompile || parsedOptions.contains(.continueBuildingAfterErrors)
+ }
 }
 
 extension Diagnostic.Message {

Sources/SwiftDriver/Execution/DriverExecutor.swift

@@ -27,6 +27,7 @@ public protocol DriverExecutor {
  func execute(workload: DriverExecutorWorkload,
  delegate: JobExecutionDelegate,
  numParallelJobs: Int,
+ continueBuildingAfterErrors: Bool,
  forceResponseFiles: Bool,
  recordedInputModificationDates: [TypedVirtualPath: Date]
  ) throws
@@ -35,6 +36,7 @@ public protocol DriverExecutor {
  func execute(jobs: [Job],
  delegate: JobExecutionDelegate,
  numParallelJobs: Int,
+ continueBuildingAfterErrors: Bool,
  forceResponseFiles: Bool,
  recordedInputModificationDates: [TypedVirtualPath: Date]
  ) throws
@@ -92,13 +94,15 @@ extension DriverExecutor {
  jobs: [Job],
  delegate: JobExecutionDelegate,
  numParallelJobs: Int,
+ continueBuildingAfterErrors: Bool,
  forceResponseFiles: Bool,
  recordedInputModificationDates: [TypedVirtualPath: Date]
  ) throws {
  try execute(
  workload: .all(jobs),
  delegate: delegate,
  numParallelJobs: numParallelJobs,
+ continueBuildingAfterErrors: continueBuildingAfterErrors,
  forceResponseFiles: forceResponseFiles,
  recordedInputModificationDates: recordedInputModificationDates)
  }

Sources/SwiftDriverExecution/MultiJobExecutor.swift

@@ -67,6 +67,9 @@ public final class MultiJobExecutor {
  /// Operation queue for executing tasks in parallel.
  let jobQueue: OperationQueue
 
+ /// Whether jobs should continue being scheduled after one fails.
+ let continueBuildingAfterErrors: Bool
+
  /// The process set to use when launching new processes.
  let processSet: ProcessSet?
 
@@ -87,6 +90,9 @@ public final class MultiJobExecutor {
  /// any newly-required job. Set only once.
  private(set) var executeAllJobsTaskBuildEngine: LLTaskBuildEngine? = nil
 
+ /// Should new jobs immediately fail.
+ private(set) var isCancelled: Bool = false
+
 
  init(
  argsResolver: ArgsResolver,
@@ -95,6 +101,7 @@ public final class MultiJobExecutor {
  workload: DriverExecutorWorkload,
  executorDelegate: JobExecutionDelegate,
  jobQueue: OperationQueue,
+ continueBuildingAfterErrors: Bool,
  processSet: ProcessSet?,
  forceResponseFiles: Bool,
  recordedInputModificationDates: [TypedVirtualPath: Date],
@@ -114,6 +121,7 @@ public final class MultiJobExecutor {
  self.fileSystem = fileSystem
  self.executorDelegate = executorDelegate
  self.jobQueue = jobQueue
+ self.continueBuildingAfterErrors = continueBuildingAfterErrors
  self.processSet = processSet
  self.forceResponseFiles = forceResponseFiles
  self.recordedInputModificationDates = recordedInputModificationDates
@@ -199,6 +207,10 @@ public final class MultiJobExecutor {
  executeAllJobsTaskBuildEngine = engine
  }
 
+ fileprivate func cancel() {
+ isCancelled = true
+ }
+
  /// After a job finishes, an incremental build may discover more jobs are needed, or if all compilations
  /// are done, will need to then add in the post-compilation rules.
  fileprivate func addRuleBeyondMandatoryCompiles(
@@ -239,6 +251,9 @@ public final class MultiJobExecutor {
  /// The number of jobs to run in parallel.
  private let numParallelJobs: Int
 
+ /// Whether jobs should continue being scheduled after one fails.
+ private let continueBuildingAfterErrors: Bool
+
  /// The process set to use when launching new processes.
  private let processSet: ProcessSet?
 
@@ -260,6 +275,7 @@ public final class MultiJobExecutor {
  executorDelegate: JobExecutionDelegate,
  diagnosticsEngine: DiagnosticsEngine,
  numParallelJobs: Int? = nil,
+ continueBuildingAfterErrors: Bool = false,
  processSet: ProcessSet? = nil,
  forceResponseFiles: Bool = false,
  recordedInputModificationDates: [TypedVirtualPath: Date] = [:],
@@ -270,6 +286,7 @@ public final class MultiJobExecutor {
  self.executorDelegate = executorDelegate
  self.diagnosticsEngine = diagnosticsEngine
  self.numParallelJobs = numParallelJobs ?? 1
+ self.continueBuildingAfterErrors = continueBuildingAfterErrors
  self.processSet = processSet
  self.forceResponseFiles = forceResponseFiles
  self.recordedInputModificationDates = recordedInputModificationDates
@@ -304,6 +321,7 @@ public final class MultiJobExecutor {
  workload: workload,
  executorDelegate: executorDelegate,
  jobQueue: jobQueue,
+ continueBuildingAfterErrors: continueBuildingAfterErrors,
  processSet: processSet,
  forceResponseFiles: forceResponseFiles,
  recordedInputModificationDates: recordedInputModificationDates,
@@ -428,11 +446,14 @@ class ExecuteJobRule: LLBuildRule {
 
  override func inputsAvailable(_ engine: LLTaskBuildEngine) {
  // Return early any of the input failed.
- guard allInputsSucceeded else {
+ guard allInputsSucceeded, !context.isCancelled else {
  return engine.taskIsComplete(DriverBuildValue.jobExecution(success: false))
  }
 
  context.jobQueue.addOperation {
+ guard !self.context.isCancelled else {
+ return engine.taskIsComplete(DriverBuildValue.jobExecution(success: false))
+ }
  self.executeJob(engine)
  }
  }
@@ -532,6 +553,11 @@ class ExecuteJobRule: LLBuildRule {
  }
 
  engine.taskIsComplete(value)
+
+ if !value.success, !context.continueBuildingAfterErrors {
+ // Allow currently running jobs to finish, but don't execute anymore
+ context.cancel()
+ }
  }
 }
 

Sources/SwiftDriverExecution/SwiftDriverExecutor.swift

@@ -60,6 +60,7 @@ public final class SwiftDriverExecutor: DriverExecutor {
  public func execute(workload: DriverExecutorWorkload,
  delegate: JobExecutionDelegate,
  numParallelJobs: Int = 1,
+ continueBuildingAfterErrors: Bool,
  forceResponseFiles: Bool = false,
  recordedInputModificationDates: [TypedVirtualPath: Date] = [:]
  ) throws {
@@ -69,6 +70,7 @@ public final class SwiftDriverExecutor: DriverExecutor {
  executorDelegate: delegate,
  diagnosticsEngine: diagnosticsEngine,
  numParallelJobs: numParallelJobs,
+ continueBuildingAfterErrors: continueBuildingAfterErrors,
  processSet: processSet,
  forceResponseFiles: forceResponseFiles,
  recordedInputModificationDates: recordedInputModificationDates)

Tests/SwiftDriverTests/JobExecutorTests.swift

@@ -306,14 +306,98 @@ final class JobExecutorTests: XCTestCase {
  $0 <<< "let bar = 3"
  }
 
- // FIXME: It's unfortunate we diagnose this twice, once for each job which uses the input.
- verifier.expect(.error("input file '\(other.description)' was modified during the build"))
  verifier.expect(.error("input file '\(other.description)' was modified during the build"))
  XCTAssertThrowsError(try driver.run(jobs: jobs))
  }
  }
  }
 
+ func testStopAfterError() throws {
+ try withTemporaryDirectory { path in
+ let foo = path.appending(component: "foo.swift")
+ try localFileSystem.writeFileContents(foo) {
+ $0 <<< """
+ struct Foo {
+ let bar: Int
+ init() {
+ self.bar = self.bar
+ }
+ }
+ """
+ }
+ let bar = path.appending(component: "bar.swift")
+ try localFileSystem.writeFileContents(bar) {
+ $0 <<< """
+ struct Bar {
+ let baz: Int
+ init() {
+ self.baz = self.baz
+ }
+ }
+ """
+ }
+
+ // Replace the error stream with one we capture here.
+ let errorStream = stderrStream
+ let errorOutputFile = path.appending(component: "dummy_error_stream")
+ TSCBasic.stderrStream = try! ThreadSafeOutputByteStream(LocalFileOutputByteStream(errorOutputFile))
+ // Restore the error stream to what it was
+ defer { TSCBasic.stderrStream = errorStream }
+
+ try assertNoDriverDiagnostics(args: "swiftc", foo.pathString, bar.pathString) { driver in
+ let jobs = try driver.planBuild()
+ XCTAssertThrowsError(try driver.run(jobs: jobs))
+ }
+
+ let invocationError = try localFileSystem.readFileContents(errorOutputFile).description
+ XCTAssertTrue(invocationError.contains("self.bar = self.bar"))
+ XCTAssertFalse(invocationError.contains("self.baz = self.baz"))
+ }
+ }
+
+ func testContinueAfterError() throws {
+ try withTemporaryDirectory { path in
+ let foo = path.appending(component: "foo.swift")
+ try localFileSystem.writeFileContents(foo) {
+ $0 <<< """
+ struct Foo {
+ let bar: Int
+ init() {
+ self.bar = self.bar
+ }
+ }
+ """
+ }
+ let bar = path.appending(component: "bar.swift")
+ try localFileSystem.writeFileContents(bar) {
+ $0 <<< """
+ struct Bar {
+ let baz: Int
+ init() {
+ self.baz = self.baz
+ }
+ }
+ """
+ }
+
+ // Replace the error stream with one we capture here.
+ let errorStream = stderrStream
+ let errorOutputFile = path.appending(component: "dummy_error_stream")
+ TSCBasic.stderrStream = try! ThreadSafeOutputByteStream(LocalFileOutputByteStream(errorOutputFile))
+ // Restore the error stream to what it was
+ defer { TSCBasic.stderrStream = errorStream }
+
+ try assertNoDriverDiagnostics(args: "swiftc", "-continue-building-after-errors", foo.pathString, bar.pathString) { driver in
+ let jobs = try driver.planBuild()
+ XCTAssertThrowsError(try driver.run(jobs: jobs))
+ }
+
+ let invocationError = try localFileSystem.readFileContents(errorOutputFile).description
+ XCTAssertTrue(invocationError.contains("self.bar = self.bar"))
+ XCTAssertTrue(invocationError.contains("self.baz = self.baz"))
+ }
+ }
+
  func testTemporaryFileWriting() throws {
  try withTemporaryDirectory { path in
  let resolver = try ArgsResolver(fileSystem: localFileSystem, temporaryDirectory: .absolute(path))

Tests/SwiftDriverTests/SwiftDriverTests.swift

@@ -1466,6 +1466,11 @@ final class SwiftDriverTests: XCTestCase {
  }
  }
 
+ func testBatchModeContinueAfterErrors() throws {
+ let driver = try Driver(args: ["swiftc", "foo1.swift", "bar1.swift", "-enable-batch-mode"])
+ XCTAssert(driver.continueBuildingAfterErrors)
+ }
+
  func testSingleThreadedWholeModuleOptimizationCompiles() throws {
  var driver1 = try Driver(args: ["swiftc", "-whole-module-optimization", "foo.swift", "bar.swift", "-module-name", "Test", "-target", "x86_64-apple-macosx10.15", "-emit-module-interface", "-emit-objc-header-path", "Test-Swift.h", "-emit-private-module-interface-path", "Test.private.swiftinterface"])
  let plannedJobs = try driver1.planBuild()
@@ -2861,6 +2866,7 @@ final class SwiftDriverTests: XCTestCase {
  func execute(workload: DriverExecutorWorkload,
  delegate: JobExecutionDelegate,
  numParallelJobs: Int,
+ continueBuildingAfterErrors: Bool,
  forceResponseFiles: Bool,
  recordedInputModificationDates: [TypedVirtualPath : Date]) throws {
  fatalError()