Introduce a severity level for issues, and a 'warning' severity (#931)

This introduces the concept of severity to the `Issue` type, represented by a new enum `Issue.Severity` with two cases: `.warning` and `.error`. Error is the default severity for all issues, matching current behavior, but warning is provided as a new option which does not cause the test the issue is associated with to be marked as a failure. In this PR, these are [SPI](https://github.com/swiftlang/swift-testing/blob/main/Documentation/SPI.md) but they could be considered for promotion to public API eventually. Additional work would be needed to permit test authors to record issues with severity < `.error`, since APIs like `Issue.record()` are not being modified at this time to allow customizing the severity. ### Motivation: There are certain situations where a problem may arise during a test that doesn't necessarily affect its outcome or signal an important problem, but is worth calling attention to. A specific example use case I have in mind is to allow the testing library to record a warning issue about problems with the arguments passed to a parameterized test, such as having duplicate arguments. ### Modifications: - Introduce `Issue.Severity` as an SPI enum. - Introduce an SPI property `severity` to `Issue` with default value `.error`. - Modify entry point logic to exit with `EXIT_SUCCESS` if all issues recorded had severity < `.error`. - Modify console output formatting logic and data structures to represent warning issues sensibly. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated. - [x] Add new tests

Author: stmontgomery

Sources/Testing/ABI/EntryPoints/ABIEntryPoint.swift

@@ -47,7 +47,7 @@ extension ABIv0 {
  /// callback.
  public static var entryPoint: EntryPoint {
  return { configurationJSON, recordHandler in
- try await Testing.entryPoint(
+ try await _entryPoint(
  configurationJSON: configurationJSON,
  recordHandler: recordHandler
  ) == EXIT_SUCCESS
@@ -87,7 +87,7 @@ typealias ABIEntryPoint_v0 = @Sendable (
 @usableFromInline func copyABIEntryPoint_v0() -> UnsafeMutableRawPointer {
  let result = UnsafeMutablePointer<ABIEntryPoint_v0>.allocate(capacity: 1)
  result.initialize { configurationJSON, recordHandler in
- try await entryPoint(
+ try await _entryPoint(
  configurationJSON: configurationJSON,
  eventStreamVersionIfNil: -1,
  recordHandler: recordHandler
@@ -104,7 +104,7 @@ typealias ABIEntryPoint_v0 = @Sendable (
 ///
 /// This function will be removed (with its logic incorporated into
 /// ``ABIv0/entryPoint-swift.type.property``) in a future update.
-private func entryPoint(
+private func _entryPoint(
  configurationJSON: UnsafeRawBufferPointer?,
  eventStreamVersionIfNil: Int? = nil,
  recordHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -20,6 +20,8 @@ private import _TestingInternals
 /// writes events to the standard error stream in addition to passing them
 /// to this function.
 ///
+/// - Returns: An exit code representing the result of running tests.
+///
 /// External callers cannot call this function directly. The can use
 /// ``ABIv0/entryPoint-swift.type.property`` to get a reference to an ABI-stable
 /// version of this function.
@@ -40,7 +42,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
 
  // Set up the event handler.
  configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
- if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
+ if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error {
  exitCode.withLock { exitCode in
  exitCode = EXIT_FAILURE
  }
@@ -270,6 +272,13 @@ public struct __CommandLineArguments_v0: Sendable {
  /// The value(s) of the `--skip` argument.
  public var skip: [String]?
 
+ /// Whether or not to include tests with the `.hidden` trait when constructing
+ /// a test filter based on these arguments.
+ ///
+ /// This property is intended for use in testing the testing library itself.
+ /// It is not parsed as a command-line argument.
+ var includeHiddenTests: Bool?
+
  /// The value of the `--repetitions` argument.
  public var repetitions: Int?
 
@@ -278,6 +287,13 @@ public struct __CommandLineArguments_v0: Sendable {
 
  /// The value of the `--experimental-attachments-path` argument.
  public var experimentalAttachmentsPath: String?
+
+ /// Whether or not the experimental warning issue severity feature should be
+ /// enabled.
+ ///
+ /// This property is intended for use in testing the testing library itself.
+ /// It is not parsed as a command-line argument.
+ var isWarningIssueRecordedEventEnabled: Bool?
 }
 
 extension __CommandLineArguments_v0: Codable {
@@ -517,6 +533,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
  filters.append(try testFilter(forRegularExpressions: args.skip, label: "--skip", membership: .excluding))
 
  configuration.testFilter = filters.reduce(.unfiltered) { $0.combining(with: $1) }
+ if args.includeHiddenTests == true {
+ configuration.testFilter.includeHiddenTests = true
+ }
 
  // Set up the iteration policy for the test run.
  var repetitionPolicy: Configuration.RepetitionPolicy = .once
@@ -547,6 +566,22 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
  configuration.exitTestHandler = ExitTest.handlerForEntryPoint()
 #endif
 
+ // Warning issues (experimental).
+ if args.isWarningIssueRecordedEventEnabled == true {
+ configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = true
+ } else {
+ switch args.eventStreamVersion {
+ case .some(...0):
+ // If the event stream version was explicitly specified to a value < 1,
+ // disable the warning issue event to maintain legacy behavior.
+ configuration.eventHandlingOptions.isWarningIssueRecordedEventEnabled = false
+ default:
+ // Otherwise the requested event stream version is ≥ 1, so don't change
+ // the warning issue event setting.
+ break
+ }
+ }
+
  return configuration
 }
 

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedIssue.swift

@@ -16,6 +16,19 @@ extension ABIv0 {
  /// assists in converting values to JSON; clients that consume this JSON are
  /// expected to write their own decoders.
  struct EncodedIssue: Sendable {
+ /// An enumeration representing the level of severity of a recorded issue.
+ ///
+ /// For descriptions of individual cases, see ``Issue/Severity-swift.enum``.
+ enum Severity: String, Sendable {
+ case warning
+ case error
+ }
+
+ /// The severity of this issue.
+ ///
+ /// - Warning: Severity is not yet part of the JSON schema.
+ var _severity: Severity
+
  /// Whether or not this issue is known to occur.
  var isKnown: Bool
 
@@ -33,6 +46,11 @@ extension ABIv0 {
  var _error: EncodedError?
 
  init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) {
+ _severity = switch issue.severity {
+ case .warning: .warning
+ case .error: .error
+ }
+
  isKnown = issue.isKnown
  sourceLocation = issue.sourceLocation
  if let backtrace = issue.sourceContext.backtrace {
@@ -48,3 +66,4 @@ extension ABIv0 {
 // MARK: - Codable
 
 extension ABIv0.EncodedIssue: Codable {}
+extension ABIv0.EncodedIssue.Severity: Codable {}

Sources/Testing/ABI/v0/Encoded/ABIv0.EncodedMessage.swift

@@ -25,6 +25,7 @@ extension ABIv0 {
  case `default`
  case skip
  case pass
+ case passWithWarnings = "_passWithWarnings"
  case passWithKnownIssue
  case fail
  case difference
@@ -44,6 +45,8 @@ extension ABIv0 {
  } else {
  .pass
  }
+ case .passWithWarnings:
+ .passWithWarnings
  case .fail:
  .fail
  case .difference:

Sources/Testing/Events/Event.swift

@@ -290,10 +290,7 @@ extension Event {
  if let configuration = configuration ?? Configuration.current {
  // The caller specified a configuration, or the current task has an
  // associated configuration. Post to either configuration's event handler.
- switch kind {
- case .expectationChecked where !configuration.deliverExpectationCheckedEvents:
- break
- default:
+ if configuration.eventHandlingOptions.shouldHandleEvent(self) {
  configuration.handleEvent(self, in: context)
  }
  } else {

Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift

@@ -162,6 +162,8 @@ extension Event.Symbol {
  return "\(_ansiEscapeCodePrefix)90m\(symbolCharacter)\(_resetANSIEscapeCode)"
  }
  return "\(_ansiEscapeCodePrefix)92m\(symbolCharacter)\(_resetANSIEscapeCode)"
+ case .passWithWarnings:
+ return "\(_ansiEscapeCodePrefix)93m\(symbolCharacter)\(_resetANSIEscapeCode)"
  case .fail:
  return "\(_ansiEscapeCodePrefix)91m\(symbolCharacter)\(_resetANSIEscapeCode)"
  case .warning:

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

@@ -56,8 +56,9 @@ extension Event {
  /// The instant at which the test started.
  var startInstant: Test.Clock.Instant
 
- /// The number of issues recorded for the test.
- var issueCount = 0
+ /// The number of issues recorded for the test, grouped by their
+ /// level of severity.
+ var issueCount: [Issue.Severity: Int] = [:]
 
  /// The number of known issues recorded for the test.
  var knownIssueCount = 0
@@ -114,27 +115,36 @@ extension Event.HumanReadableOutputRecorder {
  /// - graph: The graph to walk while counting issues.
  ///
  /// - Returns: A tuple containing the number of issues recorded in `graph`.
- private func _issueCounts(in graph: Graph<String, Event.HumanReadableOutputRecorder._Context.TestData?>?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) {
+ private func _issueCounts(in graph: Graph<String, Event.HumanReadableOutputRecorder._Context.TestData?>?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) {
  guard let graph else {
- return (0, 0, 0, "")
+ return (0, 0, 0, 0, "")
  }
- let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=)
+ let errorIssueCount = graph.compactMap(\.value?.issueCount[.error]).reduce(into: 0, +=)
+ let warningIssueCount = graph.compactMap(\.value?.issueCount[.warning]).reduce(into: 0, +=)
  let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=)
- let totalIssueCount = issueCount + knownIssueCount
+ let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount
 
  // Construct a string describing the issue counts.
- let description = switch (issueCount > 0, knownIssueCount > 0) {
- case (true, true):
+ let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) {
+ case (true, true, true):
+ " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))"
+ case (true, false, true):
  " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))"
- case (false, true):
+ case (false, true, true):
+ " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))"
+ case (false, false, true):
  " with \(knownIssueCount.counting("known issue"))"
- case (true, false):
+ case (true, true, false):
+ " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))"
+ case (true, false, false):
  " with \(totalIssueCount.counting("issue"))"
- case(false, false):
+ case(false, true, false):
+ " with \(warningIssueCount.counting("warning"))"
+ case(false, false, false):
  ""
  }
 
- return (issueCount, knownIssueCount, totalIssueCount, description)
+ return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount, description)
  }
 }
 
@@ -267,7 +277,8 @@ extension Event.HumanReadableOutputRecorder {
  if issue.isKnown {
  testData.knownIssueCount += 1
  } else {
- testData.issueCount += 1
+ let issueCount = testData.issueCount[issue.severity] ?? 0
+ testData.issueCount[issue.severity] = issueCount + 1
  }
  context.testData[id] = testData
 
@@ -355,15 +366,15 @@ extension Event.HumanReadableOutputRecorder {
  let testData = testDataGraph?.value ?? .init(startInstant: instant)
  let issues = _issueCounts(in: testDataGraph)
  let duration = testData.startInstant.descriptionOfDuration(to: instant)
- return if issues.issueCount > 0 {
+ return if issues.errorIssueCount > 0 {
  CollectionOfOne(
  Message(
  symbol: .fail,
  stringValue: "\(_capitalizedTitle(for: test)) \(testName) failed after \(duration)\(issues.description)."
  )
  ) + _formattedComments(for: test)
  } else {
-  [
+ [
  Message(
  symbol: .pass(knownIssueCount: issues.knownIssueCount),
  stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)."
@@ -400,13 +411,19 @@ extension Event.HumanReadableOutputRecorder {
  ""
  }
  let symbol: Event.Symbol
- let known: String
+ let subject: String
  if issue.isKnown {
  symbol = .pass(knownIssueCount: 1)
- known = " known"
+ subject = "a known issue"
  } else {
- symbol = .fail
- known = "n"
+ switch issue.severity {
+ case .warning:
+ symbol = .passWithWarnings
+ subject = "a warning"
+ case .error:
+ symbol = .fail
+ subject = "an issue"
+ }
  }
 
  var additionalMessages = [Message]()
@@ -435,13 +452,13 @@ extension Event.HumanReadableOutputRecorder {
  let primaryMessage: Message = if parameterCount == 0 {
  Message(
  symbol: symbol,
- stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)",
+ stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject)\(atSourceLocation): \(issue.kind)",
  conciseStringValue: String(describing: issue.kind)
  )
  } else {
  Message(
  symbol: symbol,
- stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)",
+ stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(subject) with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)",
  conciseStringValue: String(describing: issue.kind)
  )
  }
@@ -498,7 +515,7 @@ extension Event.HumanReadableOutputRecorder {
  let runStartInstant = context.runStartInstant ?? instant
  let duration = runStartInstant.descriptionOfDuration(to: instant)
 
- return if issues.issueCount > 0 {
+ return if issues.errorIssueCount > 0 {
  [
  Message(
  symbol: .fail,

Sources/Testing/Events/Recorder/Event.Symbol.swift

@@ -22,10 +22,14 @@ extension Event {
  /// The symbol to use when a test passes.
  ///
  /// - Parameters:
- /// - knownIssueCount: The number of known issues encountered by the end
- /// of the test.
+ /// - knownIssueCount: The number of known issues recorded for the test.
+ /// The default value is `0`.
  case pass(knownIssueCount: Int = 0)
 
+ /// The symbol to use when a test passes with one or more warnings.
+ @_spi(Experimental)
+ case passWithWarnings
+
  /// The symbol to use when a test fails.
  case fail
 
@@ -62,6 +66,8 @@ extension Event.Symbol {
  } else {
  ("\u{10105B}", "checkmark.diamond.fill")
  }
+ case .passWithWarnings:
+ ("\u{100123}", "questionmark.diamond.fill")
  case .fail:
  ("\u{100884}", "xmark.diamond.fill")
  case .difference:
@@ -122,6 +128,9 @@ extension Event.Symbol {
  // Unicode: HEAVY CHECK MARK
  return "\u{2714}"
  }
+ case .passWithWarnings:
+ // Unicode: QUESTION MARK
+ return "\u{003F}"
  case .fail:
  // Unicode: HEAVY BALLOT X
  return "\u{2718}"
@@ -157,6 +166,9 @@ extension Event.Symbol {
  // Unicode: SQUARE ROOT
  return "\u{221A}"
  }
+ case .passWithWarnings:
+ // Unicode: QUESTION MARK
+ return "\u{003F}"
  case .fail:
  // Unicode: MULTIPLICATION SIGN
  return "\u{00D7}"