Complete regex literal lexing

The previous regex literal lexing logic didn't implement all the heuristics needed to parse all the regex literals that the old C++ parser handles. Update it with the heuristics it needs, and better handle diagnostics for invalid cases. This unfortunately requires a bit of lexical hackery, including a heuristic to classify previous token kinds to determine if we're in regex literal position, but it's needed to handle pathological cases such as `x /^ y/` where there's an ambiguity with a binary operator.

Author: hamishknight

CodeGeneration/Sources/SyntaxSupport/Classification.swift

@@ -58,6 +58,7 @@ public let SYNTAX_CLASSIFICATIONS: [SyntaxClassification] = [
  SyntaxClassification(name: "ObjectLiteral", description: "An image, color, etc. literal."),
  SyntaxClassification(name: "OperatorIdentifier", description: "An identifier referring to an operator."),
  SyntaxClassification(name: "PoundDirectiveKeyword", description: "A `#` keyword like `#warning`."),
+ SyntaxClassification(name: "RegexLiteral", description: "A regex literal, including multiline regex literals."),
  SyntaxClassification(name: "StringInterpolationAnchor", description: "The opening and closing parenthesis of string interpolation."),
  SyntaxClassification(name: "StringLiteral", description: "A string literal including multiline string literals."),
  SyntaxClassification(name: "TypeIdentifier", description: "An identifier referring to a type."),

CodeGeneration/Sources/SyntaxSupport/ExprNodes.swift

@@ -1199,9 +1199,27 @@ public let EXPR_NODES: [Node] = [
  kind: "Expr",
  children: [
  Child(
- name: "Regex",
- kind: .token(choices: [.token(tokenKind: "RegexLiteralToken")])
- )
+ name: "OpeningPounds",
+ kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]),
+ isOptional: true
+ ),
+ Child(
+ name: "OpenSlash",
+ kind: .token(choices: [.token(tokenKind: "RegexSlashToken")])
+ ),
+ Child(
+ name: "RegexPattern",
+ kind: .token(choices: [.token(tokenKind: "RegexLiteralPatternToken")])
+ ),
+ Child(
+ name: "CloseSlash",
+ kind: .token(choices: [.token(tokenKind: "RegexSlashToken")])
+ ),
+ Child(
+ name: "ClosingPounds",
+ kind: .token(choices: [.token(tokenKind: "ExtendedRegexDelimiterToken")]),
+ isOptional: true
+ ),
  ]
  ),
 

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -178,6 +178,7 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
  PunctuatorSpec(name: "Ellipsis", kind: "ellipsis", text: "..."),
  PunctuatorSpec(name: "Equal", kind: "equal", text: "=", requiresLeadingSpace: true, requiresTrailingSpace: true),
  PunctuatorSpec(name: "ExclamationMark", kind: "exclaim_postfix", text: "!"),
+ MiscSpec(name: "ExtendedRegexDelimiter", kind: "extended_regex_delimiter", nameForDiagnostics: "extended delimiter", classification: "RegexLiteral"),
  LiteralSpec(name: "FloatingLiteral", kind: "floating_literal", nameForDiagnostics: "floating literal", classification: "FloatingLiteral"),
  MiscSpec(name: "Identifier", kind: "identifier", nameForDiagnostics: "identifier", classification: "Identifier"),
  PunctuatorSpec(name: "InfixQuestionMark", kind: "question_infix", text: "?"),
@@ -202,7 +203,8 @@ public let SYNTAX_TOKENS: [TokenSpec] = [
  PunctuatorSpec(name: "PrefixAmpersand", kind: "amp_prefix", text: "&"),
  MiscSpec(name: "PrefixOperator", kind: "oper_prefix", nameForDiagnostics: "prefix operator", classification: "OperatorIdentifier"),
  MiscSpec(name: "RawStringDelimiter", kind: "raw_string_delimiter", nameForDiagnostics: "raw string delimiter"),
- LiteralSpec(name: "RegexLiteral", kind: "regex_literal", nameForDiagnostics: "regex literal"),
+ MiscSpec(name: "RegexLiteralPattern", kind: "regex_literal_pattern", nameForDiagnostics: "regex pattern", classification: "RegexLiteral"),
+ PunctuatorSpec(name: "RegexSlash", kind: "regex_slash", text: "/", classification: "RegexLiteral"),
  PunctuatorSpec(name: "RightAngle", kind: "r_angle", text: ">"),
  PunctuatorSpec(name: "RightBrace", kind: "r_brace", text: "}"),
  PunctuatorSpec(name: "RightParen", kind: "r_paren", text: ")"),

Sources/SwiftParser/CMakeLists.txt

@@ -42,6 +42,7 @@ add_swift_host_library(SwiftParser
  Lexer/Lexeme.swift
  Lexer/LexemeSequence.swift
  Lexer/Lexer.swift
+ Lexer/RegexLiteralLexer.swift
  Lexer/UnicodeScalarExtensions.swift
 )
 

Sources/SwiftParser/Declarations.swift

@@ -1315,28 +1315,6 @@ extension Parser {
 }
 
 extension Parser {
- /// Are we at a regular expression literal that could act as an operator?
- private mutating func atRegexLiteralThatCouldBeAnOperator() -> Bool {
- guard self.at(.regexLiteral) else {
- return false
- }
-
- /// Try to re-lex at regex literal as an operator. If it succeeds and
- /// consumes the entire regex literal, we're done.
- return self.currentToken.tokenText.withBuffer {
- (buffer: UnsafeBufferPointer<UInt8>) -> Bool in
- var cursor = Lexer.Cursor(input: buffer, previous: 0)
- guard buffer[0] == UInt8(ascii: "/") else { return false }
- switch cursor.lexOperatorIdentifier(sourceBufferStart: cursor).tokenKind {
- case .unknown:
- return false
-
- default:
- return cursor.input.isEmpty
- }
- }
- }
-
  @_spi(RawSyntax)
  public mutating func parseFuncDeclaration(
  _ attrs: DeclAttributes,
@@ -1345,7 +1323,7 @@ extension Parser {
  let (unexpectedBeforeFuncKeyword, funcKeyword) = self.eat(handle)
  let unexpectedBeforeIdentifier: RawUnexpectedNodesSyntax?
  let identifier: RawTokenSyntax
- if self.at(anyIn: Operator.self) != nil || self.at(.exclamationMark, .prefixAmpersand) || self.atRegexLiteralThatCouldBeAnOperator() {
+ if self.at(anyIn: Operator.self) != nil || self.at(.exclamationMark, .prefixAmpersand) {
  var name = self.currentToken.tokenText
  if name.count > 1 && name.hasSuffix("<") && self.peek().rawTokenKind == .identifier {
  name = SyntaxText(rebasing: name.dropLast())

Sources/SwiftParser/Expressions.swift

@@ -1170,7 +1170,7 @@ extension Parser {
  )
  case (.rawStringDelimiter, _)?, (.stringQuote, _)?, (.multilineStringQuote, _)?, (.singleQuote, _)?:
  return RawExprSyntax(self.parseStringLiteral())
- case (.regexLiteral, _)?:
+ case (.extendedRegexDelimiter, _)?, (.regexSlash, _)?:
  return RawExprSyntax(self.parseRegexLiteral())
  case (.nilKeyword, let handle)?:
  let nilKeyword = self.eat(handle)
@@ -1433,13 +1433,32 @@ extension Parser {
  /// Grammar
  /// =======
  ///
- /// regular-expression-literal → '\' `Any valid regular expression characters` '\'
+ /// regular-expression-literal → '#'* '/' `Any valid regular expression characters` '/' '#'*
  @_spi(RawSyntax)
  public mutating func parseRegexLiteral() -> RawRegexLiteralExprSyntax {
- let (unexpectedBeforeLiteral, literal) = self.expect(.regexLiteral)
+ // See if we have an opening set of pounds.
+ let openPounds = self.consume(if: .extendedRegexDelimiter)
+
+ // Parse the opening slash.
+ let (unexpectedBeforeSlash, openSlash) = self.expect(.regexSlash)
+
+ // Parse the pattern and closing slash, avoiding recovery or leading trivia
+ // as the lexer should provide the tokens exactly in order without trivia,
+ // otherwise they should be treated as missing.
+ let pattern = self.expectWithoutRecoveryOrLeadingTrivia(.regexLiteralPattern)
+ let closeSlash = self.expectWithoutRecoveryOrLeadingTrivia(.regexSlash)
+
+ // Finally, parse a closing set of pounds.
+ let (unexpectedBeforeClosePounds, closePounds) = parsePoundDelimiter(.extendedRegexDelimiter, matching: openPounds)
+
  return RawRegexLiteralExprSyntax(
- unexpectedBeforeLiteral,
- regex: literal,
+ openingPounds: openPounds,
+ unexpectedBeforeSlash,
+ openSlash: openSlash,
+ regexPattern: pattern,
+ closeSlash: closeSlash,
+ unexpectedBeforeClosePounds,
+ closingPounds: closePounds,
  arena: self.arena
  )
  }