Merge pull request #101 from hamishknight/solve-for-x

Author: hamishknight

Sources/_MatchingEngine/Regex/AST/Group.swift

@@ -27,7 +27,7 @@ extension AST {
  case nonCaptureReset
 
  // (?>...)
- case atomicNonCapturing // TODO: is Oniguruma capturing?
+ case atomicNonCapturing
 
  // (?=...)
  case lookahead

Sources/_MatchingEngine/Regex/Parse/LexicalAnalysis.swift

@@ -231,7 +231,7 @@ extension Source {
  /// UniScalar -> 'u{' HexDigit{1...} '}'
  /// | 'u' HexDigit{4}
  /// | 'x{' HexDigit{1...} '}'
- /// | 'x' HexDigit{2}
+ /// | 'x' HexDigit{0...2}
  /// | 'U' HexDigit{8}
  /// | 'o{' OctalDigit{1...} '}'
  /// | OctalDigit{1...3}
@@ -240,15 +240,26 @@ extension Source {
  escapedCharacter base: Character
  ) throws -> Located<Unicode.Scalar> {
  try recordLoc { src in
+ // TODO: PCRE offers a different behavior if PCRE2_ALT_BSUX is set.
  switch base {
  // Hex numbers.
- case "u", "x":
- if src.tryEat("{") {
- let str = try src.lexUntil(eating: "}").value
- return try Source.validateUnicodeScalar(str, .hex)
+ case "u" where src.tryEat("{"), "x" where src.tryEat("{"):
+ let str = try src.lexUntil(eating: "}").value
+ return try Source.validateUnicodeScalar(str, .hex)
+
+ case "x":
+ // \x expects *up to* 2 digits.
+ guard let digits = src.tryEatPrefix(maxLength: 2, \.isHexDigit) else {
+ // In PCRE, \x without any valid hex digits is \u{0}.
+ // TODO: This doesn't appear to be followed by ICU or Oniguruma, so
+ // could be changed to throw an error if we had a parsing mode for
+ // them.
+ return Unicode.Scalar(0)
  }
- let numDigits = base == "u" ? 4 : 2
- return try src.expectUnicodeScalar(numDigits: numDigits).value
+ return try Source.validateUnicodeScalar(digits.string, .hex)
+
+ case "u":
+ return try src.expectUnicodeScalar(numDigits: 4).value
  case "U":
  return try src.expectUnicodeScalar(numDigits: 8).value
 
@@ -514,7 +525,7 @@ extension Source {
  /// Try to lex a sequence of matching options.
  ///
  /// MatchingOptionSeq -> '^' MatchingOption* | MatchingOption+
- /// | MatchingOption* '-' MatchingOption+
+ /// | MatchingOption* '-' MatchingOption*
  ///
  mutating func lexMatchingOptionSequence(
  ) throws -> AST.MatchingOptionSequence? {
@@ -527,8 +538,8 @@ extension Source {
  adding.append(opt)
  }
 
- // If the sequence begun with a caret '^', options can be added, so we're
- // done.
+ // If the sequence begun with a caret '^', options can only be added, so
+ // we're done.
  if ateCaret.value {
  return .init(caretLoc: ateCaret.location, adding: adding, minusLoc: nil,
  removing: [])

Sources/_MatchingEngine/Regex/Parse/Parse.swift

@@ -252,8 +252,6 @@ extension Parser {
  mutating func parseCCCMembers(
  into members: inout Array<CustomCC.Member>
  ) throws {
- // FIXME: Track source locations
-
  // Parse members until we see the end of the custom char class or an
  // operator.
  while source.peek() != "]" && source.peekCCBinOp() == nil {

Sources/_MatchingEngine/Utility/MissingUnicode.swift

@@ -240,8 +240,8 @@ extension Unicode {
  case spaceSeparator = "Zs"
  }
 
- // A list of unicode properties that can either be true or false.
- // https://www.unicode.org/Public/UCD/latest/ucd/PropertyAliases.txt
+ /// A list of unicode properties that can either be true or false.
+ /// https://www.unicode.org/Public/UCD/latest/ucd/PropertyAliases.txt
  public enum BinaryProperty: String, Hashable {
  case asciiHexDigit = "ASCII_Hex_Digit"
  case alphabetic = "Alphabetic"
@@ -313,7 +313,9 @@ extension Unicode {
  }
 }
 
-// Oniguruma properties that are not covered by the above.
+/// Oniguruma properties that are not covered by Unicode spellings.
+/// TODO: These should become aliases for the Block (blk) Unicode character
+/// property.
 public enum OnigurumaSpecialProperty: String, Hashable {
  case inBasicLatin = "In_Basic_Latin"
  case inLatin1Supplement = "In_Latin_1_Supplement"

Tests/RegexTests/LexTests.swift

@@ -83,9 +83,6 @@ extension RegexTests {
  _ = scalars
  }
 
-// FIXME:
-// diagnoseUniScalar(
-// "12ab", base: "x", expectedDigits: 2)
  diagnoseUniScalar(
  "12", base: "u", expectedDigits: 4)
  diagnoseUniScalar(

Tests/RegexTests/ParseTests.swift

@@ -270,6 +270,13 @@ extension RegexTests {
  parseTest(#"[\7777]"#, charClass(scalar_m("\u{1FF}"), "7"))
  parseTest(#"[\181]"#, charClass(scalar_m("\u{1}"), "8", "1"))
 
+ // We take *up to* the first two valid digits for \x. No valid digits is 0.
+ parseTest(#"\x"#, scalar("\u{0}"))
+ parseTest(#"\x5"#, scalar("\u{5}"))
+ parseTest(#"\xX"#, concat(scalar("\u{0}"), "X"))
+ parseTest(#"\x5X"#, concat(scalar("\u{5}"), "X"))
+ parseTest(#"\x12ab"#, concat(scalar("\u{12}"), "a", "b"))
+
  // MARK: Character classes
 
  parseTest(#"abc\d"#, concat("a", "b", "c", escaped(.decimalDigit)))