Integrate experimental string processing modules and enable end-to-end regex.

cmake/modules/AddSwift.cmake

@@ -634,8 +634,13 @@ function(add_libswift_module module)
  ""
  "DEPENDS"
  ${ARGN})
- set(sources ${ALSM_UNPARSED_ARGUMENTS})
- list(TRANSFORM sources PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
+ set(raw_sources ${ALSM_UNPARSED_ARGUMENTS})
+ set(sources)
+ foreach(raw_source ${raw_sources})
+ get_filename_component(
+ raw_source "${raw_source}" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
+ list(APPEND sources "${raw_source}")
+ endforeach()
 
  set(target_name "LibSwift${module}")
 

lib/Sema/CSGen.cpp

@@ -1266,18 +1266,15 @@ namespace {
  ctx.Id_Regex.str());
  return Type();
  }
- auto substringType = ctx.getSubstringType();
  auto dynCapturesType = ctx.getDynamicCapturesType();
  if (!dynCapturesType) {
  ctx.Diags.diagnose(E->getLoc(),
  diag::string_processing_lib_missing,
  "DynamicCaptures");
  return Type();
  }
- // TODO: Replace `(Substring, DynamicCaptures)` with type inferred from
- // the regex.
- auto matchType = TupleType::get({substringType, dynCapturesType}, ctx);
- return BoundGenericStructType::get(regexDecl, Type(), {matchType});
+ // TODO: Replace `DynamicCaptures` with type inferred from the regex.
+ return BoundGenericStructType::get(regexDecl, Type(), {dynCapturesType});
  }
 
  Type visitDeclRefExpr(DeclRefExpr *E) {

libswift/Sources/CMakeLists.txt

@@ -6,6 +6,8 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-add_subdirectory(ExperimentalRegex)
+if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
+ add_subdirectory(ExperimentalRegex)
+endif()
 add_subdirectory(SIL)
 add_subdirectory(Optimizer)

libswift/Sources/ExperimentalRegex/CMakeLists.txt

@@ -6,7 +6,16 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
+file(GLOB_RECURSE _LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES
+ "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
+set(LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES)
+foreach(source ${_LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES})
+ file(TO_CMAKE_PATH "${source}" source)
+ list(APPEND LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES ${source})
+endforeach()
+message(STATUS "Using Experimental String Processing library for libswift ExperimentalRegex (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
+
 add_libswift_module(ExperimentalRegex
- Regex.swift
-)
+ "${LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES}"
+ Regex.swift)
 

libswift/Sources/ExperimentalRegex/Regex.swift

@@ -5,7 +5,7 @@ public func experimental_regex_strawperson(
 ) -> UnsafePointer<CChar>? {
  guard let s = ptr else { return nil }
 
- func error(_ str: String) -> UnsafePointer<CChar>? {
+ func makeError(_ str: String) -> UnsafePointer<CChar>? {
  let count = str.utf8.count + 1
  return str.withCString {
  assert($0[count-1] == 0)
@@ -16,12 +16,15 @@ public func experimental_regex_strawperson(
  }
 
  let str = String(cString: s)
- if str.starts(with: "*") || str.starts(with: "+") || str.starts(with: "?") {
- return error("cannot start regex with quantifier")
+ do {
+ let _ = try parse(str, .traditional)
+ return nil
+ } catch {
+ return makeError(
+ "cannot parse regular expression: \(String(describing: error))")
  }
- return nil
 }
 
-public func registerParser() {
+public func registerRegexParser() {
  Parser_registerParseRegexStrawperson({ experimental_regex_strawperson($0) })
 }

libswift/Sources/Optimizer/CMakeLists.txt

@@ -6,8 +6,13 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-add_libswift_module(Optimizer
- DEPENDS SIL ExperimentalRegex)
+set(dependencies)
+list(APPEND dependencies SIL)
+if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
+ list(APPEND dependencies ExperimentalRegex)
+endif()
+
+add_libswift_module(Optimizer DEPENDS ${dependencies})
 
 add_subdirectory(Analysis)
 add_subdirectory(InstructionPasses)

libswift/Sources/Optimizer/PassManager/PassRegistration.swift

@@ -12,13 +12,19 @@
 
 import SIL
 import OptimizerBridging
+
+#if canImport(ExperimentalRegex)
 import ExperimentalRegex
+#endif
 
 @_cdecl("initializeLibSwift")
 public func initializeLibSwift() {
  registerSILClasses()
  registerSwiftPasses()
- registerParser()
+
+ #if canImport(ExperimentalRegex)
+ registerRegexParser()
+ #endif
 }
 
 private func registerPass(

stdlib/public/MatchingEngine/CMakeLists.txt

@@ -13,9 +13,17 @@
 set(swift_matching_engine_link_libraries
  swiftCore)
 
+file(GLOB_RECURSE _MATCHING_ENGINE_SOURCES
+ "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
+set(MATCHING_ENGINE_SOURCES)
+foreach(source ${_MATCHING_ENGINE_SOURCES})
+ file(TO_CMAKE_PATH "${source}" source)
+ list(APPEND MATCHING_ENGINE_SOURCES ${source})
+endforeach()
+message(STATUS "Using Experimental String Processing library for _MatchingEngine (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
 
 add_swift_target_library(swift_MatchingEngine ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
- MatchingEngine.swift
+ "${MATCHING_ENGINE_SOURCES}"
 
  SWIFT_MODULE_DEPENDS_LINUX Glibc
  SWIFT_MODULE_DEPENDS_FREEBSD Glibc
@@ -30,8 +38,6 @@ add_swift_target_library(swift_MatchingEngine ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES
  -Dswift_MatchingEngine_EXPORTS
  SWIFT_COMPILE_FLAGS
  ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
- -parse-stdlib
- -Xfrontend -enable-experimental-string-processing
  LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
 
  INSTALL_IN_COMPONENT stdlib

stdlib/public/StringProcessing/CMakeLists.txt

@@ -14,9 +14,17 @@ set(swift_string_processing_link_libraries
  swiftCore
  swift_MatchingEngine)
 
+file(GLOB_RECURSE _STRING_PROCESSING_SOURCES
+ "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_StringProcessing/*.swift")
+set(STRING_PROCESSING_SOURCES)
+foreach(source ${_STRING_PROCESSING_SOURCES})
+ file(TO_CMAKE_PATH "${source}" source)
+ list(APPEND STRING_PROCESSING_SOURCES ${source})
+endforeach()
+message(STATUS "Using Experimental String Processing library for _StringProcessing (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
 
 add_swift_target_library(swift_StringProcessing ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} IS_STDLIB
- Regex.swift
+ "${STRING_PROCESSING_SOURCES}"
 
  SWIFT_MODULE_DEPENDS_LINUX Glibc
  SWIFT_MODULE_DEPENDS_FREEBSD Glibc
@@ -31,8 +39,6 @@ add_swift_target_library(swift_StringProcessing ${SWIFT_STDLIB_LIBRARY_BUILD_TYP
  -Dswift_StringProcessing_EXPORTS
  SWIFT_COMPILE_FLAGS
  ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
- -parse-stdlib
- -Xfrontend -enable-experimental-string-processing
  LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
 
  SWIFT_MODULE_DEPENDS _MatchingEngine

test/StringProcessing/Parse/regex.swift

@@ -1,10 +1,10 @@
 // RUN: %target-typecheck-verify-swift -enable-experimental-string-processing
 // REQUIRES: libswift
 
-var s = 'abc'
+_ = 'abc'
 
-var s1 = ('*', '+', '?')
-// expected-error@-1 3{{cannot start regex with quantifier}}
+_ = ('[*', '+]', '.]')
+// expected-error@-1 {{cannot parse regular expression}}
 
-var s2 = '\w+'
-var s3 = '\'\\'
+_ = '\w+'
+_ = '\'\\'

test/StringProcessing/Runtime/regex_basic.swift

@@ -0,0 +1,52 @@
+// RUN: %target-run-simple-swift(-Xfrontend -enable-experimental-string-processing)
+
+// REQUIRES: libswift,string_processing,executable_test
+
+import StdlibUnittest
+
+var RegexBasicTests = TestSuite("RegexBasic")
+
+extension String {
+ func expectMatch<T>(
+ _ regex: Regex<T>,
+ file: String = #file,
+ line: UInt = #line
+ ) -> RegexMatch<T> {
+ guard let result = match(regex) else {
+ expectUnreachable("Failed match", file: file, line: line)
+ fatalError()
+ }
+ return result
+ }
+}
+
+RegexBasicTests.test("Basic") {
+ let input = "aabccd"
+
+ let match1 = input.expectMatch('aabcc.')
+ expectEqual("aabccd", input[match1.range])
+ expectEqual(.empty, match1.captures)
+
+ let match2 = input.expectMatch('a*b.+.')
+ expectEqual("aabccd", input[match2.range])
+ expectEqual(.empty, match2.captures)
+}
+
+RegexBasicTests.test("Captures") {
+ let input = """
+ A6F0..A6F1 ; Extend # Mn [2] BAMUM COMBINING MARK KOQNDON..BAMUM \
+ COMBINING MARK TUKWENTIS
+ """
+ let regex = '([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s+;\s+(\w+).*'
+ let match1 = input.expectMatch(regex)
+ expectEqual(input[...], input[match1.range])
+ expectEqual(
+ .tuple([
+ .substring("A6F0"),
+ .optional(.substring("A6F1")),
+ .substring("Extend")
+ ]),
+ match1.captures)
+}
+
+runAllTests()

test/StringProcessing/SILGen/regex_literal_silgen.swift

@@ -5,5 +5,5 @@ var s = 'abc'
 // CHECK: [[REGEX_STR_LITERAL:%[0-9]+]] = string_literal utf8 "abc"
 // CHECK: [[STRING_INIT:%[0-9]+]] = function_ref @$sSS21_builtinStringLiteral17utf8CodeUnitCount7isASCIISSBp_BwBi1_tcfC : $@convention(method) (Builtin.RawPointer, Builtin.Word, Builtin.Int1, @thin String.Type) -> @owned String
 // CHECK: [[REGEX_STR:%[0-9]+]] = apply [[STRING_INIT]]([[REGEX_STR_LITERAL]]
-// CHECK: [[REGEX_INIT:%[0-9]+]] = function_ref @$s17_StringProcessing5RegexV06_regexA0ACyxGSS_tcfC : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> Regex<τ_0_0>
-// CHECK: apply [[REGEX_INIT]]<(Substring, DynamicCaptures)>([[REGEX_STR]], {{%.+}}) : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> Regex<τ_0_0>
+// CHECK: [[REGEX_INIT:%[0-9]+]] = function_ref @$s17_StringProcessing5RegexV06_regexA0ACyxGSS_tcfC : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> @out Regex<τ_0_0>
+// CHECK: apply [[REGEX_INIT]]<DynamicCaptures>({{%.+}}, [[REGEX_STR]], {{%.+}}) : $@convention(method) <τ_0_0> (@owned String, @thin Regex<τ_0_0>.Type) -> @out Regex<τ_0_0>

test/StringProcessing/Sema/regex_literal_type_inference.swift

@@ -3,10 +3,10 @@
 
 let r0 = '.' // okay
 
-let r1: Regex<(Substring, DynamicCaptures)> = '.(.)' // okay
+let r1: Regex<DynamicCaptures> = '.(.)' // okay
 
-// expected-error @+2 {{cannot assign value of type 'Regex<(Substring, DynamicCaptures)>' to type 'Regex<Substring>'}}
-// expected-note @+1 {{arguments to generic parameter 'Match' ('(Substring, DynamicCaptures)' and 'Substring') are expected to be equal}}
+// expected-error @+2 {{cannot assign value of type 'Regex<DynamicCaptures>' to type 'Regex<Substring>'}}
+// expected-note @+1 {{arguments to generic parameter 'Capture' ('DynamicCaptures' and 'Substring') are expected to be equal}}
 let r2: Regex<Substring> = '.(.)'
 
 func takesRegex<Match>(_: Regex<Match>) {}

test/StringProcessing/lit.local.cfg

@@ -1,2 +1,2 @@
 if 'string_processing' not in config.available_features:
- config.unsupported = False
+ config.unsupported = True

utils/build-script-impl

@@ -225,6 +225,7 @@ KNOWN_SETTINGS=(
  swift-stdlib-experimental-hermetic-seal-at-link "0" "whether stdlib should be built with -experimental-hermetic-seal-at-link"
  swift-disable-dead-stripping "0" "turns off Darwin-specific dead stripping for Swift host tools"
  common-swift-flags "" "Flags used for Swift targets other than the stdlib, like the corelibs"
+ swift-enable-experimental-string-processing "1" "whether to build experimental string processing feature"
 
  ## FREESTANDING Stdlib Options
  swift-freestanding-flavor "" "when building the FREESTANDING stdlib, which build style to use (options: apple, linux)"
@@ -1252,6 +1253,7 @@ LIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBDISPATCH_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBICU_SOURCE_DIR="${WORKSPACE}/icu"
 LIBCXX_SOURCE_DIR="${WORKSPACE}/llvm-project/libcxx"
+EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR="${WORKSPACE}/swift-experimental-string-processing"
 
 # We cannot currently apply the normal rules of skipping here for LLVM. Even if
 # we are skipping building LLVM, we still need to at least build a few tools
@@ -2196,6 +2198,14 @@ for host in "${ALL_HOSTS[@]}"; do
  )
  fi
 
+ if [[ $(true_false "${SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING}") == "TRUE" ]] ; then
+ cmake_options=(
+ "${cmake_options[@]}"
+ -DSWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING:BOOL=TRUE
+ -DEXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR="${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}"
+ )
+ fi
+
  build_targets=(all "${SWIFT_STDLIB_TARGETS[@]}")
  if [[ $(true_false "${build_perf_testsuite_this_time}") == "TRUE" ]]; then
  native_swift_tools_path="$(build_directory_bin ${LOCAL_HOST} swift)"

utils/build-windows.bat

@@ -272,6 +272,7 @@ cmake^
  -DSWIFT_ENABLE_EXPERIMENTAL_DISTRIBUTED=YES^
  -DSWIFT_ENABLE_EXPERIMENTAL_DIFFERENTIABLE_PROGRAMMING=YES^
  -DSWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING=YES^
+ -DEXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR=%source_root%\swift-experimental-string-processing^
  -DSWIFT_INSTALL_COMPONENTS="autolink-driver;compiler;clang-resource-dir-symlink;stdlib;sdk-overlay;editor-integration;tools;testsuite-tools;sourcekit-inproc;swift-remote-mirror;swift-remote-mirror-headers"^
  -DSWIFT_PARALLEL_LINK_JOBS=8^
  -DPYTHON_EXECUTABLE:PATH=%PYTHON_HOME%\python.exe^

utils/swift_build_support/swift_build_support/products/swift.py

@@ -59,9 +59,6 @@ def __init__(self, args, toolchain, source_dir, build_dir):
  # Add static vprintf flag
  self.cmake_options.extend(self._enable_stdlib_static_vprintf)
 
- # Add experimental string processing flag.
- self.cmake_options.extend(self._enable_experimental_string_processing)
-
  # Add freestanding related flags.
  self.cmake_options.extend(self._freestanding_is_darwin)
 
@@ -171,11 +168,6 @@ def _enable_stdlib_static_vprintf(self):
  return [('SWIFT_STDLIB_STATIC_PRINT',
  self.args.build_swift_stdlib_static_print)]
 
- @property
- def _enable_experimental_string_processing(self):
- return [('SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING:BOOL',
- self.args.enable_experimental_string_processing)]
-
  @property
  def _freestanding_is_darwin(self):
  return [('SWIFT_FREESTANDING_IS_DARWIN:BOOL',