Revert "Merge pull request #40595 from hamishknight/straw-bales"

This reverts commit a67a043, reversing changes made to 9965df7. This commit or the earlier commit this commit is based on (#40531) broke the incremental bot.

cmake/modules/AddSwift.cmake

@@ -634,13 +634,8 @@ function(add_libswift_module module)
  ""
  "DEPENDS"
  ${ARGN})
- set(raw_sources ${ALSM_UNPARSED_ARGUMENTS})
- set(sources)
- foreach(raw_source ${raw_sources})
- get_filename_component(
- raw_source "${raw_source}" REALPATH BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
- list(APPEND sources "${raw_source}")
- endforeach()
+ set(sources ${ALSM_UNPARSED_ARGUMENTS})
+ list(TRANSFORM sources PREPEND "${CMAKE_CURRENT_SOURCE_DIR}/")
 
  set(target_name "LibSwift${module}")
 

include/swift/AST/ASTContext.h

@@ -621,7 +621,7 @@ class ASTContext final {
  KnownProtocolKind builtinProtocol,
  llvm::function_ref<DeclName (ASTContext &ctx)> initName) const;
 
- /// Retrieve _StringProcessing.Regex.init(_regexString: String, version: Int).
+ /// Retrieve _StringProcessing.Regex.init(_regexString: String).
  ConcreteDeclRef getRegexInitDecl(Type regexType) const;
 
  /// Retrieve the declaration of Swift.<(Int, Int) -> Bool.

include/swift/AST/Expr.h

@@ -966,23 +966,18 @@ class InterpolatedStringLiteralExpr : public LiteralExpr {
 class RegexLiteralExpr : public LiteralExpr {
  SourceLoc Loc;
  StringRef RegexText;
- unsigned Version;
 
- RegexLiteralExpr(SourceLoc loc, StringRef regexText, unsigned version,
- bool isImplicit)
+ RegexLiteralExpr(SourceLoc loc, StringRef regexText, bool isImplicit)
  : LiteralExpr(ExprKind::RegexLiteral, isImplicit), Loc(loc),
- RegexText(regexText), Version(version) {}
+ RegexText(regexText) {}
 
 public:
  static RegexLiteralExpr *createParsed(ASTContext &ctx, SourceLoc loc,
- StringRef regexText, unsigned version);
+ StringRef regexText);
 
  /// Retrieve the raw regex text.
  StringRef getRegexText() const { return RegexText; }
 
- /// Retrieve the version of the regex string.
- unsigned getVersion() const { return Version; }
-
  SourceRange getSourceRange() const { return Loc; }
 
  static bool classof(const Expr *E) {

include/swift/AST/KnownIdentifiers.def

@@ -252,7 +252,6 @@ IDENTIFIER(zero)
 // String processing
 IDENTIFIER(Regex)
 IDENTIFIER_(regexString)
-IDENTIFIER(version)
 IDENTIFIER_(StringProcessing)
 
 // Distributed actors

include/swift/Parse/ExperimentalRegexBridging.h

@@ -5,43 +5,17 @@
 extern "C" {
 #endif
 
-/// Attempt to lex a regex literal string. Takes the following arguments:
-///
-/// - CurPtrPtr: A pointer to the current pointer of lexer, which should be the
-/// start of the literal. This will be advanced to the point at
-/// which the lexer should resume, or will remain the same if this
-/// is not a regex literal.
-/// - BufferEnd: A pointer to the end of the buffer, which should not be lexed
-/// past.
-/// - ErrorOut: If an error is encountered, this will be set to the error
-/// string.
-///
-/// Returns: A bool indicating whether lexing was completely erroneous, and
-/// cannot be recovered from, or false if there either was no error,
-/// or there was a recoverable error.
-typedef bool(* RegexLiteralLexingFn)(/*CurPtrPtr*/ const char **,
- /*BufferEnd*/ const char *,
- /*ErrorOut*/ const char **);
-void Parser_registerRegexLiteralLexingFn(RegexLiteralLexingFn fn);
-
-/// Parse a regex literal string. Takes the following arguments:
-///
-/// - InputPtr: A null-terminated C string of the regex literal.
-/// - ErrorOut: A buffer accepting an error string upon error.
-/// - VersionOut: A buffer accepting a regex literal format version.
-/// - CaptureStructureOut: A buffer accepting a byte sequence representing the
-/// capture structure of the literal.
-/// - CaptureStructureSize: The size of the capture structure buffer. Must be
-/// greater than or equal to `strlen(InputPtr)`.
-typedef void(* RegexLiteralParsingFn)(/*InputPtr*/ const char *,
- /*ErrorOut*/ const char **,
- /*VersionOut*/ unsigned *,
- /*CaptureStructureOut*/ char *,
- /*CaptureStructureSize*/ unsigned);
-void Parser_registerRegexLiteralParsingFn(RegexLiteralParsingFn fn);
+typedef const char *(* ParseRegexStrawperson)(const char *);
+
+void Parser_registerParseRegexStrawperson(ParseRegexStrawperson fn);
+bool Parser_hasParseRegexStrawperson();
 
 #ifdef __cplusplus
 } // extern "C"
 #endif
 
 #endif // EXPERIMENTAL_REGEX_BRIDGING
+
+
+//const char* experimental_regex_strawperson(const char *in);
+

include/swift/Parse/Lexer.h

@@ -595,9 +595,7 @@ class Lexer {
  void lexStringLiteral(unsigned CustomDelimiterLen = 0);
  void lexEscapedIdentifier();
 
- /// Attempt to lex a regex literal, returning true if a regex literal was
- /// lexed, false if this is not a regex literal.
- bool tryLexRegexLiteral(const char *TokStart);
+ void lexRegexLiteral(const char *TokStart);
 
  void tryLexEditorPlaceholder();
  const char *findEndOfCurlyQuoteStringLiteral(const char *,
@@ -614,6 +612,9 @@ class Lexer {
 
  /// Emit diagnostics for single-quote string and suggest replacement
  /// with double-quoted equivalent.
+ ///
+ /// Or, if we're in strawperson mode, we will emit a custom
+ /// error message instead, determined by the Swift library.
  void diagnoseSingleQuoteStringLiteral(const char *TokStart,
  const char *TokEnd);
 

lib/AST/ASTContext.cpp

@@ -1221,7 +1221,7 @@ ConcreteDeclRef ASTContext::getRegexInitDecl(Type regexType) const {
  auto *spModule = getLoadedModule(Id_StringProcessing);
  DeclName name(*const_cast<ASTContext *>(this),
  DeclBaseName::createConstructor(),
- {Id_regexString, Id_version});
+ {Id_regexString});
  SmallVector<ValueDecl *, 1> results;
  spModule->lookupQualified(getRegexType(), DeclNameRef(name),
  NL_IncludeUsableFromInline, results);

lib/AST/Expr.cpp

@@ -2246,9 +2246,8 @@ SourceLoc TapExpr::getEndLoc() const {
 
 RegexLiteralExpr *
 RegexLiteralExpr::createParsed(ASTContext &ctx, SourceLoc loc,
- StringRef regexText, unsigned version) {
- return new (ctx) RegexLiteralExpr(loc, regexText, version,
- /*implicit*/ false);
+ StringRef regexText) {
+ return new (ctx) RegexLiteralExpr(loc, regexText, /*implicit*/ false);
 }
 
 void swift::simple_display(llvm::raw_ostream &out, const ClosureExpr *CE) {

lib/Parse/Lexer.cpp

@@ -33,13 +33,6 @@
 
 #include <limits>
 
-// Regex lexing delivered via libSwift.
-#include "swift/Parse/ExperimentalRegexBridging.h"
-static RegexLiteralLexingFn regexLiteralLexingFn = nullptr;
-void Parser_registerRegexLiteralLexingFn(RegexLiteralLexingFn fn) {
- regexLiteralLexingFn = fn;
-}
-
 using namespace swift;
 using namespace swift::syntax;
 
@@ -1958,46 +1951,36 @@ const char *Lexer::findEndOfCurlyQuoteStringLiteral(const char *Body,
  }
 }
 
-bool Lexer::tryLexRegexLiteral(const char *TokStart) {
+void Lexer::lexRegexLiteral(const char *TokStart) {
  assert(*TokStart == '\'');
 
- // We need to have experimental string processing enabled, and have the
- // parsing logic for regex literals available.
- if (!LangOpts.EnableExperimentalStringProcessing || !regexLiteralLexingFn)
- return false;
-
- // Ask libswift to try and lex a regex literal.
- // - Ptr will not be advanced if this is not for a regex literal.
- // - ErrStr will be set if there is any error to emit.
- // - CompletelyErroneous will be set if there was an error that cannot be
- // recovered from.
- auto *Ptr = TokStart;
- const char *ErrStr = nullptr;
- bool CompletelyErroneous = regexLiteralLexingFn(&Ptr, BufferEnd, &ErrStr);
- if (ErrStr)
- diagnose(TokStart, diag::regex_literal_parsing_error, ErrStr);
-
- // If we didn't make any lexing progress, this isn't a regex literal and we
- // should fallback to lexing as something else.
- if (Ptr == TokStart)
- return false;
-
- // Update to point to where we ended regex lexing.
- assert(Ptr > TokStart && Ptr <= BufferEnd);
- CurPtr = Ptr;
+ bool HadError = false;
+ while (true) {
+ // Check if we reached the end of the literal without terminating.
+ if (CurPtr >= BufferEnd || *CurPtr == '\n' || *CurPtr == '\r') {
+ diagnose(TokStart, diag::lex_unterminated_regex);
+ return formToken(tok::unknown, TokStart);
+ }
 
- // If the lexing was completely erroneous, form an unknown token.
- if (CompletelyErroneous) {
- assert(ErrStr);
- formToken(tok::unknown, TokStart);
- return true;
+ const auto *CharStart = CurPtr;
+ uint32_t CharValue = validateUTF8CharacterAndAdvance(CurPtr, BufferEnd);
+ if (CharValue == ~0U) {
+ diagnose(CharStart, diag::lex_invalid_utf8);
+ HadError = true;
+ continue;
+ }
+ if (CharValue == '\\' && (*CurPtr == '\'' || *CurPtr == '\\')) {
+ // Skip escaped delimiter or \.
+ CurPtr++;
+ } else if (CharValue == '\'') {
+ // End of literal, stop.
+ break;
+ }
  }
+ if (HadError)
+ return formToken(tok::unknown, TokStart);
 
- // Otherwise, we either had a successful lex, or something that was
- // recoverable.
- assert(ErrStr || CurPtr[-1] == '\'');
  formToken(tok::regex_literal, TokStart);
- return true;
 }
 
 /// lexEscapedIdentifier:
@@ -2545,11 +2528,11 @@ void Lexer::lexImpl() {
 
  case '\'':
  // If we have experimental string processing enabled, and have the parsing
- // logic for regex literals, try to lex a single quoted string as a regex
- // literal.
- if (tryLexRegexLiteral(TokStart))
- return;
-
+ // logic for regex literals, lex a single quoted string as a regex literal.
+ if (LangOpts.EnableExperimentalStringProcessing &&
+  Parser_hasParseRegexStrawperson()) {
+ return lexRegexLiteral(TokStart);
+ }
  // Otherwise lex as a string literal and emit a diagnostic.
  LLVM_FALLTHROUGH;
  case '"':

lib/Parse/ParseRegex.cpp

@@ -22,36 +22,38 @@
 
 // Regex parser delivered via libSwift
 #include "swift/Parse/ExperimentalRegexBridging.h"
-static RegexLiteralParsingFn regexLiteralParsingFn = nullptr;
-void Parser_registerRegexLiteralParsingFn(RegexLiteralParsingFn fn) {
- regexLiteralParsingFn = fn;
+static ParseRegexStrawperson parseRegexStrawperson = nullptr;
+void Parser_registerParseRegexStrawperson(ParseRegexStrawperson fn) {
+ parseRegexStrawperson = fn;
+}
+// Exposes the presence of the regex parsing function to the lexer.
+bool Parser_hasParseRegexStrawperson() {
+ return parseRegexStrawperson != nullptr;
 }
 
 using namespace swift;
 using namespace swift::syntax;
 
 ParserResult<Expr> Parser::parseExprRegexLiteral() {
  assert(Tok.is(tok::regex_literal));
- assert(regexLiteralParsingFn);
+ assert(parseRegexStrawperson);
 
  SyntaxParsingContext LocalContext(SyntaxContext,
  SyntaxKind::RegexLiteralExpr);
-
- auto regexText = Tok.getText();
+ // Strip off delimiters.
+ auto rawText = Tok.getText();
+ assert(rawText.front() == '\'' && rawText.back() == '\'');
+ auto regexText = rawText.slice(1, rawText.size() - 1);
 
  // Let the Swift library parse the contents, returning an error, or null if
  // successful.
  // TODO: We need to be able to pass back a source location to emit the error
  // at.
- const char *errorStr = nullptr;
- unsigned version;
- regexLiteralParsingFn(regexText.str().c_str(), &errorStr, &version,
- /*captureStructureOut*/ nullptr,
- /*captureStructureSize*/ 0);
+ auto *errorStr = parseRegexStrawperson(regexText.str().c_str());
  if (errorStr)
  diagnose(Tok, diag::regex_literal_parsing_error, errorStr);
 
  auto loc = consumeToken();
  return makeParserResult(
- RegexLiteralExpr::createParsed(Context, loc, regexText, version));
+ RegexLiteralExpr::createParsed(Context, loc, regexText));
 }

lib/SILGen/SILGenApply.cpp

@@ -1910,31 +1910,10 @@ buildBuiltinLiteralArgs(SILGenFunction &SGF, SGFContext C,
  RValue string = SGF.emitApplyAllocatingInitializer(
  expr, strInitDecl, std::move(strLiteralArgs),
  /*overriddenSelfType*/ Type(), SGFContext());
-
- // The version of the regex string.
- // %3 = integer_literal $Builtin.IntLiteral <version>
- auto versionIntLiteral =
- ManagedValue::forUnmanaged(SGF.B.createIntegerLiteral(
- expr, SILType::getBuiltinIntegerLiteralType(SGF.getASTContext()),
- expr->getVersion()));
-
- using Param = AnyFunctionType::Param;
- auto builtinIntTy = versionIntLiteral.getType().getASTType();
- PreparedArguments versionIntBuiltinArgs(ArrayRef<Param>{Param(builtinIntTy)});
- versionIntBuiltinArgs.add(
- expr, RValue(SGF, {versionIntLiteral}, builtinIntTy));
-
- // %4 = function_ref Int.init(_builtinIntegerLiteral: Builtin.IntLiteral)
- // %5 = apply %5(%3, ...) -> $Int
- auto intLiteralInit = ctx.getIntBuiltinInitDecl(ctx.getIntDecl());
- RValue versionInt = SGF.emitApplyAllocatingInitializer(
- expr, intLiteralInit, std::move(versionIntBuiltinArgs),
- /*overriddenSelfType*/ Type(), SGFContext());
-
- PreparedArguments args(ArrayRef<Param>{Param(ctx.getStringType()),
- Param(ctx.getIntType())});
+ PreparedArguments args(
+ ArrayRef<AnyFunctionType::Param>({
+ AnyFunctionType::Param(ctx.getStringType())}));
  args.add(expr, std::move(string));
- args.add(expr, std::move(versionInt));
  return args;
 }
 

lib/Sema/CSGen.cpp

@@ -1266,15 +1266,18 @@ namespace {
  ctx.Id_Regex.str());
  return Type();
  }
+ auto substringType = ctx.getSubstringType();
  auto dynCapturesType = ctx.getDynamicCapturesType();
  if (!dynCapturesType) {
  ctx.Diags.diagnose(E->getLoc(),
  diag::string_processing_lib_missing,
  "DynamicCaptures");
  return Type();
  }
- // TODO: Replace `DynamicCaptures` with type inferred from the regex.
- return BoundGenericStructType::get(regexDecl, Type(), {dynCapturesType});
+ // TODO: Replace `(Substring, DynamicCaptures)` with type inferred from
+ // the regex.
+ auto matchType = TupleType::get({substringType, dynCapturesType}, ctx);
+ return BoundGenericStructType::get(regexDecl, Type(), {matchType});
  }
 
  Type visitDeclRefExpr(DeclRefExpr *E) {

libswift/Sources/CMakeLists.txt

@@ -6,8 +6,6 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-if(SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING)
- add_subdirectory(ExperimentalRegex)
-endif()
+add_subdirectory(ExperimentalRegex)
 add_subdirectory(SIL)
 add_subdirectory(Optimizer)

libswift/Sources/ExperimentalRegex/CMakeLists.txt

@@ -6,16 +6,7 @@
 # See http://swift.org/LICENSE.txt for license information
 # See http://swift.org/CONTRIBUTORS.txt for Swift project authors
 
-file(GLOB_RECURSE _LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES
- "${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}/Sources/_MatchingEngine/*.swift")
-set(LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES)
-foreach(source ${_LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES})
- file(TO_CMAKE_PATH "${source}" source)
- list(APPEND LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES ${source})
-endforeach()
-message(STATUS "Using Experimental String Processing library for libswift ExperimentalRegex (${EXPERIMENTAL_STRING_PROCESSING_SOURCE_DIR}).")
-
 add_libswift_module(ExperimentalRegex
- "${LIBSWIFT_EXPERIMENTAL_REGEX_SOURCES}"
- Regex.swift)
+ Regex.swift
+)