[ASTGen] Infrastructure to implment ASTGen Incrementally

* 'ASTGenVisitor' has a reference to a legacy C++ Parser configured for
  ASTGen.
* If 'ASTGenVisitor' encounters a AST node that hasn't been migrated,
  call parse(Decl|Stmt|Expr|Type) to parse the position using the legacy
  parser.
* The legacy parser calls ASTGen's
  'swift_ASTGen_build(Decl|Stmt|Expr|Type)' for each ASTNode "parsing"
  (unless the call is not directly from the ASTGen.)

rdar://117151886

Author: rintaro

include/module.modulemap

@@ -10,6 +10,12 @@ module ASTBridging {
   export *
 }
 
+module ParseBridging {
+  header "swift/Parse/ParseBridging.h"
+  requires cplusplus
+  export *
+}
+
 module SILBridging {
   header "swift/SIL/SILBridging.h"
   requires cplusplus

include/swift/AST/ASTBridgingWrappers.def

@@ -59,8 +59,10 @@
 // optional parameters.
 AST_BRIDGING_WRAPPER_NONNULL(Decl)
 AST_BRIDGING_WRAPPER_NONNULL(DeclContext)
+AST_BRIDGING_WRAPPER_NONNULL(SourceFile)
 AST_BRIDGING_WRAPPER_NULLABLE(Stmt)
 AST_BRIDGING_WRAPPER_NULLABLE(Expr)
+AST_BRIDGING_WRAPPER_NULLABLE(Pattern)
 AST_BRIDGING_WRAPPER_NULLABLE(TypeRepr)
 
 // Misc AST types to generate wrappers for.

include/swift/Bridging/ASTGen.h

@@ -11,6 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/AST/ASTBridging.h"
+#include "swift/Parse/ParseBridging.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -40,6 +41,23 @@ void *_Nullable swift_ASTGen_parseSourceFile(const char *_Nonnull buffer,
                                              void *_Nullable ctx);
 void swift_ASTGen_destroySourceFile(void *_Nonnull sourceFile);
 
+/// Check whether the given source file round-trips correctly. Returns 0 if
+/// round-trip succeeded, non-zero otherwise.
+int swift_ASTGen_roundTripCheck(void *_Nonnull sourceFile);
+
+/// Emit parser diagnostics for given source file.. Returns non-zero if any
+/// diagnostics were emitted.
+int swift_ASTGen_emitParserDiagnostics(
+    void *_Nonnull diagEngine, void *_Nonnull sourceFile, int emitOnlyErrors,
+    int downgradePlaceholderErrorsToWarnings);
+
+// Build AST nodes for the top-level entities in the syntax.
+void swift_ASTGen_buildTopLevelASTNodes(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedDeclContext declContext, BridgedASTContext astContext,
+    BridgedLegacyParser legacyParser, void *_Nonnull outputContext,
+    void (*_Nonnull)(void *_Nonnull, void *_Nonnull));
+
 void *_Nullable swift_ASTGen_resolveMacroType(const void *_Nonnull macroType);
 void swift_ASTGen_destroyMacro(void *_Nonnull macro);
 
@@ -84,6 +102,38 @@ bool swift_ASTGen_pluginServerLoadLibraryPlugin(
     void *_Nonnull handle, const char *_Nonnull libraryPath,
     const char *_Nonnull moduleName, BridgedStringRef *_Nullable errorOut);
 
+/// Build a TypeRepr for AST node for the type at the given source location in
+/// the specified file.
+swift::TypeRepr *_Nullable swift_ASTGen_buildTypeRepr(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
+/// Build a Decl for AST node for the type at the given source location in the
+/// specified file.
+swift::Decl *_Nullable swift_ASTGen_buildDecl(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
+/// Build a Expr for AST node for the type at the given source location in the
+/// specified file.
+swift::Expr *_Nullable swift_ASTGen_buildExpr(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
+/// Build a Stmt for AST node for the type at the given source location in the
+/// specified file.
+swift::Stmt *_Nullable swift_ASTGen_buildStmt(
+    BridgedDiagnosticEngine diagEngine, void *_Nonnull sourceFile,
+    BridgedSourceLoc sourceLoc, BridgedDeclContext declContext,
+    BridgedASTContext astContext, BridgedLegacyParser legacyParser,
+    BridgedSourceLoc *_Nonnull endSourceLoc);
+
 #ifdef __cplusplus
 }
 #endif

include/swift/Parse/ParseBridging.h

@@ -0,0 +1,53 @@
+//===--- ParseBridging.h --------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2023 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_PARSE_PARSEBRIDGING_H
+#define SWIFT_PARSE_PARSEBRIDGING_H
+
+#include "swift/AST/ASTBridging.h"
+#include "swift/Basic/BasicBridging.h"
+
+#ifdef USED_IN_CPP_SOURC
+#include "swift/Parse/Parser.h"
+#else
+namespace swift {
+class Parser;
+}
+#endif
+
+SWIFT_BEGIN_NULLABILITY_ANNOTATIONS
+
+class BridgedLegacyParser {
+  swift::Parser *_Nonnull const handle;
+
+public:
+#ifdef USED_IN_CPP_SOURCE
+  BridgedLegacyParser(swift::Parser &P) : handle(&P) {}
+
+  swift::Parser &unbridged() const { return *handle; }
+#endif
+
+  BridgedASTContext getASTContext() const;
+  BridgedDiagnosticEngine getDiagnosticEngine() const;
+  BridgedStringRef getSourceBuffer() const;
+  BridgedDeclContext getCurrDeclContext() const;
+
+  BridgedExpr parseExpr(BridgedSourceLoc loc, BridgedDeclContext DC,
+                        bool isExprBasic) const;
+  BridgedDecl parseDecl(BridgedSourceLoc loc, BridgedDeclContext DC) const;
+  BridgedStmt parseStmt(BridgedSourceLoc loc, BridgedDeclContext DC) const;
+  BridgedTypeRepr parseType(BridgedSourceLoc loc, BridgedDeclContext DC) const;
+};
+
+SWIFT_END_NULLABILITY_ANNOTATIONS
+
+#endif // SWIFT_PARSE_PARSEBRIDGING_H

include/swift/Parse/Parser.h

@@ -158,6 +158,11 @@ class Parser {
   bool InSwiftKeyPath = false;
   bool InFreestandingMacroArgument = false;
 
+#if SWIFT_BUILD_SWIFT_SYNTAX
+  // This Parser is a fallback parser for ASTGen.
+  bool IsForASTGen = false;
+#endif
+
   // A cached answer to
   //     Context.LangOpts.hasFeature(Feature::NoncopyableGenerics)
   // to ensure there's no parsing performance regression.
@@ -957,7 +962,8 @@ class Parser {
 
   ParserResult<Decl> parseDecl(bool IsAtStartOfLineOrPreviousHadSemi,
                                bool IfConfigsAreDeclAttrs,
-                               llvm::function_ref<void(Decl *)> Handler);
+                               llvm::function_ref<void(Decl *)> Handler,
+                               bool fromASTGen = false);
 
   std::pair<std::vector<Decl *>, llvm::Optional<Fingerprint>>
   parseDeclListDelayed(IterableDeclContext *IDC);
@@ -1349,50 +1355,6 @@ class Parser {
   /// Get the location for a type error.
   SourceLoc getTypeErrorLoc() const;
 
-  /// Callback function used for creating a C++ AST from the syntax node at the given source location.
-  ///
-  /// The arguments to this callback are the source file to pass into ASTGen (the exported source file)
-  /// and the source location pointer to pass into ASTGen (to find the syntax node).
-  ///
-  /// The callback returns the new AST node and the ending location of the syntax node. If the AST node
-  /// is NULL, something went wrong.
-  template<typename T>
-  using ASTFromSyntaxTreeCallback = std::pair<T*, const void *>(
-      void *sourceFile, const void *sourceLoc
-  );
-
-  /// Parse by constructing a C++ AST node from the Swift syntax tree via ASTGen.
-  template<typename T>
-  ParserResult<T> parseASTFromSyntaxTree(
-      llvm::function_ref<ASTFromSyntaxTreeCallback<T>> body
-  ) {
-    if (!Context.LangOpts.hasFeature(Feature::ASTGenTypes))
-      return nullptr;
-
-    auto exportedSourceFile = SF.getExportedSourceFile();
-    if (!exportedSourceFile)
-      return nullptr;
-
-    // Perform the translation.
-    auto sourceLoc = Tok.getLoc().getOpaquePointerValue();
-    T* astNode;
-    const void *endLocPtr;
-    std::tie(astNode, endLocPtr) = body(exportedSourceFile, sourceLoc);
-
-    if (!astNode) {
-      assert(false && "Could not build AST node from syntax tree");
-      return nullptr;
-    }
-
-    // Reset the lexer to the ending location.
-    StringRef contents =
-        SourceMgr.extractText(SourceMgr.getRangeForBuffer(L->getBufferID()));
-    L->resetToOffset((const char *)endLocPtr - contents.data());
-    L->lex(Tok);
-
-    return makeParserResult(astNode);
-  }
-
   //===--------------------------------------------------------------------===//
   // Type Parsing
 
@@ -1409,9 +1371,10 @@ class Parser {
       ParseTypeReason reason);
 
   ParserResult<TypeRepr> parseType();
-  ParserResult<TypeRepr> parseType(
-      Diag<> MessageID,
-      ParseTypeReason reason = ParseTypeReason::Unspecified);
+  ParserResult<TypeRepr>
+  parseType(Diag<> MessageID,
+            ParseTypeReason reason = ParseTypeReason::Unspecified,
+            bool fromASTGen = false);
 
   /// Parse a type optionally prefixed by a list of named opaque parameters. If
   /// no params present, return 'type'. Otherwise, return 'type-named-opaque'.
@@ -1735,7 +1698,8 @@ class Parser {
   ParserResult<Expr> parseExprBasic(Diag<> ID) {
     return parseExprImpl(ID, /*isExprBasic=*/true);
   }
-  ParserResult<Expr> parseExprImpl(Diag<> ID, bool isExprBasic);
+  ParserResult<Expr> parseExprImpl(Diag<> ID, bool isExprBasic,
+                                   bool fromASTGen = false);
   ParserResult<Expr> parseExprIs();
   ParserResult<Expr> parseExprAs();
   ParserResult<Expr> parseExprArrow();
@@ -1944,7 +1908,7 @@ class Parser {
 
   bool isTerminatorForBraceItemListKind(BraceItemListKind Kind,
                                         ArrayRef<ASTNode> ParsedDecls);
-  ParserResult<Stmt> parseStmt();
+  ParserResult<Stmt> parseStmt(bool fromASTGen = false);
   ParserStatus parseExprOrStmt(ASTNode &Result);
   ParserResult<Stmt> parseStmtBreak();
   ParserResult<Stmt> parseStmtContinue();
@@ -2075,6 +2039,18 @@ class Parser {
   bool parseLegacyTildeCopyable(SourceLoc *parseTildeCopyable,
                                 ParserStatus &Status,
                                 SourceLoc &TildeCopyableLoc);
+
+  //===--------------------------------------------------------------------===//
+  // ASTGen support.
+
+  /// Parse a TypeRepr from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<TypeRepr> parseTypeReprFromSyntaxTree();
+  /// Parse a Stmt from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<Stmt> parseStmtFromSyntaxTree();
+  /// Parse a Decl from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<Decl> parseDeclFromSyntaxTree();
+  /// Parse an Expr from the syntax tree. i.e. SF->getExportedSourceFile()
+  ParserResult<Expr> parseExprFromSyntaxTree();
 };
 
 /// Describes a parsed declaration name.

lib/ASTGen/CMakeLists.txt

@@ -13,6 +13,7 @@ add_pure_swift_host_library(swiftASTGen STATIC
   Sources/ASTGen/DiagnosticsBridge.swift
   Sources/ASTGen/Exprs.swift
   Sources/ASTGen/Generics.swift
+  Sources/ASTGen/LegacyParse.swift
   Sources/ASTGen/Literals.swift
   Sources/ASTGen/Macros.swift
   Sources/ASTGen/ParameterClause.swift