Introduce ExperimentalFeatures

These allow the parser to optionally support new
Swift features that haven't yet been enabled by
default.

Author: hamishknight

CodeGeneration/Sources/SyntaxSupport/KeywordSpec.swift

@@ -10,8 +10,15 @@
 //
 //===----------------------------------------------------------------------===//
 
+import SwiftSyntax
+
 public struct KeywordSpec {
   public var name: String
+
+  /// If `true`, this is for an experimental language feature, and any public
+  /// API generated should be SPI.
+  public var isExperimental: Bool
+
   public var isLexerClassified: Bool
   public var requiresLeadingSpace: Bool
   public var requiresTrailingSpace: Bool
@@ -24,10 +31,18 @@ public struct KeywordSpec {
     }
   }
 
+  /// Retrieve the attributes that should be printed on any API for the
+  /// generated keyword.
+  public var apiAttributes: AttributeListSyntax {
+    guard isExperimental else { return "" }
+    return AttributeListSyntax("@_spi(ExperimentalLanguageFeatures)").with(\.trailingTrivia, .newline)
+  }
+
   /// `isLexerClassified` determines whether the token kind is switched from being an identifier to a keyword in the lexer.
   /// This is true for keywords that used to be considered non-contextual.
-  init(_ name: String, isLexerClassified: Bool = false, requiresLeadingSpace: Bool = false, requiresTrailingSpace: Bool = false) {
+  init(_ name: String, isExperimental: Bool = false, isLexerClassified: Bool = false, requiresLeadingSpace: Bool = false, requiresTrailingSpace: Bool = false) {
     self.name = name
+    self.isExperimental = isExperimental
     self.isLexerClassified = isLexerClassified
     self.requiresLeadingSpace = requiresLeadingSpace
     self.requiresTrailingSpace = requiresTrailingSpace

CodeGeneration/Sources/SyntaxSupport/Node.swift

@@ -40,6 +40,10 @@ public class Node {
   /// The kind of node’s supertype. This kind must have `isBase == true`
   public let base: SyntaxNodeKind
 
+  /// If `true`, this is for an experimental language feature, and any public
+  /// API generated should be SPI.
+  public let isExperimental: Bool
+
   /// When the node name is printed for diagnostics, this name is used.
   /// If `nil`, `nameForDiagnostics` will print the parent node’s name.
   public let nameForDiagnostics: String?
@@ -85,10 +89,25 @@ public class Node {
     }
   }
 
+  /// Retrieve the attributes that should be printed on any API for the
+  /// generated node. If `forRaw` is true, this is for the raw syntax node.
+  public func apiAttributes(forRaw: Bool = false) -> AttributeListSyntax {
+    let attrList = AttributeListSyntax {
+      if isExperimental {
+        "@_spi(ExperimentalLanguageFeatures)"
+      }
+      if forRaw {
+        "@_spi(RawSyntax)"
+      }
+    }
+    return attrList.with(\.trailingTrivia, attrList.isEmpty ? [] : .newline)
+  }
+
   /// Construct the specification for a layout syntax node.
   init(
     kind: SyntaxNodeKind,
     base: SyntaxNodeKind,
+    isExperimental: Bool = false,
     nameForDiagnostics: String?,
     documentation: String? = nil,
     parserFunction: String? = nil,
@@ -100,6 +119,7 @@ public class Node {
 
     self.kind = kind
     self.base = base
+    self.isExperimental = isExperimental
     self.nameForDiagnostics = nameForDiagnostics
     self.documentation = docCommentTrivia(from: documentation)
     self.parserFunction = parserFunction
@@ -204,6 +224,7 @@ public class Node {
   init(
     kind: SyntaxNodeKind,
     base: SyntaxNodeKind,
+    isExperimental: Bool = false,
     nameForDiagnostics: String?,
     documentation: String? = nil,
     parserFunction: String? = nil,
@@ -212,6 +233,7 @@ public class Node {
     self.kind = kind
     precondition(base == .syntaxCollection)
     self.base = base
+    self.isExperimental = isExperimental
     self.nameForDiagnostics = nameForDiagnostics
     self.documentation = docCommentTrivia(from: documentation)
     self.parserFunction = parserFunction

CodeGeneration/Sources/SyntaxSupport/TokenSpec.swift

@@ -22,22 +22,36 @@ public struct TokenSpec {
   }
 
   public let varOrCaseName: TokenSyntax
+
+  /// If `true`, this is for an experimental language feature, and any public
+  /// API generated should be SPI.
+  public let isExperimental: Bool
+
   public let nameForDiagnostics: String
   public let text: String?
   public let kind: Kind
 
   fileprivate init(
     name: String,
+    isExperimental: Bool = false,
     nameForDiagnostics: String,
     text: String? = nil,
     kind: Kind
   ) {
     self.varOrCaseName = .identifier(name)
+    self.isExperimental = isExperimental
     self.nameForDiagnostics = nameForDiagnostics
     self.text = text
     self.kind = kind
   }
 
+  /// Retrieve the attributes that should be printed on any API for the
+  /// generated token.
+  public var apiAttributes: AttributeListSyntax {
+    guard isExperimental else { return "" }
+    return AttributeListSyntax("@_spi(ExperimentalLanguageFeatures)").with(\.trailingTrivia, .newline)
+  }
+
   static func punctuator(name: String, text: String) -> TokenSpec {
     return TokenSpec(
       name: name,

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftparserdiagnostics/SyntaxKindNameForDiagnosticsFile.swift

@@ -16,7 +16,7 @@ import SyntaxSupport
 import Utils
 
 let syntaxKindNameForDiagnosticFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
-  DeclSyntax("import SwiftSyntax")
+  DeclSyntax("@_spi(ExperimentalLanguageFeatures) import SwiftSyntax")
 
   try! ExtensionDeclSyntax("extension SyntaxKind") {
     try VariableDeclSyntax("var nameForDiagnostics: String?") {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/KeywordFile.swift

@@ -28,7 +28,12 @@ let keywordFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     """
   ) {
     for (index, keyword) in KEYWORDS.enumerated() {
-      DeclSyntax("case \(raw: keyword.escapedName)")
+      DeclSyntax(
+        """
+        \(keyword.apiAttributes)\
+        case \(raw: keyword.escapedName)
+        """
+      )
     }
 
     try! InitializerDeclSyntax("@_spi(RawSyntax) public init?(_ text: SyntaxText)") {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/RawSyntaxNodesFile.swift

@@ -42,7 +42,7 @@ let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   for node in SYNTAX_NODES where node.kind.isBase {
     DeclSyntax(
       """
-      @_spi(RawSyntax)
+      \(node.apiAttributes(forRaw: true))\
       public protocol \(node.kind.rawType)NodeProtocol: \(raw: node.base.rawProtocolType) {}
       """
     )
@@ -51,7 +51,7 @@ let rawSyntaxNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
   for node in SYNTAX_NODES {
     try! StructDeclSyntax(
       """
-      @_spi(RawSyntax)
+      \(node.apiAttributes(forRaw: true))\
       public struct \(node.kind.rawType): \(node.kind.isBase ? node.kind.rawProtocolType : node.base.rawProtocolType)
       """
     ) {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SwiftSyntaxDoccIndex.swift

@@ -37,7 +37,7 @@ let nodesSections: String = {
 
   for (baseKind, heading) in nodeKinds {
     let baseTypes = ["\(baseKind.syntaxType)", "\(baseKind.syntaxType)Protocol", "Missing\(baseKind.syntaxType)"]
-    let leafTypes = SYNTAX_NODES.filter({ $0.base == baseKind && !$0.kind.isMissing }).map(\.kind.syntaxType.description)
+    let leafTypes = SYNTAX_NODES.filter({ $0.base == baseKind && !$0.kind.isMissing && !$0.isExperimental }).map(\.kind.syntaxType.description)
     addSection(heading: heading, types: baseTypes + leafTypes)
   }
 
@@ -48,7 +48,7 @@ let nodesSections: String = {
       "SyntaxChildrenIndex",
     ]
       + SYNTAX_NODES.flatMap({ (node: Node) -> [String] in
-        guard let node = node.collectionNode else {
+        guard let node = node.collectionNode, !node.isExperimental else {
           return []
         }
         return [node.kind.syntaxType.description]
@@ -59,9 +59,12 @@ let nodesSections: String = {
       })
   )
 
-  addSection(heading: "Attributes", types: ATTRIBUTE_NODES.map(\.kind.syntaxType.description).sorted())
+  addSection(heading: "Attributes", types: ATTRIBUTE_NODES.filter({ !$0.isExperimental }).map(\.kind.syntaxType.description).sorted())
 
-  addSection(heading: "Miscellaneous Syntax", types: SYNTAX_NODES.map(\.kind.syntaxType.description).filter({ !handledSyntaxTypes.contains($0) }))
+  addSection(
+    heading: "Miscellaneous Syntax",
+    types: SYNTAX_NODES.filter({ !$0.isExperimental }).map(\.kind.syntaxType.description).filter({ !handledSyntaxTypes.contains($0) })
+  )
 
   addSection(heading: "Traits", types: TRAITS.map { "\($0.protocolName)" })
 

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxAnyVisitorFile.swift

@@ -79,6 +79,7 @@ let syntaxAnyVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     for node in SYNTAX_NODES where !node.kind.isBase {
       DeclSyntax(
         """
+        \(node.apiAttributes())\
         override open func visit(_ node: \(node.kind.syntaxType)) -> SyntaxVisitorContinueKind {
           return visitAny(node._syntaxNode)
         }
@@ -87,6 +88,7 @@ let syntaxAnyVisitorFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
 
       DeclSyntax(
         """
+        \(node.apiAttributes())\
         override open func visitPost(_ node: \(node.kind.syntaxType)) {
           visitAnyPost(node._syntaxNode)
         }

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxBaseNodesFile.swift

@@ -26,6 +26,7 @@ let syntaxBaseNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
       /// Extension point to add common methods to all ``\(node.kind.syntaxType)`` nodes.
       ///
       ///  - Warning: Do not conform to this protocol yourself.
+      \(node.apiAttributes())\
       public protocol \(node.kind.protocolType): \(raw: node.base.protocolType) {}
       """
     )
@@ -59,6 +60,7 @@ let syntaxBaseNodesFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     try! StructDeclSyntax(
       """
       \(node.documentation)
+      \(node.apiAttributes())\
       public struct \(node.kind.syntaxType): \(node.kind.protocolType), SyntaxHashable
       """
     ) {

CodeGeneration/Sources/generate-swiftsyntax/templates/swiftsyntax/SyntaxCollectionsFile.swift

@@ -33,6 +33,7 @@ let syntaxCollectionsFile = SourceFileSyntax(leadingTrivia: copyrightHeader) {
     try! StructDeclSyntax(
       """
       \(documentation)
+      \(node.node.apiAttributes())\
       public struct \(node.kind.syntaxType): SyntaxCollection, SyntaxHashable
       """
     ) {