feat(linter/plugins): add SourceCode#tokensAndComments getter (#16503)

Add a getter `tokensAndComments` to `SourceCode`. ESLint has this property, but it's undocumented, but many of ESLint's built-in rules rely on it. Probably other ESLint plugins rely on it too.

Author: overlookmotel

apps/oxlint/src-js/plugins/source_code.ts

@@ -19,11 +19,13 @@ import {
 import { resetScopeManager, SCOPE_MANAGER } from "./scope.ts";
 import * as scopeMethods from "./scope.ts";
 import { resetTokens } from "./tokens.ts";
+import { tokens, tokensAndComments, initTokens, initTokensAndComments } from "./tokens.ts";
 import * as tokenMethods from "./tokens.ts";
 import { debugAssertIsNonNull } from "../utils/asserts.ts";
 
 import type { Program } from "../generated/types.d.ts";
 import type { Ranged } from "./location.ts";
+import type { Token, CommentToken } from "./tokens.ts";
 import type { BufferWithArrays, Node } from "./types.ts";
 import type { ScopeManager } from "./scope.ts";
 
@@ -178,6 +180,22 @@ export const SOURCE_CODE = Object.freeze({
     return lines;
   },
 
+  /**
+   * Array of all tokens and comments in the file, in source order.
+   */
+  // This property is present in ESLint's `SourceCode`, but is undocumented
+  get tokensAndComments(): (Token | CommentToken)[] {
+    if (tokensAndComments === null) {
+      if (tokens === null) {
+        if (sourceText === null) initSourceText();
+        initTokens();
+      }
+      initTokensAndComments();
+    }
+    debugAssertIsNonNull(tokensAndComments);
+    return tokensAndComments;
+  },
+
   /**
    * Get the source code for the given node.
    * @param node? - The AST node to get the text for.

apps/oxlint/src-js/plugins/tokens.ts

@@ -135,7 +135,7 @@ const INCLUDE_COMMENTS_SKIP_OPTIONS: SkipOptions = { includeComments: true, skip
 // Created lazily only when needed.
 export let tokens: Token[] | null = null;
 let comments: CommentToken[] | null = null;
-let tokensAndComments: (Token | CommentToken)[] | null = null;
+export let tokensAndComments: (Token | CommentToken)[] | null = null;
 
 // TS-ESLint `parse` method.
 // Lazy-loaded only when needed, as it's a lot of code.
@@ -178,7 +178,7 @@ export function initTokens() {
  * Caller must ensure `tokens` and `comments` are initialized before calling this function,
  * by calling `initTokens()` if `tokens === null`.
  */
-function initTokensAndComments() {
+export function initTokensAndComments() {
   debugAssertIsNonNull(tokens);
   debugAssertIsNonNull(comments);
 

apps/oxlint/test/fixtures/tokens/output.snap.md

@@ -25,7 +25,33 @@
  8 | `-> // Trailing comment
    `----
 
-Found 0 warnings and 1 error.
+  x tokens-plugin(tokens): Tokens and comments:
+  | Line              loc=1:0-1:18    range=0-18 " Leading comment"
+  | Keyword           loc=3:0-3:3     range=20-23 "let"
+  | Identifier        loc=3:4-3:5     range=24-25 "x"
+  | Punctuator        loc=3:6-3:7     range=26-27 "="
+  | Block             loc=3:8-3:28    range=28-48 " inline comment "
+  | Numeric           loc=3:29-3:30   range=49-50 "1"
+  | Punctuator        loc=3:30-3:31   range=50-51 ";"
+  | Line              loc=5:0-5:18    range=53-71 " Another comment"
+  | Keyword           loc=6:0-6:3     range=72-75 "let"
+  | Identifier        loc=6:4-6:5     range=76-77 "y"
+  | Punctuator        loc=6:6-6:7     range=78-79 "="
+  | Numeric           loc=6:8-6:9     range=80-81 "2"
+  | Punctuator        loc=6:9-6:10    range=81-82 ";"
+  | Line              loc=8:0-8:19    range=84-103 " Trailing comment"
+   ,-[files/index.js:1:1]
+ 1 | ,-> // Leading comment
+ 2 | |   
+ 3 | |   let x = /* inline comment */ 1;
+ 4 | |   
+ 5 | |   // Another comment
+ 6 | |   let y = 2;
+ 7 | |   
+ 8 | `-> // Trailing comment
+   `----
+
+Found 0 warnings and 2 errors.
 Finished in Xms on 1 file using X threads.
 ```
 

apps/oxlint/test/fixtures/tokens/plugin.ts

@@ -2,10 +2,14 @@ import type { Plugin, Rule } from "#oxlint";
 
 const rule: Rule = {
   create(context) {
-    const { sourceCode } = context,
-      { ast } = sourceCode;
+    const { sourceCode } = context;
 
-    // Note: Comments should not appear in `ast.tokens`
+    // Get this first to check it works before `sourceText` or `ast` are accessed
+    const { tokensAndComments } = sourceCode;
+
+    const { ast } = sourceCode;
+
+    // `ast.tokens` does not include comments
     context.report({
       message:
         `Tokens:\n` +
@@ -23,6 +27,24 @@ const rule: Rule = {
       node: { range: [0, sourceCode.text.length] },
     });
 
+    // `sourceCode.tokensAndComments` does include comments
+    context.report({
+      message:
+        `Tokens and comments:\n` +
+        tokensAndComments
+          .map(
+            ({ type, loc, range, value }) =>
+              `${type.padEnd(17)} ` +
+              `loc=${loc.start.line}:${loc.start.column}-${loc.end.line}:${loc.end.column} `.padEnd(
+                16,
+              ) +
+              `range=${range[0]}-${range[1]} `.padEnd(10) +
+              `"${value}"`,
+          )
+          .join("\n"),
+      node: { range: [0, sourceCode.text.length] },
+    });
+
     return {};
   },
 };