ci: speed up CI pipeline and parallelize tests

.scripts/build-autorest.yml

@@ -0,0 +1,5 @@
+steps:
+  - template: install.yml
+  - script: |
+      pnpm --filter @azure-tools/rlc-common build && pnpm --filter @autorest/typescript build
+    displayName: "Build AutoRest TypeScript dependencies"

.scripts/build-typespec-emitter.yml

@@ -0,0 +1,5 @@
+steps:
+  - template: install.yml
+  - script: |
+      pnpm --filter @azure-tools/rlc-common build && pnpm --filter @azure-tools/typespec-ts build
+    displayName: "Build TypeSpec emitter dependencies"

.scripts/build.yml

@@ -1,14 +1,5 @@
 steps:
-  - task: NodeTool@0
-    inputs:
-      versionSpec: "20.x"
-    displayName: "Use Node 20"
-  - script: |
-      npm install -g pnpm
-    displayName: "Install Pnpm"
-  - script: |
-      pnpm install
-    displayName: "Install package dependencies"
+  - template: install.yml
   - script: |
       pnpm build
     displayName: "Build"

.scripts/ci.yml

@@ -39,7 +39,7 @@ stages:
           name: ${{ parameters.LinuxPool }}
           demands: ImageOverride -equals ${{ parameters.OSVmImage }}
         steps:
-          - template: build.yml
+          - template: build-autorest.yml
           - script: |
               export MAIN_VERSION=$(node -p -e "require('./package.json').version")
               echo "##vso[task.setvariable variable=MAIN_VERSION;isOutput=true;]MAIN_VERSION"
@@ -135,7 +135,7 @@ stages:
           name: $[coalesce(variables['Pool'], '')]
           demands: ImageOverride -equals $(OSVmImage)
         steps:
-          - template: build.yml
+          - template: build-autorest.yml
           - script: npm run generate-swaggers
             workingDirectory: $(Build.SourcesDirectory)/packages/autorest.typescript
             displayName: "Generate HLC Test Clients"
@@ -157,7 +157,7 @@ stages:
           name: $[coalesce(variables['Pool'], '')]
           demands: ImageOverride -equals $(OSVmImage)
         steps:
-          - template: build.yml
+          - template: build-autorest.yml
           - script: npm run rlc-generate-swaggers
             workingDirectory: $(Build.SourcesDirectory)/packages/autorest.typescript
             displayName: "Generate RLC Test Clients"
@@ -190,7 +190,7 @@ stages:
           # Should set "demands:" to select image from azsdk-pool, but this would require creating separate
           # matrixes for the two pools.  Instead, we can use the default image from azsdk-pool for linux.
         steps:
-          - template: build.yml
+          - template: build-autorest.yml
           - task: NodeTool@0
             inputs:
               versionSpec: $(NodeTestVersion)
@@ -243,7 +243,7 @@ stages:
           name: $[coalesce(variables['Pool'], '')]
           demands: ImageOverride -equals $(OSVmImage)
         steps:
-          - template: build.yml
+          - template: build-typespec-emitter.yml
           - script: npm run smoke-test
             workingDirectory: $(Build.SourcesDirectory)/packages/typespec-test
             displayName: "Generate Code From TypeSpec"
@@ -269,7 +269,7 @@ stages:
           name: $[coalesce(variables['Pool'], '')]
           vmImage: $[coalesce(variables['OSVmImage'], '')]
         steps:
-          - template: build.yml
+          - template: build-typespec-emitter.yml
           - script: npm run smoke-test:contoso && npm run smoke-test:todo
             workingDirectory: $(Build.SourcesDirectory)/packages/typespec-test
             displayName: "Generate Contoso Code From TypeSpec"
@@ -290,7 +290,7 @@ stages:
           # Should set "demands:" to select image from azsdk-pool, but this would require creating separate
           # matrixes for the two pools.  Instead, we can use the default image from azsdk-pool for linux.
         steps:
-          - template: build.yml
+          - template: build-typespec-emitter.yml
           - script: npm run check-format
             workingDirectory: $(Build.SourcesDirectory)/packages/rlc-common
             displayName: "Check format for rlc common lib"
@@ -358,7 +358,7 @@ stages:
           # Should set "demands:" to select image from azsdk-pool, but this would require creating separate
           # matrixes for the two pools.  Instead, we can use the default image from azsdk-pool for linux.
         steps:
-          - template: build.yml
+          - template: build-typespec-emitter.yml
           - script: npm run test:azure
             workingDirectory: $(Build.SourcesDirectory)/packages/typespec-ts
             displayName: "Generate Azure Test Clients from Spector"

.scripts/install.yml

@@ -0,0 +1,23 @@
+steps:
+  - task: NodeTool@0
+    inputs:
+      versionSpec: "20.x"
+    displayName: "Use Node 20"
+  - script: |
+      npm install -g pnpm
+    displayName: "Install Pnpm"
+  - script: |
+      pnpm config set store-dir $(Pipeline.Workspace)/.pnpm-store
+    displayName: "Configure pnpm store"
+  - task: Cache@2
+    inputs:
+      key: 'pnpm | "$(Agent.OS)" | pnpm-lock.yaml'
+      restoreKeys: |
+        pnpm | "$(Agent.OS)"
+      path: $(Pipeline.Workspace)/.pnpm-store
+    displayName: "Restore pnpm store cache"
+  - script: |
+      pnpm install --frozen-lockfile
+    displayName: "Install package dependencies"
+    env:
+      PUPPETEER_SKIP_DOWNLOAD: "true"

.scripts/nightly-prepare.yml

@@ -13,9 +13,21 @@ steps:
   - script: |
       npm install -g pnpm
     displayName: "Install Pnpm"
+  - script: |
+      pnpm config set store-dir $(Pipeline.Workspace)/.pnpm-store
+    displayName: "Configure pnpm store"
+  - task: Cache@2
+    inputs:
+      key: 'pnpm | "$(Agent.OS)" | pnpm-lock.yaml'
+      restoreKeys: |
+        pnpm | "$(Agent.OS)"
+      path: $(Pipeline.Workspace)/.pnpm-store
+    displayName: "Restore pnpm store cache"
   - script: |
       pnpm install --no-frozen-lockfile
     displayName: "Install package dependencies"
+    env:
+      PUPPETEER_SKIP_DOWNLOAD: "true"
   - script: |
-      pnpm build
-    displayName: "Build"
+      pnpm --filter @azure-tools/rlc-common build && pnpm --filter @azure-tools/typespec-ts build
+    displayName: "Build TypeSpec emitter dependencies"

.scripts/smoke-test.yml

@@ -16,7 +16,7 @@ jobs:
       name: $[coalesce(variables['Pool'], '')]
       demands: ImageOverride -equals $(OSVmImage)
     steps:
-      - template: build.yml
+      - template: build-autorest.yml
       - script: npm run clone:specs
         workingDirectory: $(Build.SourcesDirectory)/packages/autorest.typescript
         displayName: "Clone Specs Repository"

packages/typespec-ts/package.json

@@ -20,8 +20,8 @@
     "clean": "rimraf ./dist ./typespec-output",
     "build": "tsc -p .",
     "test": "npm run test-next && npm run unit-test && npm run integration-test-ci",
-    "test:azure": "npm run test-next && npm run copy:typespec && npm run integration-test-ci:azure-rlc && npm run integration-test-ci:azure-modular",
-    "test:standard": "npm run test-next && npm run copy:typespec && npm run unit-test&& npm run integration-test-ci:rlc && npm run integration-test-ci:modular",
+    "test:azure": "npm run test-next && npm run copy:typespec && npm-run-all --parallel integration-test-ci:azure-rlc integration-test-ci:azure-modular",
+    "test:standard": "npm run test-next && npm run copy:typespec && npm run unit-test && npm-run-all --parallel integration-test-ci:rlc integration-test-ci:modular",
     "lint": "eslint src --ext .ts --max-warnings=0",
     "lint:fix": "eslint src --fix --ext .ts",
     "format": "npm run -s prettier -- --write",
@@ -58,7 +58,12 @@
     "stop-test-server": "npx tsp-spector server stop",
     "unit-test": "npm-run-all --parallel unit-test:rlc unit-test:modular",
     "unit-test:rlc": "cross-env TS_NODE_PROJECT=tsconfig.json mocha -r ts-node/register --experimental-specifier-resolution=node --experimental-modules=true  --timeout 36000 './test/unit/**/*.spec.ts'",
-    "unit-test:modular": "cross-env TS_NODE_PROJECT=tsconfig.test.json NODE_OPTIONS='--max-old-space-size=8192' mocha -r ts-node/register --experimental-specifier-resolution=node --experimental-modules=true  --no-timeout './test/modularUnit/**/*.spec.ts' --reporter-options maxDiffSize=20000",
+    "unit-test:modular": "npm-run-all unit-test:modular:core unit-test:modular:scenarios",
+    "unit-test:modular:core": "cross-env TS_NODE_PROJECT=tsconfig.test.json NODE_OPTIONS='--max-old-space-size=8192' mocha -r ts-node/register --experimental-specifier-resolution=node --experimental-modules=true --no-timeout --ignore './test/modularUnit/scenario-specs/**/*.spec.ts' './test/modularUnit/**/*.spec.ts' --reporter-options maxDiffSize=20000",
+    "unit-test:modular:scenarios": "npm-run-all regen-scenario-specs unit-test:modular:scenarios:run check:scenario-specs",
+    "unit-test:modular:scenarios:run": "cross-env TS_NODE_PROJECT=tsconfig.test.json NODE_OPTIONS='--max-old-space-size=8192' mocha -r ts-node/register --experimental-specifier-resolution=node --experimental-modules=true --no-timeout --parallel --jobs 4 './test/modularUnit/scenario-specs/**/*.spec.ts' --reporter-options maxDiffSize=20000",
+    "regen-scenario-specs": "npx tsx ./test/commands/generate-modular-scenario-specs.ts",
+    "check:scenario-specs": "npx tsx ./test/commands/check-modular-scenario-specs.ts",
     "regen-docs": "npm run build && tspd doc . --enable-experimental --output-dir ./website/src/content/docs/docs/emitters/clients/typespec-ts/reference --skip-js"
   },
   "author": "Jose Heredia <joheredi@microsoft.com>",

packages/typespec-ts/test/commands/check-modular-scenario-specs.ts

@@ -0,0 +1,54 @@
+import {
+  getActualScenarioSpecFiles,
+  getExpectedScenarioSpecFiles
+} from "./modular-scenario-specs.js";
+
+function main(): void {
+  const expectedFiles = getExpectedScenarioSpecFiles();
+  const actualFiles = getActualScenarioSpecFiles();
+
+  const missingFiles = [...expectedFiles.keys()].filter(
+    (file) => !actualFiles.has(file)
+  );
+  const extraFiles = [...actualFiles.keys()].filter(
+    (file) => !expectedFiles.has(file)
+  );
+  const staleFiles = [...expectedFiles.entries()]
+    .filter(
+      ([file, content]) =>
+        actualFiles.has(file) && actualFiles.get(file) !== content
+    )
+    .map(([file]) => file);
+
+  if (
+    missingFiles.length === 0 &&
+    extraFiles.length === 0 &&
+    staleFiles.length === 0
+  ) {
+    console.log(
+      `Scenario wrapper specs are up to date (${expectedFiles.size} files).`
+    );
+    return;
+  }
+
+  const parts: string[] = [
+    "Modular scenario wrapper specs are out of date.",
+    "Run `npm run regen-scenario-specs` in `packages/typespec-ts` and commit the resulting changes."
+  ];
+
+  if (missingFiles.length > 0) {
+    parts.push(`Missing files:\n- ${missingFiles.join("\n- ")}`);
+  }
+
+  if (extraFiles.length > 0) {
+    parts.push(`Unexpected files:\n- ${extraFiles.join("\n- ")}`);
+  }
+
+  if (staleFiles.length > 0) {
+    parts.push(`Changed files:\n- ${staleFiles.join("\n- ")}`);
+  }
+
+  throw new Error(parts.join("\n\n"));
+}
+
+main();

packages/typespec-ts/test/commands/gen-cadl-ranch.js

@@ -31,7 +31,7 @@ async function generateTypeSpecs(tag = "rlc", isDebugging, pathFilter) {
     list = list.filter((tsp) => tsp.outputPath === pathFilter);
   }
 
-  const maxConcurrentWorkers = 4;
+  const maxConcurrentWorkers = parseInt(process.env.GEN_WORKERS || "4", 10);
   let activePromises = [];
   for (const tsp of list) {
     if (isDebugging === true && tsp.debug !== true) {

packages/typespec-ts/test/commands/generate-modular-scenario-specs.ts

@@ -0,0 +1,34 @@
+import { mkdirSync, readFileSync, rmSync, writeFileSync } from "fs";
+import path from "path";
+import {
+  getExpectedScenarioSpecFiles,
+  scenarioSpecsRoot
+} from "./modular-scenario-specs.js";
+
+function main(): void {
+  const expectedFiles = getExpectedScenarioSpecFiles();
+  rmSync(scenarioSpecsRoot, { recursive: true, force: true });
+
+  for (const [relativeFilePath, content] of expectedFiles) {
+    const wrapperPath = path.join(scenarioSpecsRoot, relativeFilePath);
+    const wrapperDirectory = path.dirname(wrapperPath);
+    mkdirSync(wrapperDirectory, { recursive: true });
+    writeIfChanged(wrapperPath, content);
+  }
+
+  console.log(
+    `Generated ${expectedFiles.size} modular scenario wrapper specs.`
+  );
+}
+
+function writeIfChanged(filePath: string, content: string): void {
+  try {
+    if (readFileSync(filePath, "utf8") === content) {
+      return;
+    }
+  } catch {}
+
+  writeFileSync(filePath, content);
+}
+
+main();

packages/typespec-ts/test/commands/modular-scenario-specs.ts

@@ -0,0 +1,117 @@
+import { Dirent, readdirSync, readFileSync } from "fs";
+import path from "path";
+import { getDirname } from "../../src/utils/dirname.js";
+
+const { __dirname } = getDirname(import.meta.url);
+
+export const modularUnitRoot = path.resolve(__dirname, "..", "modularUnit");
+export const scenarioSpecsRoot = path.join(modularUnitRoot, "scenario-specs");
+
+export function getExpectedScenarioSpecFiles(): Map<string, string> {
+  const expectedFiles = new Map<string, string>();
+  const directories = listScenarioDirectories(modularUnitRoot).sort();
+
+  for (const scenarioDirectory of directories) {
+    const relativeFilePath = `${scenarioDirectory}.spec.ts`;
+    const wrapperPath = path.join(scenarioSpecsRoot, relativeFilePath);
+    const wrapperDirectory = path.dirname(wrapperPath);
+    const helperImport = getHelperImportPath(wrapperDirectory);
+    const normalizedScenarioDirectory = scenarioDirectory
+      .split(path.sep)
+      .join("/");
+    const content = `// Generated by test/commands/generate-modular-scenario-specs.ts
+import { registerScenarioDirectoryTests } from "${helperImport}";
+
+registerScenarioDirectoryTests("${normalizedScenarioDirectory}");
+`;
+
+    expectedFiles.set(relativeFilePath.split(path.sep).join("/"), content);
+  }
+
+  return expectedFiles;
+}
+
+export function getActualScenarioSpecFiles(): Map<string, string> {
+  return collectFiles(scenarioSpecsRoot);
+}
+
+function collectFiles(root: string): Map<string, string> {
+  const files = new Map<string, string>();
+  if (!exists(root)) {
+    return files;
+  }
+
+  for (const entry of readdirSync(root, { withFileTypes: true }).sort(
+    compareDirents
+  )) {
+    const fullPath = path.join(root, entry.name);
+    if (entry.isDirectory()) {
+      for (const [relativePath, content] of collectFiles(fullPath)) {
+        files.set(
+          path.join(entry.name, relativePath).split(path.sep).join("/"),
+          content
+        );
+      }
+      continue;
+    }
+
+    if (entry.isFile() && entry.name.endsWith(".spec.ts")) {
+      files.set(entry.name, readFileSync(fullPath, "utf8"));
+    }
+  }
+
+  return files;
+}
+
+function exists(directory: string): boolean {
+  try {
+    readdirSync(directory);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function listScenarioDirectories(modularUnitDirectory: string): string[] {
+  const scenariosRoot = path.join(modularUnitDirectory, "scenarios");
+  return findScenarioDirectories(scenariosRoot, scenariosRoot);
+}
+
+function findScenarioDirectories(
+  scenariosRoot: string,
+  currentDirectory: string
+): string[] {
+  const entries = readdirSync(currentDirectory, {
+    withFileTypes: true
+  }).sort(compareDirents);
+  const directories: string[] = [];
+
+  if (entries.some((entry) => entry.isFile() && entry.name.endsWith(".md"))) {
+    directories.push(path.relative(scenariosRoot, currentDirectory));
+  }
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) {
+      continue;
+    }
+
+    directories.push(
+      ...findScenarioDirectories(
+        scenariosRoot,
+        path.join(currentDirectory, entry.name)
+      )
+    );
+  }
+
+  return directories;
+}
+
+function compareDirents(left: Dirent, right: Dirent): number {
+  return left.name.localeCompare(right.name);
+}
+
+function getHelperImportPath(wrapperDirectory: string): string {
+  const helperPath = path.join(modularUnitRoot, "scenarioHarness.ts");
+  const relativePath = path.relative(wrapperDirectory, helperPath);
+  return relativePath.replace(/\.ts$/, ".js").split(path.sep).join("/");
+}