Merge pull request #141 from aminya/faster-llvm [skip ci]

Author: aminya

cspell.config.yaml

@@ -8,6 +8,7 @@ ignorePaths:
   - .git/
   - dist/
   - dev/cpp_vcpkg_project
+  - "**/node_modules/"
 words:
   - aarch
   - aminya

dev/scripts/pack-exe.ts

@@ -1,3 +1,4 @@
+/* eslint-disable import/no-extraneous-dependencies */
 import { node } from "execa"
 
 function getPlatformName() {

dist/node12/setup_cpp.js

@@ -29,9 +29,9 @@
     "docs": "shx rm -rf packages/*/README.md && pnpm -r exec readme --path ../../dev/readme/template.md -y && pnpm -r exec ts-readme",
     "format": "run-s lint.prettier",
     "lint": "run-p --aggregate-output --continue-on-error lint.cspell lint.eslint lint.prettier lint.tsc",
-    "lint.cspell": "cspell lint --no-progress --show-suggestions",
+    "lint.cspell": "cspell lint --no-progress --show-suggestions --cache --cache-location ./.cache/cspell/.cspellcache",
     "lint.eslint": "eslint **/*.{ts,tsx,js,jsx,cjs,mjs,json,yaml} --no-error-on-unmatched-pattern --cache --cache-location ./.cache/eslint/ --fix",
-    "lint.prettier": "prettier --write .",
+    "lint.prettier": "prettier --list-different --write .",
     "lint.tsc": "tsc --noEmit",
     "pack.exe": "shx rm -rf ./dist/tsconfig.tsbuildinfo && ts-node --esm ./dev/scripts/pack-exe.ts",
     "prepare": "pnpm run -r build && pnpm run -w build",

dist/node12/setup_cpp.js.map

@@ -1,4 +1,4 @@
-import { setupLLVM, VERSIONS, getUrl, setupClangTools, getLinuxUrl } from "../llvm"
+import { setupLLVM, setupClangTools } from "../llvm"
 import { getSpecificVersionAndUrl } from "../../utils/setup/version"
 import { isUrlOnline } from "is-url-online"
 import { setupTmpDir, testBin } from "../../utils/tests/test-helpers"
@@ -9,6 +9,7 @@ import { chmodSync } from "fs"
 import { getVersion } from "../../versions/versions"
 import { ubuntuVersion } from "../../utils/env/ubuntu_version"
 import * as io from "@actions/io"
+import { getLinuxUrl, getUrl, VERSIONS } from "../llvm_url"
 
 jest.setTimeout(400000)
 async function testUrl(version: string) {

dist/node16/setup_cpp.js

@@ -1,319 +1,57 @@
 import { join, addExeExt } from "patha"
 import { delimiter } from "path"
-import semverLte from "semver/functions/lte"
 import semverMajor from "semver/functions/major"
-import { isUrlOnline } from "is-url-online"
-import { InstallationInfo, PackageInfo, setupBin } from "../utils/setup/setupBin"
-import { extractExe, extractTarByExe } from "../utils/setup/extract"
-import {
-  getSpecificVersionAndUrl,
-  getSpecificVersions,
-  getVersions,
-  semverCoerceIfInvalid,
-} from "../utils/setup/version"
+import { InstallationInfo, setupBin } from "../utils/setup/setupBin"
+import { semverCoerceIfInvalid } from "../utils/setup/version"
 import { setupMacOSSDK } from "../macos-sdk/macos-sdk"
 import { addEnv } from "../utils/env/addEnv"
-import { setOutput } from "@actions/core"
 import { setupAptPack, updateAptAlternatives } from "../utils/setup/setupAptPack"
 import { info, warning } from "ci-log"
 import { existsSync } from "fs"
 import ciDetect from "@npmcli/ci-detect"
 import { setupGcc } from "../gcc/gcc"
 import { getVersion } from "../versions/versions"
-import { isArch } from "../utils/env/isArch"
 import { isUbuntu } from "../utils/env/isUbuntu"
+import { getLLVMPackageInfo } from "./llvm_url"
 
-//================================================
-// Version
-//================================================
-
-/** The specific and minimum LLVM versions supported by this action. */
-export const VERSIONS: Set<string> = getVersions([
-  "3.5.0",
-  "3.5.1",
-  "3.5.2",
-  "3.6.0",
-  "3.6.1",
-  "3.6.2",
-  "3.7.0",
-  "3.7.1",
-  "3.8.0",
-  "3.8.1",
-  "3.9.0",
-  "3.9.1",
-  "4.0.0",
-  "4.0.1",
-  "5.0.0",
-  "5.0.1",
-  "5.0.2",
-  "6.0.0",
-  "6.0.1",
-  "7.0.0",
-  "7.0.1",
-  "7.1.0",
-  "8.0.0",
-  "8.0.1",
-  "9.0.0",
-  "9.0.1",
-  "10.0.0",
-  "10.0.1",
-  "11.0.0",
-  "11.0.1",
-  "11.1.0",
-  "12.0.0",
-  "12.0.1",
-  "13.0.0",
-  "13.0.1",
-  "14.0.0",
-  "14.0.1",
-  "14.0.2",
-  "14.0.3",
-  "14.0.4",
-  "14.0.5",
-  "14.0.6",
-  "15.0.0",
-  "15.0.1",
-  "15.0.2",
-])
-
-//================================================
-// URL
-//================================================
-
-/** Gets a LLVM download URL for GitHub. */
-function getGitHubUrl(version: string, prefix: string, suffix: string): string {
-  const file = `${prefix}${version}${suffix}`
-  return `https://github.com/llvm/llvm-project/releases/download/llvmorg-${version}/${file}`
-}
-
-/** Gets a LLVM download URL for https://releases.llvm.org. */
-function getReleaseUrl(version: string, prefix: string, suffix: string): string {
-  const file = `${prefix}${version}${suffix}`
-  return `https://releases.llvm.org/${version}/${file}`
-}
-
-/** The LLVM versions that were never released for the Darwin platform. */
-const DARWIN_MISSING: Set<string> = new Set([
-  "3.5.1",
-  "3.6.1",
-  "3.6.2",
-  "3.7.1",
-  "3.8.1",
-  "3.9.1",
-  "6.0.1",
-  "7.0.1",
-  "7.1.0",
-  "8.0.1",
-  "11.0.1",
-  "11.1.0",
-  "12.0.1",
-])
-
-/** Gets an LLVM download URL for the Darwin platform. */
-function getDarwinUrl(version: string): string | null {
-  if (DARWIN_MISSING.has(version)) {
-    return null
-  }
-
-  const darwin = version === "9.0.0" ? "-darwin-apple" : "-apple-darwin"
-  const prefix = "clang+llvm-"
-  const suffix = `-x86_64${darwin}.tar.xz`
-  if (semverLte(version, "9.0.1")) {
-    return getReleaseUrl(version, prefix, suffix)
-  } else {
-    return getGitHubUrl(version, prefix, suffix)
-  }
-}
-
-/**
- * The LLVM versions that should use the last RC version instead of the release version for the Linux (Ubuntu) platform.
- * This is useful when there were binaries released for the Linux (Ubuntu) platform for the last RC version but not for
- * the actual release version.
- */
-const UBUNTU_RC: Map<string, string> = new Map()
-
-/**
- * The (latest) Ubuntu versions for each LLVM version.
- *
- * https://github.com/llvm/llvm-project/releases/tag/llvmorg-14.0.1 or https://releases.llvm.org/14.0.1
- */
-// TODO change based on ubuntu version
-const UBUNTU_SUFFIX_MAP: { [key: string]: string } = {
-  "3.5.0": "-ubuntu-14.04",
-  "3.5.1": "",
-  "3.5.2": "-ubuntu-14.04",
-  "3.6.0": "-ubuntu-14.04",
-  "3.6.1": "-ubuntu-14.04",
-  "3.6.2": "-ubuntu-14.04",
-  "3.7.0": "-ubuntu-14.04",
-  "3.7.1": "-ubuntu-14.04",
-  "3.8.0": "-ubuntu-16.04",
-  "3.8.1": "-ubuntu-16.04",
-  "3.9.0": "-ubuntu-16.04",
-  "3.9.1": "-ubuntu-16.04",
-  "4.0.0": "-ubuntu-16.04",
-  "5.0.0": "-ubuntu16.04",
-  "5.0.1": "-ubuntu-16.04",
-  "5.0.2": "-ubuntu-16.04",
-  "6.0.0": "-ubuntu-16.04",
-  "6.0.1": "-ubuntu-16.04",
-  "7.0.0": "-ubuntu-16.04",
-  "7.0.1": "-ubuntu-18.04",
-  "7.1.0": "-ubuntu-14.04",
-  "8.0.0": "-ubuntu-18.04",
-  "9.0.0": "-ubuntu-18.04",
-  "9.0.1": "-ubuntu-16.04",
-  "10.0.0": "-ubuntu-18.04",
-  "10.0.1": "-ubuntu-16.04",
-  "11.0.0": "-ubuntu-20.04",
-  "11.0.1": "-ubuntu-16.04",
-  "11.1.0": "-ubuntu-16.04",
-  "12.0.0": "-ubuntu-20.04",
-  "12.0.1": "-ubuntu-16.04",
-  "13.0.0": "-ubuntu-20.04",
-  "13.0.0-ubuntu-16.04": "-ubuntu-16.04",
-  "13.0.0-ubuntu-20.04": "-ubuntu-20.04",
-  "13.0.1": "-ubuntu-18.04",
-  "13.0.1-ubuntu-18.04": "-ubuntu-18.04",
-  "14.0.0": "-ubuntu-18.04",
-  // "14.0.1": "-ubuntu-18.04",  // only available for powerpc64le
-  "15.0.2": "-rhel86",
+export async function setupLLVM(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
+  const installationInfo = await setupLLVMWithoutActivation(version, setupDir, arch)
+  await activateLLVM(installationInfo.installDir ?? setupDir, version)
+  return installationInfo
 }
 
-/** The latest supported LLVM version for the Linux (Ubuntu) platform. */
-const MAX_UBUNTU: string = "15.0.2"
-
-/** Gets an LLVM download URL for the Linux (Ubuntu) platform. */
-export function getLinuxUrl(versionGiven: string): string {
-  let version = versionGiven
-
-  const rc = UBUNTU_RC.get(version)
-  if (rc !== undefined) {
-    version = rc
-  }
-
-  let linuxVersion: string
-  // ubuntu-version is specified
-  if (version.includes("ubuntu")) {
-    const givenUbuntuVersion = version.replace(/-ubuntu-.*/, "")
-    if (!VERSIONS.has(givenUbuntuVersion)) {
-      throw new Error(`Unsupported Ubuntu version: ${givenUbuntuVersion}`)
-    }
-    linuxVersion = version.replace(givenUbuntuVersion, "")
-    version = getSpecificVersions(VERSIONS, givenUbuntuVersion)[0]
-  } else if (version !== "" && version in UBUNTU_SUFFIX_MAP) {
-    linuxVersion = UBUNTU_SUFFIX_MAP[version]
-  } else {
-    // default to the maximum version
-    linuxVersion = UBUNTU_SUFFIX_MAP[MAX_UBUNTU]
-    warning(`Falling back to LLVM version ${MAX_UBUNTU} ${linuxVersion} for the Ubuntu.`)
-  }
-
-  const prefix = "clang+llvm-"
+let installedDeps = false
 
-  let suffix: string
-  if (version === "5.0.0") {
-    suffix = `-linux-x86_64${linuxVersion}.tar.xz`
-  } else if (linuxVersion.includes("-rhel86")) {
-    suffix = `-x86_64-unknown-linux-gnu${linuxVersion}.tar.xz`
-  } else {
-    suffix = `-x86_64-linux-gnu${linuxVersion}.tar.xz`
-  }
+async function setupLLVMWithoutActivation(version: string, setupDir: string, arch: string) {
+  const installationInfoPromise = setupBin("llvm", version, getLLVMPackageInfo, setupDir, arch)
 
-  if (semverLte(version, "9.0.1")) {
-    return getReleaseUrl(version, prefix, suffix)
+  let depsPromise: Promise<void>
+  if (!installedDeps) {
+    depsPromise = setupLLVMDeps(arch)
+    // eslint-disable-next-line require-atomic-updates
+    installedDeps = true
   } else {
-    return getGitHubUrl(version, prefix, suffix)
+    depsPromise = Promise.resolve()
   }
-}
-
-/** The LLVM versions that were never released for the Windows platform. */
-const WIN32_MISSING: Set<string> = new Set(["10.0.1"])
 
-/** Gets an LLVM download URL for the Windows platform. */
-async function getWin32Url(version: string): Promise<string | null> {
-  if (WIN32_MISSING.has(version)) {
-    return null
-  }
+  // install LLVM and its dependencies in parallel
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  const [installationInfo, _] = await Promise.all([installationInfoPromise, depsPromise])
 
-  const prefix = "LLVM-"
-  const suffix = semverLte(version, "3.7.0") ? "-win32.exe" : "-win64.exe"
-
-  const olderThan9_1 = semverLte(version, "9.0.1")
-  let url: string
-  let fallback = false
-  if (olderThan9_1) {
-    url = getReleaseUrl(version, prefix, suffix)
-    if (!(await isUrlOnline(url))) {
-      fallback = true // fallback to github
-    }
-  }
-  if (fallback || !olderThan9_1) {
-    url = getGitHubUrl(version, prefix, suffix)
-  }
-
-  return url!
-}
-
-/** Gets an LLVM download URL. */
-export function getUrl(platform: string, version: string): string | null | Promise<string | null> {
-  switch (platform) {
-    case "darwin":
-      return getDarwinUrl(version)
-    case "linux":
-      return getLinuxUrl(version)
-    case "win32":
-      return getWin32Url(version)
-    default:
-      return null
-  }
-}
-
-//================================================
-// Exports
-//================================================
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-async function getLLVMPackageInfo(version: string, platform: NodeJS.Platform, _arch: string): Promise<PackageInfo> {
-  const [specificVersion, url] = await getSpecificVersionAndUrl(VERSIONS, platform, version, getUrl)
-  setOutput("version", specificVersion)
-  return {
-    url,
-    extractedFolderName: "",
-    binRelativeDir: "bin",
-    binFileName: addExeExt("clang"),
-    extractFunction:
-      platform === "win32"
-        ? extractExe
-        : (file: string, dest: string) => {
-            return extractTarByExe(file, dest, ["--strip-components=1"])
-          },
-  }
-}
-
-export async function setupLLVM(version: string, setupDir: string, arch: string): Promise<InstallationInfo> {
-  const installationInfo = await _setupLLVM(version, setupDir, arch)
-  await activateLLVM(installationInfo.installDir ?? setupDir, version)
   return installationInfo
 }
 
-let didInit = false
-async function _setupLLVM(version: string, setupDir: string, arch: string) {
-  const installationInfo = await setupBin("llvm", version, getLLVMPackageInfo, setupDir, arch)
-  if (!didInit) {
-    if (process.platform === "linux") {
-      // install llvm build dependencies
-      await setupGcc(getVersion("gcc", undefined), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
-      if (isArch()) {
-        // setupPacmanPack("ncurses")
-        // TODO: install libtinfo ?
-      } else if (isUbuntu()) {
-        await setupAptPack("libtinfo-dev")
-      }
+async function setupLLVMDeps(arch: string) {
+  if (process.platform === "linux") {
+    // install llvm build dependencies
+    await setupGcc(getVersion("gcc", undefined), "", arch) // using llvm requires ld, an up to date libstdc++, etc. So, install gcc first
+
+    if (isUbuntu()) {
+      await setupAptPack("libtinfo-dev")
     }
-    // eslint-disable-next-line require-atomic-updates
-    didInit = true
+    // TODO: install libtinfo on other distros
+    // setupPacmanPack("ncurses")
   }
-  return installationInfo
 }
 
 export async function activateLLVM(directory: string, versionGiven: string) {
@@ -378,7 +116,7 @@ export function setupClangTools(version: string, setupDir: string, arch: string)
   if (ciDetect() === "github-actions") {
     addLLVMLoggingMatcher()
   }
-  return _setupLLVM(version, setupDir, arch)
+  return setupLLVMWithoutActivation(version, setupDir, arch)
 }
 
 function addLLVMLoggingMatcher() {