diff --git a/.changeset/new-steaks-eat.md b/.changeset/new-steaks-eat.md new file mode 100644 index 00000000..5a112263 --- /dev/null +++ b/.changeset/new-steaks-eat.md @@ -0,0 +1,8 @@ +--- +"@nodesecure/tarball": major +"@nodesecure/scanner": minor +"@nodesecure/rc": minor +"@nodesecure/tree-walker": minor +--- + +Implement Node.js worker_threads with a custom Pool to scan packages tarball with JS-X-Ray diff --git a/.gitignore b/.gitignore index 7dc05a3f..e0fb35bd 100644 --- a/.gitignore +++ b/.gitignore @@ -63,7 +63,9 @@ typings/ *.tsbuildinfo nsecure-result.json -temp/ dist/ +temp/ +temp.ts temp.js temp.mjs +.claude diff --git a/workspaces/conformance/package.json b/workspaces/conformance/package.json index 3364f38a..4923bcac 100644 --- a/workspaces/conformance/package.json +++ b/workspaces/conformance/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types", "spdx:refresh": "node ./scripts/fetchSpdxLicenses.js" diff --git a/workspaces/contact/package.json b/workspaces/contact/package.json index 7cb27ecb..3f436c00 100644 --- a/workspaces/contact/package.json +++ b/workspaces/contact/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "npm run build && tsd && attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, diff --git a/workspaces/contact/test/ContactExtractor.spec.ts b/workspaces/contact/test/ContactExtractor.spec.ts index e724a4c3..5f3285b6 100644 --- a/workspaces/contact/test/ContactExtractor.spec.ts +++ b/workspaces/contact/test/ContactExtractor.spec.ts @@ -1,6 +1,6 @@ // Import Node.js Dependencies import assert from "node:assert"; -import { describe, test } from "node:test"; +import { describe, test, mock } from "node:test"; import { join } from "node:path"; import { readFileSync } from "node:fs"; @@ -13,6 +13,7 @@ import { ContactExtractor, type ContactExtractorPackageMetadata } from "../src/index.ts"; +import { NsResolver } from "../src/NsResolver.class.ts"; // CONSTANTS const kManifestFixturePath = join(import.meta.dirname, "fixtures", "manifest"); @@ -110,10 +111,14 @@ describe("ContactExtractor", () => { }); test("Given a Contact with a non-existing email domain, it must be identified as expired", async() => { + const mockedGetExpired = mock.method(NsResolver.prototype, "getExpired"); const extractor = new ContactExtractor({ highlight: [] }); const expiredEmail = "john.doe+test@somenonexistentdomainongoogle9991254874x54x54.com"; + mockedGetExpired.mock.mockImplementation( + () => Promise.resolve([expiredEmail]) + ); const dependencies: Record = { kleur: { @@ -127,6 +132,7 @@ describe("ContactExtractor", () => { const { expired } = await extractor.fromDependencies(dependencies); assert.deepEqual(expired, [expiredEmail]); + mockedGetExpired.mock.restore(); }); }); @@ -185,12 +191,18 @@ describe("ContactExtractor", () => { }); test("Given a manifest with only active emails it shouldn't have any expired email", async() => { + const mockedGetExpired = mock.method( + NsResolver.prototype, + "getExpired", + () => Promise.resolve([]) + ); const extractor = new ContactExtractor({ highlight: [] }); const { expired } = await extractor.fromManifest(kManifest); assert.deepEqual(expired, []); + mockedGetExpired.mock.restore(); }); test("Given a Contact with a non-existing email domain, it must be identified as expired", async() => { @@ -198,9 +210,18 @@ describe("ContactExtractor", () => { highlight: [] }); const expiredEmail = "john.doe+test@somenonexistentdomainongoogle9991254874x54x54.com"; + const mockedGetExpired = mock.method( + NsResolver.prototype, + "getExpired", + () => Promise.resolve([expiredEmail]) + ); - const { expired } = await extractor.fromManifest({ ...kManifest, author: { ...kManifest.author!, email: expiredEmail } }); + const { expired } = await extractor.fromManifest({ + ...kManifest, + author: { ...kManifest.author!, email: expiredEmail } + }); assert.deepEqual(expired, [expiredEmail]); + mockedGetExpired.mock.restore(); }); }); @@ -270,12 +291,19 @@ describe("ContactExtractor", () => { }); test("Given a packument with only active emails it shouldn't have any expired email", async() => { + const mockedGetExpired = mock.method( + NsResolver.prototype, + "getExpired", + () => Promise.resolve([]) + ); + const extractor = new ContactExtractor({ highlight: [] }); const { expired } = await extractor.fromPackument(kPackument); assert.deepEqual(expired, []); + mockedGetExpired.mock.restore(); }); test("Given a Contact with a non-existing email domain, it must be identified as expired", async() => { @@ -283,6 +311,11 @@ describe("ContactExtractor", () => { highlight: [] }); const expiredEmail = "john.doe+test@somenonexistentdomainongoogle9991254874x54x54.com"; + const mockedGetExpired = mock.method( + NsResolver.prototype, + "getExpired", + () => Promise.resolve([expiredEmail]) + ); const versions = Object.entries(kPackument.versions) .reduce((acc: Record, [version, value]) => { return { @@ -294,8 +327,12 @@ describe("ContactExtractor", () => { }; }, {}); - const { expired } = await extractor.fromPackument({ ...kPackument, versions }); + const { expired } = await extractor.fromPackument({ + ...kPackument, + versions + }); assert.deepEqual(expired, [expiredEmail]); + mockedGetExpired.mock.restore(); }); }); }); diff --git a/workspaces/flags/package.json b/workspaces/flags/package.json index 53e574e1..425b050c 100644 --- a/workspaces/flags/package.json +++ b/workspaces/flags/package.json @@ -5,7 +5,7 @@ "scripts": { "build": "tsc -b & cp -R ./src/flags ./dist/flags", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types", "generateFlags": "node scripts/generateFlags.ts" diff --git a/workspaces/fs-walk/package.json b/workspaces/fs-walk/package.json index c7510b17..3766f745 100644 --- a/workspaces/fs-walk/package.json +++ b/workspaces/fs-walk/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, diff --git a/workspaces/github/package.json b/workspaces/github/package.json index 85e41914..f44500c1 100644 --- a/workspaces/github/package.json +++ b/workspaces/github/package.json @@ -8,7 +8,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, diff --git a/workspaces/gitlab/package.json b/workspaces/gitlab/package.json index 57114a37..182a5d3a 100644 --- a/workspaces/gitlab/package.json +++ b/workspaces/gitlab/package.json @@ -8,7 +8,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, diff --git a/workspaces/i18n/package.json b/workspaces/i18n/package.json index 11904b2b..7d44820f 100644 --- a/workspaces/i18n/package.json +++ b/workspaces/i18n/package.json @@ -7,7 +7,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types", "build:documentation": "node ./scripts/buildDocumentation.ts" diff --git a/workspaces/mama/package.json b/workspaces/mama/package.json index 99a48dcc..d0ec1173 100644 --- a/workspaces/mama/package.json +++ b/workspaces/mama/package.json @@ -8,7 +8,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "npm run build && tsd && attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, diff --git a/workspaces/rc/package.json b/workspaces/rc/package.json index 1b61554e..2264a2e1 100644 --- a/workspaces/rc/package.json +++ b/workspaces/rc/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "npm run build && tsd && attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, @@ -45,7 +45,7 @@ "ajv": "8.18.0" }, "dependencies": { - "@nodesecure/js-x-ray": "14.3.0", + "@nodesecure/js-x-ray": "15.0.0", "@nodesecure/npm-types": "^1.2.0", "@nodesecure/vulnera": "3.1.0", "@openally/config": "^1.0.1", diff --git a/workspaces/scanner/package.json b/workspaces/scanner/package.json index 3095a13f..66899c3e 100644 --- a/workspaces/scanner/package.json +++ b/workspaces/scanner/package.json @@ -22,7 +22,7 @@ "lint": "eslint src test", "prepublishOnly": "npm run build && pkg-ok", "test": "c8 -r html npm run test-only && npm run test-types", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only" }, "publishConfig": { @@ -68,7 +68,7 @@ "@nodesecure/contact": "^3.0.0", "@nodesecure/flags": "^3.0.3", "@nodesecure/i18n": "^4.1.0", - "@nodesecure/js-x-ray": "14.3.0", + "@nodesecure/js-x-ray": "15.0.0", "@nodesecure/mama": "^2.2.0", "@nodesecure/npm-registry-sdk": "4.5.2", "@nodesecure/npm-types": "^1.3.0", diff --git a/workspaces/scanner/src/class/TarballScanner.class.ts b/workspaces/scanner/src/class/TarballScanner.class.ts new file mode 100644 index 00000000..e10835e0 --- /dev/null +++ b/workspaces/scanner/src/class/TarballScanner.class.ts @@ -0,0 +1,219 @@ +// Import Third-party Dependencies +import { Mutex } from "@openally/mutex"; +import { + extractAndResolve, + scanDirOrArchive, + NpmTarballWorkerPool, + type PacoteProvider, + type ScanResultPayload +} from "@nodesecure/tarball"; +import { + DefaultCollectableSet, + type CollectableSetData +} from "@nodesecure/js-x-ray"; +import { ManifestManager } from "@nodesecure/mama"; + +// Import Internal Dependencies +import { StatsCollector } from "./StatsCollector.class.ts"; +import { TempDirectory } from "./TempDirectory.class.ts"; +import { Logger, ScannerLoggerEvents } from "./logger.class.ts"; + +type CollectableMetadata = { spec?: string; }; + +export interface ScanContext { + name: string; + version: string; + ref: any; + registry?: string; + location?: string; + isRootNode: boolean; +} + +export interface TarballScannerOptions { + tempDir: TempDirectory; + statsCollector: StatsCollector; + pacoteProvider?: PacoteProvider; + collectables: DefaultCollectableSet[]; + maxConcurrency: number; + logger: Logger; + workers?: boolean | number; +} + +export class TarballScanner { + #locker: Mutex; + #tempDir: TempDirectory; + #statsCollector: StatsCollector; + #pacoteProvider: PacoteProvider | undefined; + #collectables: DefaultCollectableSet[]; + #collectableTypes: string[]; + #workerPool: NpmTarballWorkerPool | null; + #logger: Logger; + + constructor( + options: TarballScannerOptions + ) { + const { + tempDir, + statsCollector, + pacoteProvider, + collectables, + maxConcurrency, + logger, + workers + } = options; + + this.#tempDir = tempDir; + this.#statsCollector = statsCollector; + this.#pacoteProvider = pacoteProvider; + this.#collectables = collectables; + this.#collectableTypes = collectables.map((collectable) => collectable.type); + this.#logger = logger; + + this.#locker = new Mutex({ concurrency: maxConcurrency }); + + this.#workerPool = workers + ? new NpmTarballWorkerPool({ + workerCount: typeof workers === "number" ? workers : undefined + }) + : null; + } + + async scan( + context: ScanContext + ): Promise { + if (this.#workerPool && !context.isRootNode) { + await this.#scanWithWorkers(context); + } + else { + await this.#scanDirect(context); + } + + this.#logger.tick(ScannerLoggerEvents.analysis.tarball); + } + + async #scanWithWorkers( + context: ScanContext + ): Promise { + const { + name, + version, + ref, + registry, + location + } = context; + + const spec = `${name}@${version}`; + const hasLocation = typeof location !== "undefined"; + + const mama = await this.#extract(spec, registry); + + const result = await this.#statsCollector.track( + `tarball.scanDirOrArchive ${spec}`, + "tarball-scan", + () => this.#workerPool!.scan({ + location: mama.location!, + astAnalyserOptions: { + optionalWarnings: hasLocation + }, + collectableTypes: this.#collectableTypes + }) + ); + + this.#applyResult(ref, result); + this.#mergeCollectables(result.collectables); + } + + async #scanDirect( + context: ScanContext + ): Promise { + const { + name, + version, + ref, + registry, + location = process.cwd(), + isRootNode + } = context; + + const spec = `${name}@${version}`; + const hasLocation = typeof context.location !== "undefined"; + + using _ = await this.#locker.acquire(); + + const mama = await (isRootNode ? + ManifestManager.fromPackageJSON(location) : + extractAndResolve(this.#tempDir.location, { + spec, + registry, + pacoteProvider: this.#pacoteProvider + }) + ); + + await this.#statsCollector.track( + `tarball.scanDirOrArchive ${spec}`, + "tarball-scan", + () => scanDirOrArchive(mama, ref, { + astAnalyserOptions: { + optionalWarnings: hasLocation, + collectables: this.#collectables + } + }) + ); + } + + async #extract( + spec: string, + registry?: string + ): Promise { + using _ = await this.#locker.acquire(); + + return extractAndResolve(this.#tempDir.location, { + spec, + registry, + pacoteProvider: this.#pacoteProvider + }); + } + + #applyResult( + ref: any, + result: ScanResultPayload + ): void { + const { description, engines, repository, scripts, author, integrity } = result; + Object.assign(ref, { description, engines, repository, scripts, author, integrity }); + + ref.warnings.push(...result.warnings); + ref.licenses = result.licenses; + ref.uniqueLicenseIds = result.uniqueLicenseIds; + ref.type = result.type; + ref.size = result.size; + ref.composition.extensions.push(...result.composition.extensions); + ref.composition.files.push(...result.composition.files); + ref.composition.minified = result.composition.minified; + ref.composition.unused.push(...result.composition.unused); + ref.composition.missing.push(...result.composition.missing); + ref.composition.required_files = result.composition.required_files; + ref.composition.required_nodejs = result.composition.required_nodejs; + ref.composition.required_thirdparty = result.composition.required_thirdparty; + ref.composition.required_subpath = result.composition.required_subpath; + + const flags = result.flags.filter( + (flag) => flag !== "hasWarnings" || !ref.flags.includes("hasWarnings") + ); + ref.flags.push(...flags); + } + + #mergeCollectables( + serialized: CollectableSetData[] = [] + ): void { + for (const data of serialized) { + const sharedSet = this.#collectables.find( + (collectable) => collectable.type === data.type + ); + sharedSet && DefaultCollectableSet.mergeData(sharedSet, data); + } + } + + async [Symbol.asyncDispose](): Promise { + await this.#workerPool?.terminate(); + } +} diff --git a/workspaces/scanner/src/depWalker.ts b/workspaces/scanner/src/depWalker.ts index a407eb92..3a440b52 100644 --- a/workspaces/scanner/src/depWalker.ts +++ b/workspaces/scanner/src/depWalker.ts @@ -5,17 +5,14 @@ import { readFileSync } from "node:fs"; // Import Third-party Dependencies import pacote from "pacote"; import * as npmRegistrySDK from "@nodesecure/npm-registry-sdk"; -import { Mutex, MutexRelease } from "@openally/mutex"; import { - extractAndResolve, - scanDirOrArchive, type PacoteProvider } from "@nodesecure/tarball"; -import { DefaultCollectableSet, type CollectableSet } from "@nodesecure/js-x-ray"; +import { DefaultCollectableSet } from "@nodesecure/js-x-ray"; import * as Vulnera from "@nodesecure/vulnera"; import { npm } from "@nodesecure/tree-walker"; import { parseAuthor } from "@nodesecure/utils"; -import { ManifestManager, parseNpmSpec } from "@nodesecure/mama"; +import { parseNpmSpec } from "@nodesecure/mama"; import type { ManifestVersion, PackageJSON, WorkspacesPackageJSON } from "@nodesecure/npm-types"; import { getNpmRegistryURL } from "@nodesecure/npm-registry-sdk"; import type Config from "@npmcli/config"; @@ -36,6 +33,7 @@ import { StatsCollector } from "./class/StatsCollector.class.ts"; import { RegistryTokenStore } from "./registry/RegistryTokenStore.ts"; import { TempDirectory } from "./class/TempDirectory.class.ts"; import { Logger, ScannerLoggerEvents } from "./class/logger.class.ts"; +import { TarballScanner } from "./class/TarballScanner.class.ts"; import type { Dependency, DependencyVersion, @@ -125,7 +123,8 @@ export async function depWalker( registry, npmRcConfig, npmRcEntries = {}, - maxConcurrency = 8 + maxConcurrency = 8, + workers } = options; const statsCollector = new StatsCollector({ logger }, { isVerbose }); @@ -210,11 +209,15 @@ export async function depWalker( const fetchedMetadataPackages = new Set(); const operationsQueue: Promise[] = []; - const locker = new Mutex({ concurrency: maxConcurrency }); - locker.on( - MutexRelease, - () => logger.tick(ScannerLoggerEvents.analysis.tarball) - ); + await using tarballScanner = new TarballScanner({ + tempDir, + statsCollector, + pacoteProvider, + collectables, + maxConcurrency, + logger, + workers + }); const rootDepsOptions: npm.WalkOptions = { maxDepth, @@ -302,17 +305,15 @@ export async function depWalker( } } - const scanDirOptions = { - ref: dependency.versions[version] as any, - location, - isRootNode: scanRootNode && name === manifest.name, - registry, - statsCollector, - pacoteProvider, - collectables - }; operationsQueue.push( - scanDirOrArchiveEx(name, version, locker, tempDir, scanDirOptions) + tarballScanner.scan({ + name, + version, + ref: dependency.versions[version] as any, + location, + isRootNode: scanRootNode && name === manifest.name, + registry + }) ); } @@ -425,7 +426,10 @@ export async function depWalker( } } -function extractHighlightedIdentifiers(collectables: DefaultCollectableSet[], identifiersToHighlight: Set) { +function extractHighlightedIdentifiers( + collectables: DefaultCollectableSet[], + identifiersToHighlight: Set +) { if (identifiersToHighlight.size === 0) { return []; } @@ -444,55 +448,6 @@ function extractHighlightedIdentifiers(collectables: DefaultCollectableSet[]; - } -) { - using _ = await locker.acquire(); - - const spec = `${name}@${version}`; - - const { - registry, - location = process.cwd(), - isRootNode, - ref, - statsCollector, - pacoteProvider, - collectables - } = options; - - const mama = await (isRootNode ? - ManifestManager.fromPackageJSON(location) : - extractAndResolve(tempDir.location, { - spec, - registry, - pacoteProvider - }) - ); - - await statsCollector.track(`tarball.scanDirOrArchive ${spec}`, - "tarball-scan", - () => scanDirOrArchive(mama, ref, { - astAnalyserOptions: { - optionalWarnings: typeof location !== "undefined", - collectables - } - })); -} - function isLocalManifest( verDescriptor: DependencyVersion, manifest: PackageJSON | WorkspacesPackageJSON | ManifestVersion, diff --git a/workspaces/scanner/src/types.ts b/workspaces/scanner/src/types.ts index e0b85588..da98425b 100644 --- a/workspaces/scanner/src/types.ts +++ b/workspaces/scanner/src/types.ts @@ -347,6 +347,15 @@ export interface Options { * @default false */ isVerbose?: boolean; + + /** + * Enable worker threads for parallel tarball scanning. + * - `true` uses the default worker count (4) + * - `number` sets an explicit worker count + * + * @default false + */ + readonly workers?: boolean | number; } export interface TokenStore { diff --git a/workspaces/scanner/test/NpmRegistryProvider.spec.ts b/workspaces/scanner/test/NpmRegistryProvider.spec.ts index 1f4ab44b..22c31f9f 100644 --- a/workspaces/scanner/test/NpmRegistryProvider.spec.ts +++ b/workspaces/scanner/test/NpmRegistryProvider.spec.ts @@ -782,8 +782,6 @@ describe("NpmRegistryProvider", () => { }); describe("enrichDependencyConfusionWarnings", async() => { - const message = "The org 'foo' is not claimed on the public registry"; - const privateRegistry = "https://registry.npmjs.org/private"; test("should add a warning when the org is not found on the public npm registry", async(t) => { @@ -797,15 +795,17 @@ describe("NpmRegistryProvider", () => { }); const warnings: DependencyConfusionWarning[] = []; await provider.enrichScopedDependencyConfusionWarnings(warnings, "foo"); - assert.deepEqual(warnings, [{ - type: "dependency-confusion", - message, - metadata: { - name: "@foo/utils" - } - }]); + + assert.strictEqual(warnings.length, 1); + const [warning] = warnings; + assert.strictEqual(warning.type, "dependency-confusion"); + assert.match(warning.message, /.*'foo'.*public.*/); + assert.deepEqual(warning.metadata, { + name: "@foo/utils" + }); assert.strictEqual(mockOrg.mock.callCount(), 1); }); + test("should not add a warning when the error is not a 404", async(t) => { const mockOrg = t.mock.fn(() => { throw new HttpieOnHttpError({ @@ -815,6 +815,7 @@ describe("NpmRegistryProvider", () => { statusCode: 500 }); }); + const provider = new NpmRegistryProvider("@foo/utils", "2.5.9", { npmApiClient: { ...defaultNpmApiClient, @@ -824,13 +825,16 @@ describe("NpmRegistryProvider", () => { }); const warnings: DependencyConfusionWarning[] = []; await provider.enrichScopedDependencyConfusionWarnings(warnings, "foo"); + assert.deepEqual(warnings, []); assert.strictEqual(mockOrg.mock.callCount(), 1); }); + test("should not not add a dependency confusion warning when the org exist on the public registry", async(t) => { const mockOrg = t.mock.fn(async(_) => { return {}; }); + const provider = new NpmRegistryProvider("@foo/utils", "2.5.9", { npmApiClient: { ...defaultNpmApiClient, @@ -840,6 +844,7 @@ describe("NpmRegistryProvider", () => { }); const warnings: DependencyConfusionWarning[] = []; await provider.enrichScopedDependencyConfusionWarnings(warnings, "foo"); + assert.deepEqual(warnings, []); assert.strictEqual(mockOrg.mock.callCount(), 1); assert.deepEqual(mockOrg.mock.calls[0].arguments, ["@foo/utils"]); diff --git a/workspaces/scanner/test/depWalker.spec.ts b/workspaces/scanner/test/depWalker.spec.ts index 915929fe..f1d32e94 100644 --- a/workspaces/scanner/test/depWalker.spec.ts +++ b/workspaces/scanner/test/depWalker.spec.ts @@ -139,7 +139,10 @@ test("execute depWalker on pkg.gitdeps", { skip }, async(test) => { const result = await depWalker( pkgGitdeps, - { ...structuredClone(kDefaultWalkerOptions), isVerbose: true }, + { + ...structuredClone(kDefaultWalkerOptions), + isVerbose: true + }, logger ); const resultAsJSON = JSON.parse(JSON.stringify(result.dependencies, null, 2)); @@ -192,7 +195,8 @@ test("execute depWalker on typo-squatting (with location)", { skip }, async(test pkgTypoSquatting, { ...structuredClone(kDefaultWalkerOptions), - location: "" + location: "", + isVerbose: true }, logger ); @@ -201,9 +205,9 @@ test("execute depWalker on typo-squatting (with location)", { skip }, async(test const warning = result.warnings[0]; assert.equal(warning.type, "typo-squatting"); - assert.strictEqual( + assert.match( result.warnings[0].message, - "Dependency 'mecha' is similar to the following popular packages: fecha, mocha" + /.*'mecha'.*fecha, mocha/ ); const walkErrors = errors(); @@ -229,7 +233,10 @@ test("execute depWalker on typo-squatting (with no location)", { skip }, async(t const result = await depWalker( pkgTypoSquatting, - structuredClone(kDefaultWalkerOptions), + { + ...structuredClone(kDefaultWalkerOptions), + isVerbose: true + }, logger ); @@ -328,7 +335,7 @@ test("fetch payload of pacote on the npm registry", { skip }, async() => { assert.strictEqual(typeof result.rootDependency.integrity, "string"); }); -test("fetch payload of pacote on the gitlab registry", { skip }, async() => { +test.skip("fetch payload of pacote on the gitlab registry", { skip }, async() => { const result = await from("pacote", { registry: "https://gitlab.com/api/v4/packages/npm/", maxDepth: 10, @@ -465,8 +472,10 @@ describe("scanner.cwd()", () => { type PartialIdentifer = Omit & { location: { file: string | null; }; }; -function sortIdentifiers(identifiers: PartialIdentifer[]) { - return identifiers.slice().sort((a, b) => uniqueIdenfier(a).localeCompare(uniqueIdenfier(b))); +function sortIdentifiers( + identifiers: PartialIdentifer[] +) { + return identifiers.toSorted((a, b) => uniqueIdenfier(a).localeCompare(uniqueIdenfier(b))); } function uniqueIdenfier(identifer: PartialIdentifer) { diff --git a/workspaces/scanner/test/fixtures/verify/express-result.json b/workspaces/scanner/test/fixtures/verify/express-result.json index 9e928b42..66faf22d 100644 --- a/workspaces/scanner/test/fixtures/verify/express-result.json +++ b/workspaces/scanner/test/fixtures/verify/express-result.json @@ -1,1160 +1,1160 @@ { - "index.js": { - "./lib/express": { - "unsafe": false, - "inTry": false, - "location": { - "start": { - "line": 11, - "column": 17 - }, - "end": { - "line": 11, - "column": 41 - } - } - } - }, - "lib\\application.js": { - "finalhandler": { + "lib\\express.js": { + "body-parser": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 19 - }, - "end": { - "line": 16, - "column": 42 - } - } + "location": [ + [ + 15, + 17 + ], + [ + 15, + 39 + ] + ] }, - "./router": { + "events": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 13 - }, - "end": { - "line": 17, - "column": 32 - } - } + "location": [ + [ + 16, + 19 + ], + [ + 16, + 36 + ] + ] }, - "methods": { + "merge-descriptors": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 14 - }, - "end": { - "line": 18, - "column": 32 - } - } + "location": [ + [ + 17, + 12 + ], + [ + 17, + 40 + ] + ] }, - "./middleware/init": { + "./application": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 19, - "column": 17 - }, - "end": { - "line": 19, - "column": 45 - } - } + "location": [ + [ + 18, + 12 + ], + [ + 18, + 36 + ] + ] }, - "./middleware/query": { + "./router/route": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 20, - "column": 12 - }, - "end": { - "line": 20, - "column": 41 - } - } + "location": [ + [ + 19, + 12 + ], + [ + 19, + 37 + ] + ] }, - "debug": { + "./router": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 21, - "column": 12 - }, - "end": { - "line": 21, - "column": 28 - } - } + "location": [ + [ + 20, + 13 + ], + [ + 20, + 32 + ] + ] }, - "./view": { + "./request": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 22, - "column": 11 - }, - "end": { - "line": 22, - "column": 28 - } - } + "location": [ + [ + 21, + 10 + ], + [ + 21, + 30 + ] + ] }, - "http": { + "./response": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 23, - "column": 11 - }, - "end": { - "line": 23, - "column": 26 - } - } + "location": [ + [ + 22, + 10 + ], + [ + 22, + 31 + ] + ] }, - "./utils": { + "./middleware/query": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 26, - "column": 19 - }, - "end": { - "line": 26, - "column": 37 - } - } + "location": [ + [ + 79, + 16 + ], + [ + 79, + 45 + ] + ] }, - "depd": { + "serve-static": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 27, - "column": 16 - }, - "end": { - "line": 27, - "column": 31 - } - } - }, - "array-flatten": { + "location": [ + [ + 81, + 17 + ], + [ + 81, + 40 + ] + ] + } + }, + "index.js": { + "./lib/express": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 28, - "column": 14 - }, - "end": { - "line": 28, - "column": 38 - } - } - }, + "location": [ + [ + 11, + 17 + ], + [ + 11, + 41 + ] + ] + } + }, + "lib\\middleware\\query.js": { "utils-merge": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 29, - "column": 12 - }, - "end": { - "line": 29, - "column": 34 - } - } + "location": [ + [ + 15, + 12 + ], + [ + 15, + 34 + ] + ] }, - "path": { + "parseurl": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 30, - "column": 14 - }, - "end": { - "line": 30, - "column": 29 - } - } + "location": [ + [ + 16, + 15 + ], + [ + 16, + 34 + ] + ] }, + "qs": { + "unsafe": false, + "inTry": false, + "location": [ + [ + 17, + 9 + ], + [ + 17, + 22 + ] + ] + } + }, + "lib\\middleware\\init.js": { "setprototypeof": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 31, - "column": 21 - }, - "end": { - "line": 31, - "column": 46 - } - } + "location": [ + [ + 16, + 21 + ], + [ + 16, + 46 + ] + ] } }, - "lib\\express.js": { - "body-parser": { + "lib\\application.js": { + "finalhandler": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 15, - "column": 17 - }, - "end": { - "line": 15, - "column": 39 - } - } + "location": [ + [ + 16, + 19 + ], + [ + 16, + 42 + ] + ] }, - "events": { + "./router": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 19 - }, - "end": { - "line": 16, - "column": 36 - } - } + "location": [ + [ + 17, + 13 + ], + [ + 17, + 32 + ] + ] }, - "merge-descriptors": { + "methods": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 12 - }, - "end": { - "line": 17, - "column": 40 - } - } + "location": [ + [ + 18, + 14 + ], + [ + 18, + 32 + ] + ] }, - "./application": { + "./middleware/init": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 12 - }, - "end": { - "line": 18, - "column": 36 - } - } + "location": [ + [ + 19, + 17 + ], + [ + 19, + 45 + ] + ] }, - "./router/route": { + "./middleware/query": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 19, - "column": 12 - }, - "end": { - "line": 19, - "column": 37 - } - } + "location": [ + [ + 20, + 12 + ], + [ + 20, + 41 + ] + ] }, - "./router": { + "debug": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 20, - "column": 13 - }, - "end": { - "line": 20, - "column": 32 - } - } + "location": [ + [ + 21, + 12 + ], + [ + 21, + 28 + ] + ] }, - "./request": { + "./view": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 21, - "column": 10 - }, - "end": { - "line": 21, - "column": 30 - } - } + "location": [ + [ + 22, + 11 + ], + [ + 22, + 28 + ] + ] }, - "./response": { + "http": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 22, - "column": 10 - }, - "end": { - "line": 22, - "column": 31 - } - } + "location": [ + [ + 23, + 11 + ], + [ + 23, + 26 + ] + ] }, - "./middleware/query": { + "./utils": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 79, - "column": 16 - }, - "end": { - "line": 79, - "column": 45 - } - } + "location": [ + [ + 26, + 19 + ], + [ + 26, + 37 + ] + ] }, - "serve-static": { + "depd": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 81, - "column": 17 - }, - "end": { - "line": 81, - "column": 40 - } - } - } - }, - "lib\\middleware\\init.js": { - "setprototypeof": { + "location": [ + [ + 27, + 16 + ], + [ + 27, + 31 + ] + ] + }, + "array-flatten": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 21 - }, - "end": { - "line": 16, - "column": 46 - } - } - } - }, - "lib\\middleware\\query.js": { + "location": [ + [ + 28, + 14 + ], + [ + 28, + 38 + ] + ] + }, "utils-merge": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 15, - "column": 12 - }, - "end": { - "line": 15, - "column": 34 - } - } + "location": [ + [ + 29, + 12 + ], + [ + 29, + 34 + ] + ] }, - "parseurl": { + "path": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 15 - }, - "end": { - "line": 16, - "column": 34 - } - } + "location": [ + [ + 30, + 14 + ], + [ + 30, + 29 + ] + ] }, - "qs": { + "setprototypeof": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 9 - }, - "end": { - "line": 17, - "column": 22 - } - } + "location": [ + [ + 31, + 21 + ], + [ + 31, + 46 + ] + ] } }, - "lib\\request.js": { - "accepts": { + "lib\\router\\index.js": { + "./route": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 14 - }, - "end": { - "line": 16, - "column": 32 - } - } + "location": [ + [ + 16, + 12 + ], + [ + 16, + 30 + ] + ] }, - "depd": { + "./layer": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 16 - }, - "end": { - "line": 17, - "column": 31 - } - } + "location": [ + [ + 17, + 12 + ], + [ + 17, + 30 + ] + ] }, - "net": { + "methods": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 11 - }, - "end": { - "line": 18, - "column": 25 - } - } + "location": [ + [ + 18, + 14 + ], + [ + 18, + 32 + ] + ] }, - "type-is": { + "utils-merge": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 19, - "column": 13 - }, - "end": { - "line": 19, - "column": 31 - } - } + "location": [ + [ + 19, + 12 + ], + [ + 19, + 34 + ] + ] }, - "http": { + "debug": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 20, - "column": 11 - }, - "end": { - "line": 20, - "column": 26 - } - } + "location": [ + [ + 20, + 12 + ], + [ + 20, + 28 + ] + ] }, - "fresh": { + "depd": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 21, - "column": 12 - }, - "end": { - "line": 21, - "column": 28 - } - } + "location": [ + [ + 21, + 16 + ], + [ + 21, + 31 + ] + ] }, - "range-parser": { + "array-flatten": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 22, - "column": 17 - }, - "end": { - "line": 22, - "column": 40 - } - } + "location": [ + [ + 22, + 14 + ], + [ + 22, + 38 + ] + ] }, "parseurl": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 23, - "column": 12 - }, - "end": { - "line": 23, - "column": 31 - } - } + "location": [ + [ + 23, + 15 + ], + [ + 23, + 34 + ] + ] }, - "proxy-addr": { + "setprototypeof": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 24, - "column": 16 - }, - "end": { - "line": 24, - "column": 37 - } - } + "location": [ + [ + 24, + 21 + ], + [ + 24, + 46 + ] + ] } }, - "lib\\response.js": { - "safe-buffer": { + "lib\\router\\layer.js": { + "path-to-regexp": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 15, - "column": 13 - }, - "end": { - "line": 15, - "column": 35 - } - } + "location": [ + [ + 16, + 17 + ], + [ + 16, + 42 + ] + ] }, - "content-disposition": { + "debug": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 25 - }, - "end": { - "line": 16, - "column": 55 - } - } - }, - "depd": { + "location": [ + [ + 17, + 12 + ], + [ + 17, + 28 + ] + ] + } + }, + "lib\\router\\route.js": { + "debug": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 16 - }, - "end": { - "line": 17, - "column": 31 - } - } + "location": [ + [ + 16, + 12 + ], + [ + 16, + 28 + ] + ] }, - "encodeurl": { + "array-flatten": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 16 - }, - "end": { - "line": 18, - "column": 36 - } - } + "location": [ + [ + 17, + 14 + ], + [ + 17, + 38 + ] + ] }, - "escape-html": { + "./layer": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 19, - "column": 17 - }, - "end": { - "line": 19, - "column": 39 - } - } + "location": [ + [ + 18, + 12 + ], + [ + 18, + 30 + ] + ] }, - "http": { + "methods": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 20, - "column": 11 - }, - "end": { - "line": 20, - "column": 26 - } - } + "location": [ + [ + 19, + 14 + ], + [ + 19, + 32 + ] + ] + } + }, + "lib\\utils.js": { + "safe-buffer": { + "unsafe": false, + "inTry": false, + "location": [ + [ + 15, + 13 + ], + [ + 15, + 35 + ] + ] }, - "./utils": { + "content-disposition": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 29, - "column": 17 - }, - "end": { - "line": 29, - "column": 35 - } - } + "location": [ + [ + 16, + 25 + ], + [ + 16, + 55 + ] + ] }, - "on-finished": { + "content-type": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 22, - "column": 17 - }, - "end": { - "line": 22, - "column": 39 - } - } + "location": [ + [ + 17, + 18 + ], + [ + 17, + 41 + ] + ] }, - "path": { + "depd": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 23, - "column": 11 - }, - "end": { - "line": 23, - "column": 26 - } - } + "location": [ + [ + 18, + 16 + ], + [ + 18, + 31 + ] + ] }, - "statuses": { + "array-flatten": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 24, - "column": 15 - }, - "end": { - "line": 24, - "column": 34 - } - } + "location": [ + [ + 19, + 14 + ], + [ + 19, + 38 + ] + ] }, - "utils-merge": { + "send": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 25, - "column": 12 - }, - "end": { - "line": 25, - "column": 34 - } - } + "location": [ + [ + 20, + 11 + ], + [ + 20, + 26 + ] + ] }, - "cookie-signature": { + "etag": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 26, - "column": 11 - }, - "end": { - "line": 26, - "column": 38 - } - } + "location": [ + [ + 21, + 11 + ], + [ + 21, + 26 + ] + ] }, - "cookie": { + "proxy-addr": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 30, - "column": 13 - }, - "end": { - "line": 30, - "column": 30 - } - } + "location": [ + [ + 22, + 16 + ], + [ + 22, + 37 + ] + ] }, - "send": { + "qs": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 31, - "column": 11 - }, - "end": { - "line": 31, - "column": 26 - } - } + "location": [ + [ + 23, + 9 + ], + [ + 23, + 22 + ] + ] }, - "vary": { + "querystring": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 35, - "column": 11 - }, - "end": { - "line": 35, - "column": 26 - } - } + "location": [ + [ + 24, + 18 + ], + [ + 24, + 40 + ] + ] } }, - "lib\\router\\index.js": { - "./route": { + "lib\\request.js": { + "accepts": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 12 - }, - "end": { - "line": 16, - "column": 30 - } - } + "location": [ + [ + 16, + 14 + ], + [ + 16, + 32 + ] + ] }, - "./layer": { + "depd": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 12 - }, - "end": { - "line": 17, - "column": 30 - } - } + "location": [ + [ + 17, + 16 + ], + [ + 17, + 31 + ] + ] }, - "methods": { + "net": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 14 - }, - "end": { - "line": 18, - "column": 32 - } - } + "location": [ + [ + 18, + 11 + ], + [ + 18, + 25 + ] + ] }, - "utils-merge": { + "type-is": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 19, - "column": 12 - }, - "end": { - "line": 19, - "column": 34 - } - } + "location": [ + [ + 19, + 13 + ], + [ + 19, + 31 + ] + ] }, - "debug": { + "http": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 20, - "column": 12 - }, - "end": { - "line": 20, - "column": 28 - } - } + "location": [ + [ + 20, + 11 + ], + [ + 20, + 26 + ] + ] }, - "depd": { + "fresh": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 21, - "column": 16 - }, - "end": { - "line": 21, - "column": 31 - } - } + "location": [ + [ + 21, + 12 + ], + [ + 21, + 28 + ] + ] }, - "array-flatten": { + "range-parser": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 22, - "column": 14 - }, - "end": { - "line": 22, - "column": 38 - } - } + "location": [ + [ + 22, + 17 + ], + [ + 22, + 40 + ] + ] }, "parseurl": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 23, - "column": 15 - }, - "end": { - "line": 23, - "column": 34 - } - } + "location": [ + [ + 23, + 12 + ], + [ + 23, + 31 + ] + ] }, - "setprototypeof": { + "proxy-addr": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 24, - "column": 21 - }, - "end": { - "line": 24, - "column": 46 - } - } + "location": [ + [ + 24, + 16 + ], + [ + 24, + 37 + ] + ] } }, - "lib\\router\\layer.js": { - "path-to-regexp": { + "lib\\view.js": { + "debug": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 17 - }, - "end": { - "line": 16, - "column": 42 - } - } + "location": [ + [ + 16, + 12 + ], + [ + 16, + 28 + ] + ] }, - "debug": { + "path": { + "unsafe": false, + "inTry": false, + "location": [ + [ + 17, + 11 + ], + [ + 17, + 26 + ] + ] + }, + "fs": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 12 - }, - "end": { - "line": 17, - "column": 28 - } - } + "location": [ + [ + 18, + 9 + ], + [ + 18, + 22 + ] + ] } }, - "lib\\router\\route.js": { - "debug": { + "lib\\response.js": { + "safe-buffer": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 12 - }, - "end": { - "line": 16, - "column": 28 - } - } + "location": [ + [ + 15, + 13 + ], + [ + 15, + 35 + ] + ] }, - "array-flatten": { + "content-disposition": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 14 - }, - "end": { - "line": 17, - "column": 38 - } - } + "location": [ + [ + 16, + 25 + ], + [ + 16, + 55 + ] + ] }, - "./layer": { + "depd": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 12 - }, - "end": { - "line": 18, - "column": 30 - } - } + "location": [ + [ + 17, + 16 + ], + [ + 17, + 31 + ] + ] }, - "methods": { - "unsafe": false, - "inTry": false, - "location": { - "start": { - "line": 19, - "column": 14 - }, - "end": { - "line": 19, - "column": 32 - } - } - } - }, - "lib\\utils.js": { - "safe-buffer": { + "encodeurl": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 15, - "column": 13 - }, - "end": { - "line": 15, - "column": 35 - } - } + "location": [ + [ + 18, + 16 + ], + [ + 18, + 36 + ] + ] }, - "content-disposition": { + "escape-html": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 25 - }, - "end": { - "line": 16, - "column": 55 - } - } + "location": [ + [ + 19, + 17 + ], + [ + 19, + 39 + ] + ] }, - "content-type": { + "http": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 18 - }, - "end": { - "line": 17, - "column": 41 - } - } + "location": [ + [ + 20, + 11 + ], + [ + 20, + 26 + ] + ] }, - "depd": { + "./utils": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 16 - }, - "end": { - "line": 18, - "column": 31 - } - } + "location": [ + [ + 29, + 17 + ], + [ + 29, + 35 + ] + ] }, - "array-flatten": { + "on-finished": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 19, - "column": 14 - }, - "end": { - "line": 19, - "column": 38 - } - } + "location": [ + [ + 22, + 17 + ], + [ + 22, + 39 + ] + ] }, - "send": { + "path": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 20, - "column": 11 - }, - "end": { - "line": 20, - "column": 26 - } - } + "location": [ + [ + 23, + 11 + ], + [ + 23, + 26 + ] + ] }, - "etag": { + "statuses": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 21, - "column": 11 - }, - "end": { - "line": 21, - "column": 26 - } - } + "location": [ + [ + 24, + 15 + ], + [ + 24, + 34 + ] + ] }, - "proxy-addr": { + "utils-merge": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 22, - "column": 16 - }, - "end": { - "line": 22, - "column": 37 - } - } + "location": [ + [ + 25, + 12 + ], + [ + 25, + 34 + ] + ] }, - "qs": { + "cookie-signature": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 23, - "column": 9 - }, - "end": { - "line": 23, - "column": 22 - } - } + "location": [ + [ + 26, + 11 + ], + [ + 26, + 38 + ] + ] }, - "querystring": { - "unsafe": false, - "inTry": false, - "location": { - "start": { - "line": 24, - "column": 18 - }, - "end": { - "line": 24, - "column": 40 - } - } - } - }, - "lib\\view.js": { - "debug": { + "cookie": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 16, - "column": 12 - }, - "end": { - "line": 16, - "column": 28 - } - } + "location": [ + [ + 30, + 13 + ], + [ + 30, + 30 + ] + ] }, - "path": { + "send": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 17, - "column": 11 - }, - "end": { - "line": 17, - "column": 26 - } - } + "location": [ + [ + 31, + 11 + ], + [ + 31, + 26 + ] + ] }, - "fs": { + "vary": { "unsafe": false, "inTry": false, - "location": { - "start": { - "line": 18, - "column": 9 - }, - "end": { - "line": 18, - "column": 22 - } - } + "location": [ + [ + 35, + 11 + ], + [ + 35, + 26 + ] + ] } } } diff --git a/workspaces/tarball/README.md b/workspaces/tarball/README.md index a123ec1f..5ca05f3f 100644 --- a/workspaces/tarball/README.md +++ b/workspaces/tarball/README.md @@ -37,6 +37,7 @@ console.log(scanResult); - [SourceCode](./docs/SourceCode.md) - [NpmTarball](./docs/NpmTarball.md) +- [NpmTarballWorkerPool](./docs/NpmTarballWorkerPool.md) --- diff --git a/workspaces/tarball/docs/NpmTarballWorkerPool.md b/workspaces/tarball/docs/NpmTarballWorkerPool.md new file mode 100644 index 00000000..73d4a7db --- /dev/null +++ b/workspaces/tarball/docs/NpmTarballWorkerPool.md @@ -0,0 +1,94 @@ +# NpmTarballWorkerPool + +`NpmTarballWorkerPool` is a worker-thread-based pool for scanning multiple NPM tarballs concurrently. It maintains a fixed number of worker threads and internally queues tasks when all workers are busy, dispatching them as workers become available. + +The class extends Node.js `EventEmitter`. + +```ts +import { NpmTarballWorkerPool } from "@nodesecure/tarball"; + +await using pool = new NpmTarballWorkerPool({ + workerCount: 4 +}); + +const results = await Promise.all([ + pool.scan({ location: "/path/to/package-a" }), + pool.scan({ location: "/path/to/package-b" }), + pool.scan({ location: "/path/to/package-c" }), +]); +console.log(results); +``` + +## API + +### `new NpmTarballWorkerPool(options?)` + +Creates a new worker pool and spawns the configured number of worker threads immediately. + +```ts +type WorkerFactory = (events: PooledWorkerEvents) => WorkerHandle; + +interface NpmTarballWorkerPoolOptions { + /** + * Number of workers in the pool + * @default 4 + */ + workerCount?: number; + + /** + * Factory used to create each worker in the pool. + * Defaults to creating a real PooledWorker backed by a worker thread. + * Override in tests to inject a mock without patching modules. + */ + workerFactory?: WorkerFactory; +} +``` + +### `scan(task: WorkerTask): Promise` + +Submits a scan task to the pool. If a worker is available it starts immediately; otherwise the task is queued and dispatched as soon as a worker becomes free. + +```ts +interface WorkerTask { + /** + * Location of the package to scan (e.g. tarball path or directory path). + */ + location: string; + /** + * Options for the AST analyser. + * `collectables` is not supported and should not be provided, + * as collectable sets are managed separately via the `collectableTypes` option. + */ + astAnalyserOptions?: Omit; + /** + * Collectable types to gather during scanning (e.g. "url", "hostname"). + * Results are serialized and returned in ScanResultPayload.collectables. + */ + collectableTypes?: Type[]; +} +``` + +The returned `ScanResultPayload` contains the full analysis result for the package: composition (files, extensions, dependencies), licenses, flags, warnings, and more. + +> [!CAUTION] +> Calling `scan()` after `terminate()` will immediately reject the returned promise. + +### `terminate(): Promise` + +Gracefully shuts down the pool. All running worker threads are terminated and any pending (queued) tasks are rejected with a termination error. + +### `[Symbol.asyncDispose](): Promise` + +Called automatically by the `await using` statement. Delegates to `terminate()`. + +## Events + +### `error` + +Emitted when a worker thread encounters an unhandled error. The associated task is rejected and the worker is returned to the available pool. + +```ts +pool.on("error", (error: Error) => { + console.error("Worker error:", error); +}); +``` diff --git a/workspaces/tarball/package.json b/workspaces/tarball/package.json index 1d4f6805..c0643d06 100644 --- a/workspaces/tarball/package.json +++ b/workspaces/tarball/package.json @@ -17,7 +17,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, @@ -47,10 +47,12 @@ "dependencies": { "@nodesecure/conformance": "^1.2.1", "@nodesecure/fs-walk": "^2.0.0", - "@nodesecure/js-x-ray": "14.3.0", + "@nodesecure/js-x-ray": "15.0.0", "@nodesecure/mama": "^2.2.0", "@nodesecure/npm-types": "^1.2.0", "@nodesecure/utils": "^2.3.0", + "@openally/result": "2.0.0", + "hyperid": "3.3.0", "ipaddr.js": "2.3.0", "pacote": "^21.0.0" }, diff --git a/workspaces/tarball/src/class/DependencyCollectableSet.class.ts b/workspaces/tarball/src/class/DependencyCollectableSet.class.ts index a655f5fb..e7a40837 100644 --- a/workspaces/tarball/src/class/DependencyCollectableSet.class.ts +++ b/workspaces/tarball/src/class/DependencyCollectableSet.class.ts @@ -6,7 +6,9 @@ import { ManifestManager, parseNpmSpec } from "@nodesecure/mama"; import { type Dependency, type CollectableSet, - type CollectableInfos + type CollectableSetData, + type CollectableInfos, + type SourceArrayLocation } from "@nodesecure/js-x-ray"; import type { NodeImport } from "@nodesecure/npm-types"; @@ -110,7 +112,7 @@ export class DependencyCollectableSet implements CollectableSet + Record > = Object.create(null); #values: Set = new Set(); #files: Set = new Set(); @@ -157,7 +159,7 @@ export class DependencyCollectableSet implements CollectableSet + { metadata, location }: CollectableInfos ) { if (!metadata) { return; @@ -170,7 +172,8 @@ export class DependencyCollectableSet implements CollectableSet { + return { + type: this.type, + entries: [] + }; + } } diff --git a/workspaces/tarball/src/class/NpmTarballWorkerPool.class.ts b/workspaces/tarball/src/class/NpmTarballWorkerPool.class.ts new file mode 100644 index 00000000..b95788d0 --- /dev/null +++ b/workspaces/tarball/src/class/NpmTarballWorkerPool.class.ts @@ -0,0 +1,253 @@ +// Import Node.js Dependencies +import { EventEmitter } from "node:events"; +import path from "node:path"; + +// Import Third-party Dependencies +import hyperid from "hyperid"; +import type { + AstAnalyserOptions, + Type +} from "@nodesecure/js-x-ray"; + +// Import Internal Dependencies +import { + PooledWorker, + type WorkerHandle, + type PooledWorkerEvents +} from "./PooledWorker.class.ts"; +import type { ScanResultPayload } from "../types.ts"; + +export type WorkerFactory = (events: PooledWorkerEvents) => WorkerHandle; + +export interface NpmTarballWorkerPoolOptions { + /** + * Number of workers in the pool + * @default 4 + */ + workerCount?: number; + + /** + * Factory used to create each worker in the pool. + * Defaults to creating a real PooledWorker backed by a worker thread. + * Override in tests to inject a mock without patching modules. + */ + workerFactory?: WorkerFactory; +} + +export interface WorkerTask { + /** + * Location of the package to scan (e.g. tarball path or directory path). + */ + location: string; + /** + * Options for the AST analyser. + * `collectables` is not supported and should not be provided, + * as collectable sets are managed separately via the `collectableTypes` option. + */ + astAnalyserOptions?: Omit; + /** + * Collectable types to gather during scanning (e.g. "url", "hostname"). + * Results are serialized and returned in ScanResultPayload.collectables. + */ + collectableTypes?: Type[]; +} + +export interface WorkerTaskWithId extends WorkerTask { + id: string; +} + +type WorkerTaskResultOk = { + id: string; + result: ScanResultPayload; +}; + +type WorkerTaskResultErr = { + id: string; + error: string; +}; + +export type WorkerTaskResult = + | WorkerTaskResultOk + | WorkerTaskResultErr; + +interface TaskPromiseHandler { + resolve: (result: ScanResultPayload) => void; + reject: (error: Error) => void; +} + +/** + * O(1) amortized FIFO queue using a head-pointer to avoid + * the O(n) cost of Array.shift(). + */ +class TaskQueue { + #items: WorkerTaskWithId[] = []; + #head = 0; + + enqueue( + task: WorkerTaskWithId + ): void { + this.#items.push(task); + } + + dequeue(): WorkerTaskWithId | undefined { + if (this.#head >= this.#items.length) { + return undefined; + } + const item = this.#items[this.#head++]; + if (this.#head > 0 && this.#head >= this.#items.length / 2) { + this.#items = this.#items.slice(this.#head); + this.#head = 0; + } + + return item; + } + + clear(): void { + this.#items = []; + this.#head = 0; + } +} + +export class NpmTarballWorkerPool extends EventEmitter { + #generateTaskId = hyperid(); + #workers: WorkerHandle[] = []; + #availableWorkers: WorkerHandle[] = []; + #processingTasks: Map = new Map(); + #waitingTasks = new TaskQueue(); + #isTerminated = false; + + constructor( + options: NpmTarballWorkerPoolOptions = {} + ) { + super(); + + const { workerCount = 4, workerFactory } = options; + const workerPath = path.join( + import.meta.dirname, + "NpmTarballWorkerScript.js" + ); + const factory: WorkerFactory = workerFactory ?? + ((events) => new PooledWorker(workerPath, events)); + + for (let i = 0; i < workerCount; i++) { + const worker = factory({ + onComplete: ( + worker, + message + ) => this.#onWorkerComplete(worker, message), + onError: ( + worker, + error, + taskId + ) => this.#onWorkerError(worker, error, taskId) + }); + + this.#workers.push(worker); + this.#availableWorkers.push(worker); + } + } + + #onWorkerComplete( + worker: WorkerHandle, + message: WorkerTaskResult + ): void { + const handler = this.#processingTasks.get(message.id); + if (handler) { + this.#processingTasks.delete(message.id); + + if ("error" in message) { + handler.reject(new Error(message.error)); + } + else { + handler.resolve(message.result); + } + } + + const nextTask = this.#waitingTasks.dequeue(); + if (nextTask) { + worker.execute(nextTask); + } + else { + this.#availableWorkers.push(worker); + } + } + + #onWorkerError( + worker: WorkerHandle, + error: Error, + taskId: string | null + ): void { + if (taskId) { + const handler = this.#processingTasks.get(taskId); + if (handler) { + this.#processingTasks.delete(taskId); + handler.reject(error); + } + } + + this.emit("error", error); + const nextTask = this.#waitingTasks.dequeue(); + if (nextTask) { + worker.execute(nextTask); + } + else { + this.#availableWorkers.push(worker); + } + } + + scan( + task: WorkerTask + ): Promise { + if (this.#isTerminated) { + return Promise.reject( + new Error("NpmTarballWorkerPool has been terminated") + ); + } + + const fullTask: WorkerTaskWithId = { + id: this.#generateTaskId(), + ...task + }; + + const { + promise, + resolve, + reject + } = Promise.withResolvers(); + this.#processingTasks.set( + fullTask.id, + { resolve, reject } + ); + + const availableWorker = this.#availableWorkers.pop() ?? null; + if (availableWorker) { + availableWorker.execute(fullTask); + } + else { + this.#waitingTasks.enqueue(fullTask); + } + + return promise; + } + + async terminate(): Promise { + this.#isTerminated = true; + + const terminationError = new Error("NpmTarballWorkerPool terminated"); + for (const handler of this.#processingTasks.values()) { + handler.reject(terminationError); + } + this.#processingTasks.clear(); + this.#waitingTasks.clear(); + this.#availableWorkers = []; + + await Promise.all( + this.#workers.map((worker) => worker.terminate()) + ); + this.#workers = []; + } + + [Symbol.asyncDispose](): Promise { + return this.terminate(); + } +} diff --git a/workspaces/tarball/src/class/NpmTarballWorkerScript.ts b/workspaces/tarball/src/class/NpmTarballWorkerScript.ts new file mode 100644 index 00000000..7faf3486 --- /dev/null +++ b/workspaces/tarball/src/class/NpmTarballWorkerScript.ts @@ -0,0 +1,61 @@ +// Import Node.js Dependencies +import { parentPort } from "node:worker_threads"; + +// Import Third-party Dependencies +import { + DefaultCollectableSet +} from "@nodesecure/js-x-ray"; + +// Import Internal Dependencies +import { scanPackageCore } from "../tarball.ts"; + +import type { + WorkerTaskWithId, + WorkerTaskResult +} from "./NpmTarballWorkerPool.class.ts"; + +if (!parentPort) { + throw new Error("This script must be run as a worker thread."); +} + +parentPort.on("message", onWorkerMessage); + +async function onWorkerMessage( + task: WorkerTaskWithId +) { + let message: WorkerTaskResult; + + try { + const collectables = (task.collectableTypes ?? []).map( + (type) => new DefaultCollectableSet(type) + ); + + const result = await scanPackageCore( + task.location, + { + ...task.astAnalyserOptions, + collectables + } + ); + + message = { + id: task.id, + result: { + ...result, + collectables: collectables.map((set) => set.toJSON()) + } + }; + } + catch (error) { + const messageError = error instanceof Error ? + error.message : + String(error); + + message = { + id: task.id, + error: messageError + }; + } + + parentPort?.postMessage(message); +} diff --git a/workspaces/tarball/src/class/PooledWorker.class.ts b/workspaces/tarball/src/class/PooledWorker.class.ts new file mode 100644 index 00000000..2274399f --- /dev/null +++ b/workspaces/tarball/src/class/PooledWorker.class.ts @@ -0,0 +1,72 @@ +// Import Node.js Dependencies +import { Worker } from "node:worker_threads"; + +// Import Internal Dependencies +import type { + WorkerTaskWithId, + WorkerTaskResult +} from "./NpmTarballWorkerPool.class.ts"; + +export interface WorkerHandle { + isAvailable: boolean; + execute( + task: WorkerTaskWithId + ): void; + terminate(): Promise; +} + +export interface PooledWorkerEvents { + onComplete: ( + worker: WorkerHandle, + result: WorkerTaskResult + ) => void; + onError: ( + worker: WorkerHandle, + error: Error, + taskId: string | null + ) => void; +} + +export class PooledWorker implements WorkerHandle { + #worker: Worker; + #currentTaskId: string | null = null; + #events: PooledWorkerEvents; + + constructor( + workerPath: string, + events: PooledWorkerEvents + ) { + this.#events = events; + this.#worker = new Worker(workerPath); + + this.#worker.on("message", (message: WorkerTaskResult) => { + this.#currentTaskId = null; + this.#events.onComplete(this, message); + }); + + this.#worker.on("error", (error: Error) => { + const taskId = this.#currentTaskId; + this.#currentTaskId = null; + this.#events.onError(this, error, taskId); + }); + } + + get isAvailable(): boolean { + return this.#currentTaskId === null; + } + + execute( + task: WorkerTaskWithId + ): void { + if (!this.isAvailable) { + throw new Error(`Worker is busy with task ${this.#currentTaskId}`); + } + + this.#currentTaskId = task.id; + this.#worker.postMessage(task); + } + + terminate(): Promise { + return this.#worker.terminate(); + } +} diff --git a/workspaces/tarball/src/constants.ts b/workspaces/tarball/src/constants.ts new file mode 100644 index 00000000..bc049fc7 --- /dev/null +++ b/workspaces/tarball/src/constants.ts @@ -0,0 +1,5 @@ +export const NATIVE_CODE_EXTENSIONS = new Set([".gyp", ".c", ".cpp", ".node", ".so", ".h"]); + +export const NPM_TOKEN = typeof process.env.NODE_SECURE_TOKEN === "string" ? + { token: process.env.NODE_SECURE_TOKEN } : + {}; diff --git a/workspaces/tarball/src/index.ts b/workspaces/tarball/src/index.ts index fa3f53f8..c0e45d7e 100644 --- a/workspaces/tarball/src/index.ts +++ b/workspaces/tarball/src/index.ts @@ -1,4 +1,9 @@ export * from "./tarball.ts"; export * from "./class/NpmTarball.class.ts"; export * from "./class/DependencyCollectableSet.class.ts"; - +export { + NpmTarballWorkerPool, + type WorkerTask, + type NpmTarballWorkerPoolOptions +} from "./class/NpmTarballWorkerPool.class.ts"; +export * from "./types.ts"; diff --git a/workspaces/tarball/src/tarball.ts b/workspaces/tarball/src/tarball.ts index df2f0991..2038a569 100644 --- a/workspaces/tarball/src/tarball.ts +++ b/workspaces/tarball/src/tarball.ts @@ -9,10 +9,7 @@ import { type AstAnalyserOptions } from "@nodesecure/js-x-ray"; import * as conformance from "@nodesecure/conformance"; -import { - ManifestManager, - type PackageModuleType -} from "@nodesecure/mama"; +import { ManifestManager, type PackageModuleType } from "@nodesecure/mama"; import pacote from "pacote"; // Import Internal Dependencies @@ -26,124 +23,116 @@ import { getEmptyPackageWarning, getSemVerWarning } from "./warnings.ts"; - -export interface DependencyRef { - id: number; - type: PackageModuleType; - usedBy: Record; - isDevDependency: boolean; - existOnRemoteRegistry: boolean; - flags: string[]; - description: string; - size: number; - author: Record; - engines: Record; - repository: any; - scripts: Record; - warnings: any; - licenses: conformance.SpdxFileLicenseConformance[]; - uniqueLicenseIds: string[]; - gitUrl: string | null; - alias: Record; - composition: { - extensions: string[]; - files: string[]; - minified: string[]; - unused: string[]; - missing: string[]; - required_files: string[]; - required_nodejs: string[]; - required_thirdparty: string[]; - required_subpath: Record; - }; -} - -// CONSTANTS -const kNativeCodeExtensions = new Set([".gyp", ".c", ".cpp", ".node", ".so", ".h"]); -const kNpmToken = typeof process.env.NODE_SECURE_TOKEN === "string" ? - { token: process.env.NODE_SECURE_TOKEN } : - {}; +import { + NATIVE_CODE_EXTENSIONS, + NPM_TOKEN +} from "./constants.ts"; +import type { + ScanResultPayload, + DependencyRef +} from "./types.ts"; export interface ScanOptions { astAnalyserOptions?: AstAnalyserOptions; } -export async function scanDirOrArchive( +export async function scanPackageCore( locationOrManifest: string | ManifestManager, - ref: DependencyRef, - options: ScanOptions = {} -): Promise { - const { astAnalyserOptions } = options; - - const mama = await ManifestManager.fromPackageJSON( - locationOrManifest - ); - const tarex = new NpmTarball(mama); - + astAnalyserOptions?: AstAnalyserOptions +): Promise { + const mama = await ManifestManager.fromPackageJSON(locationOrManifest); const dependencySet = new DependencyCollectableSet(mama); + const tarex = new NpmTarball(mama); const { composition, - conformance, + conformance: conformanceResult, code } = await tarex.scanFiles({ ...astAnalyserOptions, - collectables: [...astAnalyserOptions?.collectables ?? [], dependencySet] + collectables: [ + ...astAnalyserOptions?.collectables ?? [], + dependencySet + ] }); - { - const { description, engines, repository, scripts } = mama.document; - Object.assign(ref, { - description, engines, repository, scripts, - author: mama.author, - integrity: mama.isWorkspace ? null : mama.integrity - }); - } - - if ( - composition.files.length === 1 && - composition.files.includes("package.json") - ) { - ref.warnings.push(getEmptyPackageWarning()); + const warnings: Warning[] = []; + if (composition.files.length === 1 && composition.files.includes("package.json")) { + warnings.push(getEmptyPackageWarning()); } - if (mama.hasZeroSemver) { - ref.warnings.push(getSemVerWarning(mama.document.version!)); + warnings.push(getSemVerWarning(mama.document.version!)); } - ref.warnings.push(...code.warnings); + warnings.push(...code.warnings); - const { - files, - dependencies, - flags - } = dependencySet.extract(); - - ref.licenses = conformance.licenses; - ref.uniqueLicenseIds = conformance.uniqueLicenseIds; - ref.type = mama.moduleType; - ref.size = composition.size; - ref.composition.extensions.push(...composition.ext); - ref.composition.files.push(...composition.files); - ref.composition.required_thirdparty = dependencies.thirdparty; - ref.composition.required_subpath = dependencies.subpathImports; - ref.composition.unused.push(...dependencies.unused); - ref.composition.missing.push(...dependencies.missing); - ref.composition.required_files = [...files]; - ref.composition.required_nodejs = dependencies.nodeJs; - ref.composition.minified = code.minified; - - ref.flags.push(...booleanToFlags({ - ...flags, - hasExternalCapacity: code.flags.hasExternalCapacity || flags.hasExternalCapacity, - hasNoLicense: conformance.uniqueLicenseIds.length === 0, - hasMultipleLicenses: conformance.uniqueLicenseIds.length > 1, - hasMinifiedCode: code.minified.length > 0, - hasWarnings: ref.warnings.length > 0 && !ref.flags.includes("hasWarnings"), - hasBannedFile: composition.files.some((path) => isSensitiveFile(path)), - hasNativeCode: mama.flags.isNative || - composition.files.some((file) => kNativeCodeExtensions.has(path.extname(file))), - hasScript: mama.flags.hasUnsafeScripts - })); + const { files, dependencies, flags } = dependencySet.extract(); + const { description, engines, repository, scripts } = mama.document; + + return { + description, + engines, + repository, + scripts, + author: mama.author, + integrity: mama.isWorkspace ? null : mama.integrity, + type: mama.moduleType, + size: composition.size, + licenses: conformanceResult.licenses, + uniqueLicenseIds: conformanceResult.uniqueLicenseIds, + warnings, + flags: Array.from(booleanToFlags({ + ...flags, + hasExternalCapacity: code.flags.hasExternalCapacity || flags.hasExternalCapacity, + hasNoLicense: conformanceResult.uniqueLicenseIds.length === 0, + hasMultipleLicenses: conformanceResult.uniqueLicenseIds.length > 1, + hasMinifiedCode: code.minified.length > 0, + hasWarnings: warnings.length > 0, + hasBannedFile: composition.files.some((filePath) => isSensitiveFile(filePath)), + hasNativeCode: mama.flags.isNative || + composition.files.some((file) => NATIVE_CODE_EXTENSIONS.has(path.extname(file))), + hasScript: mama.flags.hasUnsafeScripts + })), + composition: { + extensions: [...composition.ext], + files: composition.files, + minified: code.minified, + unused: dependencies.unused, + missing: dependencies.missing, + required_files: [...files], + required_nodejs: dependencies.nodeJs, + required_thirdparty: dependencies.thirdparty, + required_subpath: dependencies.subpathImports + } + }; +} + +export async function scanDirOrArchive( + locationOrManifest: string | ManifestManager, + ref: DependencyRef, + options: ScanOptions = {} +): Promise { + const result = await scanPackageCore(locationOrManifest, options.astAnalyserOptions); + + const { description, engines, repository, scripts, author, integrity } = result; + Object.assign(ref, { description, engines, repository, scripts, author, integrity }); + + ref.warnings.push(...result.warnings); + ref.licenses = result.licenses; + ref.uniqueLicenseIds = result.uniqueLicenseIds; + ref.type = result.type as PackageModuleType; + ref.size = result.size; + ref.composition.extensions.push(...result.composition.extensions); + ref.composition.files.push(...result.composition.files); + ref.composition.minified = result.composition.minified; + ref.composition.unused.push(...result.composition.unused); + ref.composition.missing.push(...result.composition.missing); + ref.composition.required_files = result.composition.required_files; + ref.composition.required_nodejs = result.composition.required_nodejs; + ref.composition.required_thirdparty = result.composition.required_thirdparty; + ref.composition.required_subpath = result.composition.required_subpath; + + const flags = result.flags.filter((flag) => flag !== "hasWarnings" || !ref.flags.includes("hasWarnings")); + ref.flags.push(...flags); } export interface ScannedPackageResult { @@ -173,23 +162,22 @@ export async function scanPackage( ): Promise { const { astAnalyserOptions } = options; - const mama = await ManifestManager.fromPackageJSON( - manifestOrLocation - ); + const mama = await ManifestManager.fromPackageJSON(manifestOrLocation); const extractor = new NpmTarball(mama); - const dependencySet = new DependencyCollectableSet(mama); const { composition, - conformance, + conformance: conformanceResult, code } = await extractor.scanFiles({ ...astAnalyserOptions, - collectables: [...astAnalyserOptions?.collectables ?? [], dependencySet] + collectables: [ + ...(astAnalyserOptions?.collectables ?? []), + dependencySet + ] }); - // Check for empty package const warnings = [...code.warnings]; if (composition.files.length === 1 && composition.files.includes("package.json")) { warnings.push(getEmptyPackageWarning()); @@ -202,8 +190,8 @@ export async function scanPackage( minified: code.minified }, directorySize: composition.size, - uniqueLicenseIds: conformance.uniqueLicenseIds, - licenses: conformance.licenses, + uniqueLicenseIds: conformanceResult.uniqueLicenseIds, + licenses: conformanceResult.licenses, ast: { dependencies: dependencySet.dependencies, warnings @@ -236,13 +224,11 @@ export async function extractAndResolve( spec, tarballLocation, { - ...kNpmToken, + ...NPM_TOKEN, registry, cache: `${os.homedir()}/.npm` } ); - return ManifestManager.fromPackageJSON( - tarballLocation - ); + return ManifestManager.fromPackageJSON(tarballLocation); } diff --git a/workspaces/tarball/src/types.ts b/workspaces/tarball/src/types.ts new file mode 100644 index 00000000..d003bb66 --- /dev/null +++ b/workspaces/tarball/src/types.ts @@ -0,0 +1,62 @@ +// Import Third-party Dependencies +import type * as conformance from "@nodesecure/conformance"; +import type { + CollectableSetData +} from "@nodesecure/js-x-ray"; +import type { + PackageModuleType +} from "@nodesecure/mama"; + +export interface Composition { + extensions: string[]; + files: string[]; + minified: string[]; + unused: string[]; + missing: string[]; + required_files: string[]; + required_nodejs: string[]; + required_thirdparty: string[]; + required_subpath: Record; +} + +export interface ScanResultPayload { + description?: string; + engines?: Record; + repository?: any; + scripts?: Record; + author?: any; + integrity?: string | null; + type: string; + size: number; + licenses: conformance.SpdxFileLicenseConformance[]; + uniqueLicenseIds: string[]; + warnings: any[]; + flags: string[]; + composition: Composition; + /** + * Serialized collectable entries populated by the worker thread. + * Only present when `collectableTypes` was specified in the WorkerTask. + */ + collectables?: CollectableSetData[]; +} + +export interface DependencyRef { + id: number; + type: PackageModuleType; + usedBy: Record; + isDevDependency: boolean; + existOnRemoteRegistry: boolean; + flags: string[]; + description: string; + size: number; + author: Record; + engines: Record; + repository: any; + scripts: Record; + warnings: any; + licenses: conformance.SpdxFileLicenseConformance[]; + uniqueLicenseIds: string[]; + gitUrl: string | null; + alias: Record; + composition: Composition; +} diff --git a/workspaces/tarball/test/DependencyCollectableSet.spec.ts b/workspaces/tarball/test/DependencyCollectableSet.spec.ts index 14a0e58f..5bd7d1e3 100644 --- a/workspaces/tarball/test/DependencyCollectableSet.spec.ts +++ b/workspaces/tarball/test/DependencyCollectableSet.spec.ts @@ -33,7 +33,9 @@ describe("DependencyCollectableSet", () => { const dependencyCollectableSet = new DependencyCollectableSet(mama); dependencyCollectableSet.add("fs", { - file: ".", location: [[0, 0], [0, 0]], metadata: { + file: ".", + location: [[0, 0], [0, 0]], + metadata: { unsafe: false, inTry: false, relativeFile: "file1.js" @@ -41,7 +43,9 @@ describe("DependencyCollectableSet", () => { }); dependencyCollectableSet.add("http", { - file: ".", location: [[0, 0], [0, 0]], metadata: { + file: ".", + location: [[0, 0], [0, 0]], + metadata: { unsafe: false, inTry: true, relativeFile: "file2.js" @@ -49,7 +53,9 @@ describe("DependencyCollectableSet", () => { }); dependencyCollectableSet.add("lodash.isequal", { - file: ".", location: [[0, 0], [0, 0]], metadata: { + file: ".", + location: [[0, 0], [0, 0]], + metadata: { unsafe: false, inTry: false, @@ -61,17 +67,20 @@ describe("DependencyCollectableSet", () => { "file1.js": { fs: { unsafe: false, - inTry: false + inTry: false, + location: [[0, 0], [0, 0]] } }, "file2.js": { http: { unsafe: false, - inTry: true + inTry: true, + location: [[0, 0], [0, 0]] }, "lodash.isequal": { unsafe: false, - inTry: false + inTry: false, + location: [[0, 0], [0, 0]] } } }); diff --git a/workspaces/tarball/test/NpmTarballWorkerPool.spec.ts b/workspaces/tarball/test/NpmTarballWorkerPool.spec.ts new file mode 100644 index 00000000..b01d5748 --- /dev/null +++ b/workspaces/tarball/test/NpmTarballWorkerPool.spec.ts @@ -0,0 +1,271 @@ +// Import Node.js Dependencies +import { describe, test, beforeEach } from "node:test"; +import assert from "node:assert"; + +// Import Internal Dependencies +import { + NpmTarballWorkerPool +} from "../src/class/NpmTarballWorkerPool.class.ts"; +import type { + ScanResultPayload +} from "../src/types.ts"; +import { + MockPooledWorker, + capturedWorkers, + terminateCallCount, + resetMockState +} from "./mocks/MockPooledWorker.ts"; +import type { + PooledWorkerEvents +} from "../src/class/PooledWorker.class.ts"; + +// CONSTANTS +const kFakeScanResult: ScanResultPayload = { + type: "module", + size: 1024, + licenses: [], + uniqueLicenseIds: ["MIT"], + warnings: [], + flags: [], + composition: { + extensions: [".js"], + files: ["index.js"], + minified: [], + unused: [], + missing: [], + required_files: [], + required_nodejs: [], + required_thirdparty: [], + required_subpath: {} + } +}; + +function kWorkerFactory( + events: PooledWorkerEvents +): MockPooledWorker { + return new MockPooledWorker(events); +} + +describe("NpmTarballWorkerPool", () => { + beforeEach(() => resetMockState()); + + describe("construction", () => { + test("should create 4 workers by default", async() => { + const pool = new NpmTarballWorkerPool({ + workerFactory: kWorkerFactory + }); + + assert.strictEqual(capturedWorkers.length, 4); + + await pool.terminate(); + }); + + test("should create the specified number of workers", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 2, + workerFactory: kWorkerFactory + }); + + assert.strictEqual(capturedWorkers.length, 2); + + await pool.terminate(); + }); + }); + + describe("scan()", () => { + test("should resolve with the scan result when the worker completes successfully", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + const [worker] = capturedWorkers; + + const scanPromise = pool.scan({ location: "/fake/path" }); + worker.simulateSuccess(kFakeScanResult); + + const result = await scanPromise; + assert.deepEqual(result, kFakeScanResult); + + await pool.terminate(); + }); + + test("should reject when the worker returns an error result", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + const [worker] = capturedWorkers; + + const scanPromise = pool.scan({ location: "/fake/path" }); + worker.simulateErrorResult("scan failed: invalid package"); + + await assert.rejects( + scanPromise, + { message: "scan failed: invalid package" } + ); + + await pool.terminate(); + }); + + test("should reject immediately if the pool has already been terminated", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + await pool.terminate(); + + await assert.rejects( + pool.scan({ location: "/fake/path" }), + { message: "NpmTarballWorkerPool has been terminated" } + ); + }); + + test("should queue tasks when all workers are busy and dispatch them once a worker is free", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + const [worker] = capturedWorkers; + + const scan1 = pool.scan({ location: "/fake/path/1" }); + const scan2 = pool.scan({ location: "/fake/path/2" }); + + // The only worker is occupied by scan1 - scan2 must be queued. + assert.ok( + !worker.isAvailable, + "Worker should be busy with the first task" + ); + + worker.simulateSuccess(kFakeScanResult); + assert.deepEqual(await scan1, kFakeScanResult); + + // After scan1 completes the worker should have immediately picked up scan2. + assert.ok( + !worker.isAvailable, + "Worker should now be processing the queued task" + ); + + worker.simulateSuccess(kFakeScanResult); + assert.deepEqual(await scan2, kFakeScanResult); + + await pool.terminate(); + }); + }); + + describe("terminate()", () => { + test("should reject all in-flight tasks with a termination error", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + // Start a scan but never drive the worker to completion. + const scanPromise = pool.scan({ location: "/fake/path" }); + + await pool.terminate(); + + await assert.rejects( + scanPromise, + { message: "NpmTarballWorkerPool terminated" } + ); + }); + + test("should cause subsequent scan() calls to reject", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + await pool.terminate(); + + await assert.rejects( + pool.scan({ location: "/fake/path" }), + { message: "NpmTarballWorkerPool has been terminated" } + ); + }); + + test("should call terminate() on every worker", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 3, + workerFactory: kWorkerFactory + }); + await pool.terminate(); + + assert.strictEqual(terminateCallCount, 3); + }); + }); + + describe("[Symbol.asyncDispose]", () => { + test("should terminate the pool, causing subsequent scan() calls to reject", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + await pool[Symbol.asyncDispose](); + + await assert.rejects( + pool.scan({ location: "/fake/path" }), + { message: "NpmTarballWorkerPool has been terminated" } + ); + }); + }); + + describe("worker errors", () => { + test("should emit an 'error' event on the pool when a worker crashes", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + const [worker] = capturedWorkers; + + const crashError = new Error("unexpected worker crash"); + const { + promise: errorEventPromise, resolve + } = Promise.withResolvers(); + pool.on("error", (error) => resolve(error)); + + const scanPromise = pool.scan({ location: "/fake/path" }); + worker.simulateCrash(crashError); + + assert.strictEqual( + await errorEventPromise, + crashError, + "Pool should emit the original error object" + ); + await assert.rejects( + scanPromise, + { message: "unexpected worker crash" } + ); + + await pool.terminate(); + }); + + test("should continue processing queued tasks after a worker crash", async() => { + const pool = new NpmTarballWorkerPool({ + workerCount: 1, + workerFactory: kWorkerFactory + }); + const [worker] = capturedWorkers; + + // Suppress unhandled-error event so the test does not throw. + pool.on("error", () => { + // No-op + }); + + const scan1 = pool.scan({ location: "/fake/path/1" }); + const scan2 = pool.scan({ location: "/fake/path/2" }); + + worker.simulateCrash(new Error("crash")); + await assert.rejects(scan1, { message: "crash" }); + + // The pool should have immediately dispatched scan2 to the recovered worker. + assert.ok( + !worker.isAvailable, + "Worker should be processing the queued task after crash" + ); + + worker.simulateSuccess(kFakeScanResult); + assert.deepEqual(await scan2, kFakeScanResult); + + await pool.terminate(); + }); + }); +}); diff --git a/workspaces/tarball/test/SourceCodeScanner.spec.ts b/workspaces/tarball/test/SourceCodeScanner.spec.ts index a92a3412..c589d2a3 100644 --- a/workspaces/tarball/test/SourceCodeScanner.spec.ts +++ b/workspaces/tarball/test/SourceCodeScanner.spec.ts @@ -68,8 +68,8 @@ describe("SourceCodeScanner", () => { assert.deepEqual( files, [ - "src\\index.js", - "src\\foo.js" + path.join("src", "index.js"), + path.join("src", "foo.js") ].sort() ); }); @@ -99,7 +99,7 @@ describe("SourceCodeScanner", () => { files, [ "index.js", - "src\\deps.js" + path.join("src", "deps.js") ].sort() ); }); @@ -187,7 +187,10 @@ describe("SourceCodeScanner", () => { if (firstReport.ok) { const { files, dependencies } = depsSet.extract(); assert.ok(dependencies.nodeJs.includes("node:http")); - assert.ok(files.has("src\\bar.ts")); + + const normalizedFiles = Array.from(files) + .map((file) => path.normalize(file)); + assert.ok(normalizedFiles.includes(path.join("src", "bar.ts"))); } else { assert.fail("First report should be ok"); @@ -200,8 +203,8 @@ describe("SourceCodeScanner", () => { assert.deepEqual( files, [ - "src\\index.ts", - "src\\bar.ts" + path.join("src", "index.ts"), + path.join("src", "bar.ts") ].sort() ); }); diff --git a/workspaces/tarball/test/mocks/MockPooledWorker.ts b/workspaces/tarball/test/mocks/MockPooledWorker.ts new file mode 100644 index 00000000..a9cf4c5d --- /dev/null +++ b/workspaces/tarball/test/mocks/MockPooledWorker.ts @@ -0,0 +1,71 @@ +// Import Internal Dependencies +import type { + WorkerHandle, + PooledWorkerEvents +} from "../../src/class/PooledWorker.class.ts"; +import type { + ScanResultPayload +} from "../../src/types.ts"; + +export const capturedWorkers: MockPooledWorker[] = []; +export let terminateCallCount = 0; + +export function resetMockState(): void { + capturedWorkers.length = 0; + terminateCallCount = 0; +} + +export class MockPooledWorker implements WorkerHandle { + currentTaskId: string | null = null; + #events: PooledWorkerEvents; + + constructor( + events: PooledWorkerEvents + ) { + this.#events = events; + capturedWorkers.push(this); + } + + get isAvailable(): boolean { + return this.currentTaskId === null; + } + + execute( + task: { id: string; } + ): void { + if (!this.isAvailable) { + throw new Error(`Worker is busy with task ${this.currentTaskId}`); + } + this.currentTaskId = task.id; + } + + simulateSuccess( + result: ScanResultPayload + ): void { + const id = this.currentTaskId!; + this.currentTaskId = null; + this.#events.onComplete(this, { id, result }); + } + + simulateErrorResult( + errorMsg: string + ): void { + const id = this.currentTaskId!; + this.currentTaskId = null; + this.#events.onComplete(this, { id, error: errorMsg }); + } + + simulateCrash( + error: Error + ): void { + const taskId = this.currentTaskId; + this.currentTaskId = null; + this.#events.onError(this, error, taskId); + } + + terminate(): Promise { + terminateCallCount++; + + return Promise.resolve(0); + } +} diff --git a/workspaces/tree-walker/package.json b/workspaces/tree-walker/package.json index b3b2c34d..83369b74 100644 --- a/workspaces/tree-walker/package.json +++ b/workspaces/tree-walker/package.json @@ -8,7 +8,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" }, @@ -37,7 +37,7 @@ }, "homepage": "https://github.com/NodeSecure/tree/master/workspaces/tree-walker#readme", "dependencies": { - "@nodesecure/js-x-ray": "14.3.0", + "@nodesecure/js-x-ray": "15.0.0", "@nodesecure/mama": "2.2.0", "@nodesecure/npm-registry-sdk": "4.5.2", "@nodesecure/npm-types": "^1.1.0", diff --git a/workspaces/utils/package.json b/workspaces/utils/package.json index 80744f07..7a5226e8 100644 --- a/workspaces/utils/package.json +++ b/workspaces/utils/package.json @@ -11,7 +11,7 @@ "scripts": { "build": "tsc -b", "prepublishOnly": "npm run build", - "test-only": "node --test ./test/**/*.spec.ts", + "test-only": "node --test \"./test/**/*.spec.ts\"", "test-types": "attw --pack . --profile esm-only", "test": "c8 -r html npm run test-only && npm run test-types" },