diff --git a/packages/trees/README.md b/packages/trees/README.md index 59faf50ba..0e0db354c 100644 --- a/packages/trees/README.md +++ b/packages/trees/README.md @@ -206,6 +206,7 @@ From `packages/trees`: ```bash bun test bun run benchmark +bun run benchmark:core bun run test:e2e bun run tsc bun run build @@ -265,6 +266,41 @@ bun ws trees benchmark -- --case=linux --compare tmp/fileListToTree-baseline.jso mismatches. That makes it useful both for performance regressions and for catching accidental behavior changes while refactoring. +For core tree primitive profiling, use the dedicated benchmark runner: + +```bash +bun ws trees benchmark:core +``` + +If you care most about large datasets, run a filtered large-shape subset: + +```bash +bun ws trees benchmark:core -- --case=large-wide --case=large-monorepo --case=linux +``` + +This benchmark isolates core tree costs by preparing fixture-backed tree data up +front and timing only primitive calls. The `createTree` timing reflects the real +initialization path (`createTree` + `setMounted(true)` + initial `rebuildTree`). +`rebuildTree` can run either as unchanged hot rebuilds or as changed-state +rebuilds via `--rebuild-mode=expanded-copy`. + +To better mirror the trees-dev virtualization workload, benchmark cases are +built with `sort: false` and `flattenEmptyDirectories: true`. + +It also supports `--json`, `--compare`, and `--case` filters, plus: + +- `--create-iterations` to batch multiple create+mount+initial-rebuild calls per + measured sample +- `--rebuild-iterations` to batch multiple `rebuildTree` calls per measured + sample +- `--rebuild-mode` to choose unchanged rebuilds or a changed-state mode + (`expanded-copy`) with stronger update-path signal +- `--feature-profile` to switch between `virtualized-card` realism, + `root-default`, and `minimal` core-only feature overhead + +Those batching flags improve confidence for fast operations by reducing timer +jitter while still reporting per-call milliseconds. + # Credits and Acknolwedgements The core of this library's underlying tree implementation started as a hard fork diff --git a/packages/trees/package.json b/packages/trees/package.json index c6ab4fa3e..c808affcc 100644 --- a/packages/trees/package.json +++ b/packages/trees/package.json @@ -35,6 +35,8 @@ "scripts": { "build": "tsdown --clean", "benchmark": "bun run ./scripts/benchmarkFileListToTree.ts", + "benchmark:file-list-to-tree": "bun run ./scripts/benchmarkFileListToTree.ts", + "benchmark:core": "bun run ./scripts/benchmarkTreeCorePrimitives.ts", "dev": "echo 'Watching for changes…' && tsdown --watch --log-level error", "test": "bun test", "coverage": "bun test --coverage", diff --git a/packages/trees/scripts/benchmarkFileListToTree.ts b/packages/trees/scripts/benchmarkFileListToTree.ts index 392c42c0d..da3af50c7 100644 --- a/packages/trees/scripts/benchmarkFileListToTree.ts +++ b/packages/trees/scripts/benchmarkFileListToTree.ts @@ -1,16 +1,29 @@ import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; -import type { FileTreeData } from '../src/types'; import { benchmarkFileListToTreeStages, type FileListToTreeStageName, } from '../src/utils/fileListToTree'; +import { + type BenchmarkEnvironment, + calculateDeltaPercent, + formatMs, + formatSignedMs, + formatSignedPercent, + getEnvironment, + parseNonNegativeInteger, + parsePositiveInteger, + printTable, + summarizeSamples, + type TimingSummary, +} from './lib/benchmarkUtils'; import { type FileListToTreeBenchmarkCase, filterBenchmarkCases, getFileListToTreeBenchmarkCases, } from './lib/fileListToTreeBenchmarkData'; +import { checksumFileTreeData } from './lib/treeBenchmarkChecksums'; interface BenchmarkConfig { runs: number; @@ -20,22 +33,6 @@ interface BenchmarkConfig { comparePath?: string; } -interface BenchmarkEnvironment { - bunVersion: string; - platform: string; - arch: string; -} - -interface TimingSummary { - runs: number; - meanMs: number; - medianMs: number; - p95Ms: number; - minMs: number; - maxMs: number; - stdDevMs: number; -} - interface CaseSummary extends TimingSummary { name: string; source: FileListToTreeBenchmarkCase['source']; @@ -121,26 +118,6 @@ const STAGE_ORDER: FileListToTreeStageName[] = [ 'hashTreeKeys', ]; -function parsePositiveInteger(value: string, flagName: string): number { - const parsed = Number.parseInt(value, 10); - if (!Number.isFinite(parsed) || parsed <= 0) { - throw new Error( - `Invalid ${flagName} value '${value}'. Expected a positive integer.` - ); - } - return parsed; -} - -function parseNonNegativeInteger(value: string, flagName: string): number { - const parsed = Number.parseInt(value, 10); - if (!Number.isFinite(parsed) || parsed < 0) { - throw new Error( - `Invalid ${flagName} value '${value}'. Expected a non-negative integer.` - ); - } - return parsed; -} - function printHelpAndExit(): never { console.log('Usage: bun ws trees benchmark -- [options]'); console.log(''); @@ -209,136 +186,6 @@ function parseArgs(argv: string[]): BenchmarkConfig { return config; } -function percentile(sortedValues: number[], percentileRank: number): number { - if (sortedValues.length === 0) { - return 0; - } - - const rank = (sortedValues.length - 1) * percentileRank; - const lowerIndex = Math.floor(rank); - const upperIndex = Math.ceil(rank); - const lower = sortedValues[lowerIndex] ?? sortedValues[0] ?? 0; - const upper = - sortedValues[upperIndex] ?? sortedValues[sortedValues.length - 1] ?? lower; - if (lowerIndex === upperIndex) { - return lower; - } - - const interpolation = rank - lowerIndex; - return lower + (upper - lower) * interpolation; -} - -function summarizeSamples(samples: number[]): TimingSummary { - if (samples.length === 0) { - return { - runs: 0, - meanMs: 0, - medianMs: 0, - p95Ms: 0, - minMs: 0, - maxMs: 0, - stdDevMs: 0, - }; - } - - const sortedSamples = [...samples].sort((left, right) => left - right); - const total = samples.reduce((sum, value) => sum + value, 0); - const mean = total / samples.length; - const variance = - samples.reduce((sum, value) => sum + (value - mean) ** 2, 0) / - samples.length; - - return { - runs: samples.length, - meanMs: mean, - medianMs: percentile(sortedSamples, 0.5), - p95Ms: percentile(sortedSamples, 0.95), - minMs: sortedSamples[0] ?? 0, - maxMs: sortedSamples[sortedSamples.length - 1] ?? 0, - stdDevMs: Math.sqrt(variance), - }; -} - -function formatMs(value: number): string { - return value.toFixed(3); -} - -function formatSignedMs(value: number): string { - const prefix = value > 0 ? '+' : ''; - return `${prefix}${value.toFixed(3)}`; -} - -function formatSignedPercent(value: number): string { - if (!Number.isFinite(value)) { - return value > 0 ? '+inf%' : value < 0 ? '-inf%' : '0.0%'; - } - - const prefix = value > 0 ? '+' : ''; - return `${prefix}${value.toFixed(1)}%`; -} - -function checksumTree(tree: FileTreeData): number { - let checksum = 0; - - for (const [id, node] of Object.entries(tree)) { - checksum += id.length; - checksum += node.name.length; - checksum += node.path.length; - - if (node.children != null) { - checksum += node.children.direct.length; - for (const child of node.children.direct) { - checksum += child.length; - } - if (node.children.flattened != null) { - checksum += node.children.flattened.length; - for (const child of node.children.flattened) { - checksum += child.length; - } - } - } - - if (node.flattens != null) { - checksum += node.flattens.length; - for (const path of node.flattens) { - checksum += path.length; - } - } - } - - return checksum; -} - -function printTable(rows: Record[], headers: string[]): void { - const widths = headers.map((header) => { - const valueWidth = rows.reduce( - (max, row) => Math.max(max, row[header]?.length ?? 0), - header.length - ); - return valueWidth; - }); - - const formatRow = (row: Record) => - headers - .map((header, index) => (row[header] ?? '').padEnd(widths[index])) - .join(' ') - .trimEnd(); - - const headerRow = Object.fromEntries( - headers.map((header) => [header, header]) - ); - console.log(formatRow(headerRow)); - console.log( - widths - .map((width) => '-'.repeat(width)) - .join(' ') - .trimEnd() - ); - for (const row of rows) { - console.log(formatRow(row)); - } -} - function createStageSampleStorage(): Record { return { buildPathGraph: [], @@ -348,22 +195,6 @@ function createStageSampleStorage(): Record { }; } -function getEnvironment(): BenchmarkEnvironment { - return { - bunVersion: Bun.version, - platform: process.platform, - arch: process.arch, - }; -} - -function calculateDeltaPercent(current: number, baseline: number): number { - if (baseline === 0) { - return current === 0 ? 0 : Number.POSITIVE_INFINITY; - } - - return ((current - baseline) / baseline) * 100; -} - // Benchmarks only stay comparable when the output payload has the same shape. // Load and validate the previous JSON run up front so comparison failures are // immediate instead of producing misleading deltas later on. @@ -608,7 +439,7 @@ function main() { const startTime = performance.now(); const result = benchmarkFileListToTreeStages(caseConfig.files); const elapsedMs = performance.now() - startTime; - const resultChecksum = checksumTree(result.tree); + const resultChecksum = checksumFileTreeData(result.tree); const existingChecksum = caseChecksums[caseIndex]; if (existingChecksum == null) { diff --git a/packages/trees/scripts/benchmarkTreeCorePrimitives.ts b/packages/trees/scripts/benchmarkTreeCorePrimitives.ts new file mode 100644 index 000000000..e43ad8b4a --- /dev/null +++ b/packages/trees/scripts/benchmarkTreeCorePrimitives.ts @@ -0,0 +1,870 @@ +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; + +import { createTree } from '../src/core/create-tree'; +import type { + FeatureImplementation, + TreeConfig, + TreeInstance, +} from '../src/core/types/core'; +import { contextMenuFeature } from '../src/features/context-menu/feature'; +import { gitStatusFeature } from '../src/features/git-status/feature'; +import { hotkeysCoreFeature } from '../src/features/hotkeys-core/feature'; +import { propMemoizationFeature } from '../src/features/prop-memoization/feature'; +import { renamingFeature } from '../src/features/renaming/feature'; +import { fileTreeSearchFeature } from '../src/features/search/feature'; +import { selectionFeature } from '../src/features/selection/feature'; +import { syncDataLoaderFeature } from '../src/features/sync-data-loader/feature'; +import { generateSyncDataLoaderFromTreeData } from '../src/loader/sync'; +import type { FileTreeData, FileTreeNode } from '../src/types'; +import { fileListToTree } from '../src/utils/fileListToTree'; +import { + type BenchmarkEnvironment, + calculateDeltaPercent, + formatMs, + formatSignedMs, + formatSignedPercent, + getEnvironment, + measureAverageIterationMs, + parseNonNegativeInteger, + parsePositiveInteger, + printTable, + summarizeSamples, + type TimingSummary, +} from './lib/benchmarkUtils'; +import { + type FileListToTreeBenchmarkCase, + filterBenchmarkCases, + getFileListToTreeBenchmarkCases, +} from './lib/fileListToTreeBenchmarkData'; +import { + checksumFileTreeData, + checksumTreeItems, +} from './lib/treeBenchmarkChecksums'; + +type TreeCorePrimitiveName = 'createTree' | 'rebuildTree'; +type FeatureProfileName = 'minimal' | 'root-default' | 'virtualized-card'; +type RebuildModeName = 'unchanged' | 'expanded-copy'; + +interface BenchmarkConfig { + runs: number; + warmupRuns: number; + outputJson: boolean; + caseFilters: string[]; + createIterations: number; + rebuildIterations: number; + featureProfile: FeatureProfileName; + rebuildMode: RebuildModeName; + comparePath?: string; +} + +interface PreparedBenchmarkCase { + name: string; + source: FileListToTreeBenchmarkCase['source']; + fileCount: number; + uniqueFolderCount: number; + maxDepth: number; + treeNodeCount: number; + folderNodeCount: number; + expandedItemCount: number; + expandedItemIds: string[]; + treeChecksum: number; + createConfig: TreeConfig; + rebuildInstance: TreeInstance; +} + +interface CaseSummary { + name: string; + source: FileListToTreeBenchmarkCase['source']; + fileCount: number; + uniqueFolderCount: number; + maxDepth: number; + treeNodeCount: number; + folderNodeCount: number; + expandedItemCount: number; + treeChecksum: number; + createChecksum: number; + rebuildChecksum: number; + operations: Record; +} + +interface OperationComparisonSummary { + baselineMedianMs: number; + currentMedianMs: number; + medianDeltaMs: number; + medianDeltaPct: number; + baselineMeanMs: number; + currentMeanMs: number; + meanDeltaMs: number; + meanDeltaPct: number; + baselineP95Ms: number; + currentP95Ms: number; + p95DeltaMs: number; + p95DeltaPct: number; +} + +interface CaseComparison { + name: string; + treeChecksumMatches: boolean; + createChecksumMatches: boolean; + rebuildChecksumMatches: boolean; + operations: Record; +} + +interface BenchmarkComparison { + baselinePath: string; + baselineEnvironment: BenchmarkEnvironment; + baselineConfig: BenchmarkConfig; + unmatchedCurrentCases: string[]; + unmatchedBaselineCases: string[]; + checksumMismatches: string[]; + cases: CaseComparison[]; +} + +interface BenchmarkOutput { + benchmark: 'treeCorePrimitives'; + environment: BenchmarkEnvironment; + config: BenchmarkConfig; + checksum: number; + cases: CaseSummary[]; + comparison?: BenchmarkComparison; +} + +interface LoadedBenchmarkBaseline { + path: string; + output: BenchmarkOutput; +} + +const OPERATION_ORDER: TreeCorePrimitiveName[] = ['createTree', 'rebuildTree']; + +const MINIMAL_FEATURE_PROFILE: FeatureImplementation[] = [ + syncDataLoaderFeature, +]; +const ROOT_DEFAULT_FEATURE_PROFILE: FeatureImplementation[] = [ + syncDataLoaderFeature, + selectionFeature, + hotkeysCoreFeature, + fileTreeSearchFeature, + gitStatusFeature, + contextMenuFeature, + propMemoizationFeature, +]; + +const VIRTUALIZED_CARD_FEATURE_PROFILE: FeatureImplementation[] = [ + syncDataLoaderFeature, + selectionFeature, + hotkeysCoreFeature, + fileTreeSearchFeature, + gitStatusFeature, + contextMenuFeature, + renamingFeature, + propMemoizationFeature, +]; + +function parseFeatureProfile(value: string): FeatureProfileName { + if ( + value === 'minimal' || + value === 'root-default' || + value === 'virtualized-card' + ) { + return value; + } + + throw new Error( + `Invalid --feature-profile value '${value}'. Expected one of: minimal, root-default, virtualized-card.` + ); +} + +function getFeaturesForProfile( + profile: FeatureProfileName +): FeatureImplementation[] { + if (profile === 'minimal') { + return MINIMAL_FEATURE_PROFILE; + } + if (profile === 'virtualized-card') { + return VIRTUALIZED_CARD_FEATURE_PROFILE; + } + return ROOT_DEFAULT_FEATURE_PROFILE; +} + +function parseRebuildMode(value: string): RebuildModeName { + if (value === 'unchanged' || value === 'expanded-copy') { + return value; + } + + throw new Error( + `Invalid --rebuild-mode value '${value}'. Expected one of: unchanged, expanded-copy.` + ); +} + +const DEFAULT_CONFIG: BenchmarkConfig = { + runs: 60, + warmupRuns: 8, + outputJson: false, + caseFilters: [], + createIterations: 1, + rebuildIterations: 8, + featureProfile: 'virtualized-card', + rebuildMode: 'unchanged', +}; + +function printHelpAndExit(): never { + console.log('Usage: bun ws trees benchmark:core -- [options]'); + console.log(''); + console.log('Options:'); + console.log( + ' --runs Measured runs per benchmark case (default: 60)' + ); + console.log( + ' --warmup-runs Warmup runs per benchmark case before measurement (default: 8)' + ); + console.log( + ' --create-iterations createTree + setMounted(true) + rebuildTree calls per measured sample (default: 1)' + ); + console.log( + ' --rebuild-iterations rebuildTree calls per measured sample (default: 8)' + ); + console.log( + ' --rebuild-mode Rebuild mode: unchanged | expanded-copy (default: unchanged)' + ); + console.log( + ' --feature-profile Feature profile: minimal | root-default | virtualized-card (default: virtualized-card)' + ); + console.log( + ' --case Run only cases whose name contains the filter (repeatable)' + ); + console.log( + ' --compare Compare against a prior --json benchmark run' + ); + console.log( + ' --json Emit machine-readable JSON output' + ); + console.log(' -h, --help Show this help output'); + process.exit(0); +} + +function parseArgs(argv: string[]): BenchmarkConfig { + const config: BenchmarkConfig = { ...DEFAULT_CONFIG }; + + for (let index = 0; index < argv.length; index++) { + const rawArg = argv[index]; + if (rawArg === '--help' || rawArg === '-h') { + printHelpAndExit(); + } + + if (rawArg === '--json') { + config.outputJson = true; + continue; + } + + const [flag, inlineValue] = rawArg.split('=', 2); + if ( + flag === '--runs' || + flag === '--warmup-runs' || + flag === '--create-iterations' || + flag === '--rebuild-iterations' || + flag === '--rebuild-mode' || + flag === '--feature-profile' || + flag === '--case' || + flag === '--compare' + ) { + const value = inlineValue ?? argv[index + 1]; + if (value == null) { + throw new Error(`Missing value for ${flag}`); + } + if (inlineValue == null) { + index += 1; + } + + if (flag === '--runs') { + config.runs = parsePositiveInteger(value, '--runs'); + } else if (flag === '--warmup-runs') { + config.warmupRuns = parseNonNegativeInteger(value, '--warmup-runs'); + } else if (flag === '--create-iterations') { + config.createIterations = parsePositiveInteger( + value, + '--create-iterations' + ); + } else if (flag === '--rebuild-iterations') { + config.rebuildIterations = parsePositiveInteger( + value, + '--rebuild-iterations' + ); + } else if (flag === '--rebuild-mode') { + config.rebuildMode = parseRebuildMode(value); + } else if (flag === '--feature-profile') { + config.featureProfile = parseFeatureProfile(value); + } else if (flag === '--case') { + config.caseFilters.push(value); + } else { + config.comparePath = value; + } + continue; + } + + throw new Error(`Unknown argument: ${rawArg}`); + } + + return config; +} + +function createOperationSampleStorage(): Record< + TreeCorePrimitiveName, + number[] +> { + return { + createTree: [], + rebuildTree: [], + }; +} + +function collectFolderIds(treeData: FileTreeData): { + folderNodeCount: number; + expandedItemIds: string[]; +} { + const expandedItemIds: string[] = []; + let folderNodeCount = 0; + + for (const [id, node] of Object.entries(treeData)) { + if (node.children == null) { + continue; + } + + folderNodeCount += 1; + if (id !== 'root') { + expandedItemIds.push(id); + } + } + + return { + folderNodeCount, + expandedItemIds, + }; +} + +// Resolves fixture-provided expanded folder paths to item IDs in the built +// tree. Falls back to expanding all folders when no mapping is available. +function resolveExpandedItemIds( + treeData: FileTreeData, + fallbackExpandedItemIds: string[], + expandedFolderPaths: string[] | undefined +): string[] { + if (expandedFolderPaths == null) { + return fallbackExpandedItemIds; + } + + const pathToId = new Map(); + for (const [id, node] of Object.entries(treeData)) { + if (node.children != null) { + pathToId.set(node.path, id); + } + } + + const resolvedIds: string[] = []; + const seen = new Set(); + for (const folderPath of expandedFolderPaths) { + const id = pathToId.get(folderPath); + if (id == null || id === 'root' || seen.has(id)) { + continue; + } + seen.add(id); + resolvedIds.push(id); + } + + return resolvedIds.length > 0 ? resolvedIds : fallbackExpandedItemIds; +} + +// Builds the core benchmark fixture once per case so measured runs include +// only createTree/rebuildTree work and not file-list parsing or loader setup. +function prepareBenchmarkCase( + caseConfig: FileListToTreeBenchmarkCase, + featureProfile: FeatureProfileName +): PreparedBenchmarkCase { + // Match the trees-dev virtualization workload: disable sorting and use + // flattened-directory traversal in the sync loader. + const treeData = fileListToTree(caseConfig.files, { + sortComparator: false, + }); + const { folderNodeCount, expandedItemIds: allFolderItemIds } = + collectFolderIds(treeData); + const expandedItemIds = resolveExpandedItemIds( + treeData, + allFolderItemIds, + caseConfig.expandedFolders + ); + const createConfig: TreeConfig = { + rootItemId: 'root', + dataLoader: generateSyncDataLoaderFromTreeData(treeData, { + flattenEmptyDirectories: true, + }), + getItemName: (item) => item.getItemData().name, + isItemFolder: (item) => item.getItemData().children != null, + features: [...getFeaturesForProfile(featureProfile)], + initialState: { + expandedItems: expandedItemIds, + focusedItem: null, + }, + }; + + const rebuildInstance = createTree(createConfig); + rebuildInstance.setMounted(true); + rebuildInstance.rebuildTree(); + + return { + name: caseConfig.name, + source: caseConfig.source, + fileCount: caseConfig.fileCount, + uniqueFolderCount: caseConfig.uniqueFolderCount, + maxDepth: caseConfig.maxDepth, + treeNodeCount: Object.keys(treeData).length, + folderNodeCount, + expandedItemCount: expandedItemIds.length, + expandedItemIds, + treeChecksum: checksumFileTreeData(treeData), + createConfig, + rebuildInstance, + }; +} + +// Benchmarks only stay comparable when the output payload has the same shape. +// Load and validate the previous JSON run up front so comparison failures are +// immediate instead of producing misleading deltas later on. +function readBenchmarkBaseline(comparePath: string): LoadedBenchmarkBaseline { + const resolvedPath = resolve(process.cwd(), comparePath); + const parsed = JSON.parse( + readFileSync(resolvedPath, 'utf-8') + ) as Partial | null; + + if (parsed == null || parsed.benchmark !== 'treeCorePrimitives') { + throw new Error( + `Invalid benchmark baseline at ${resolvedPath}. Expected treeCorePrimitives JSON output.` + ); + } + + if (!Array.isArray(parsed.cases)) { + throw new Error( + `Invalid benchmark baseline at ${resolvedPath}. Expected a cases array.` + ); + } + + return { + path: resolvedPath, + output: parsed as BenchmarkOutput, + }; +} + +function buildComparison( + baseline: LoadedBenchmarkBaseline, + caseSummaries: CaseSummary[] +): BenchmarkComparison { + const baselineCases = new Map( + baseline.output.cases.map((summary) => [summary.name, summary]) + ); + const currentCaseNames = new Set( + caseSummaries.map((summary) => summary.name) + ); + + const matchedCases = caseSummaries.filter((summary) => + baselineCases.has(summary.name) + ); + if (matchedCases.length === 0) { + throw new Error( + `No benchmark cases matched baseline ${baseline.path}. Regenerate the baseline or adjust --case filters.` + ); + } + + const caseComparisons = matchedCases.map((currentSummary) => { + const baselineSummary = baselineCases.get(currentSummary.name); + if (baselineSummary == null) { + throw new Error(`Missing baseline case for ${currentSummary.name}`); + } + + if ( + typeof baselineSummary.treeChecksum !== 'number' || + typeof baselineSummary.createChecksum !== 'number' || + typeof baselineSummary.rebuildChecksum !== 'number' + ) { + throw new Error( + `Baseline case ${currentSummary.name} is missing checksums. Regenerate the baseline with the current benchmark script.` + ); + } + + const operationComparisons = Object.fromEntries( + OPERATION_ORDER.map((operation) => { + const baselineOperation = baselineSummary.operations?.[operation]; + const currentOperation = currentSummary.operations[operation]; + if (baselineOperation == null) { + throw new Error( + `Missing ${operation} summary for ${currentSummary.name}. Regenerate the baseline with the current benchmark script.` + ); + } + + return [ + operation, + { + baselineMedianMs: baselineOperation.medianMs, + currentMedianMs: currentOperation.medianMs, + medianDeltaMs: + currentOperation.medianMs - baselineOperation.medianMs, + medianDeltaPct: calculateDeltaPercent( + currentOperation.medianMs, + baselineOperation.medianMs + ), + baselineMeanMs: baselineOperation.meanMs, + currentMeanMs: currentOperation.meanMs, + meanDeltaMs: currentOperation.meanMs - baselineOperation.meanMs, + meanDeltaPct: calculateDeltaPercent( + currentOperation.meanMs, + baselineOperation.meanMs + ), + baselineP95Ms: baselineOperation.p95Ms, + currentP95Ms: currentOperation.p95Ms, + p95DeltaMs: currentOperation.p95Ms - baselineOperation.p95Ms, + p95DeltaPct: calculateDeltaPercent( + currentOperation.p95Ms, + baselineOperation.p95Ms + ), + }, + ]; + }) + ) as Record; + + return { + name: currentSummary.name, + treeChecksumMatches: + baselineSummary.treeChecksum === currentSummary.treeChecksum, + createChecksumMatches: + baselineSummary.createChecksum === currentSummary.createChecksum, + rebuildChecksumMatches: + baselineSummary.rebuildChecksum === currentSummary.rebuildChecksum, + operations: operationComparisons, + }; + }); + + return { + baselinePath: baseline.path, + baselineEnvironment: baseline.output.environment, + baselineConfig: baseline.output.config, + unmatchedCurrentCases: caseSummaries + .filter((summary) => !baselineCases.has(summary.name)) + .map((summary) => summary.name), + unmatchedBaselineCases: baseline.output.cases + .filter((summary) => !currentCaseNames.has(summary.name)) + .map((summary) => summary.name), + checksumMismatches: caseComparisons + .filter( + (summary) => + !summary.treeChecksumMatches || + !summary.createChecksumMatches || + !summary.rebuildChecksumMatches + ) + .map((summary) => summary.name), + cases: caseComparisons, + }; +} + +function printComparison(comparison: BenchmarkComparison): void { + console.log(''); + console.log('Comparison vs baseline'); + console.log(`baseline=${comparison.baselinePath}`); + console.log( + `baselineBun=${comparison.baselineEnvironment.bunVersion} baselinePlatform=${comparison.baselineEnvironment.platform} baselineArch=${comparison.baselineEnvironment.arch}` + ); + console.log( + `baselineRunsPerCase=${comparison.baselineConfig.runs} baselineWarmupRunsPerCase=${comparison.baselineConfig.warmupRuns}` + ); + console.log( + `baselineCreateIterations=${comparison.baselineConfig.createIterations} baselineRebuildIterations=${comparison.baselineConfig.rebuildIterations}` + ); + console.log( + `baselineFeatureProfile=${comparison.baselineConfig.featureProfile}` + ); + console.log(`baselineRebuildMode=${comparison.baselineConfig.rebuildMode}`); + + if (comparison.unmatchedCurrentCases.length > 0) { + console.log( + `unmatchedCurrentCases=${comparison.unmatchedCurrentCases.join(', ')}` + ); + } + if (comparison.unmatchedBaselineCases.length > 0) { + console.log( + `unmatchedBaselineCases=${comparison.unmatchedBaselineCases.join(', ')}` + ); + } + if (comparison.checksumMismatches.length > 0) { + console.log( + `checksumMismatches=${comparison.checksumMismatches.join(', ')}` + ); + } + + console.log(''); + console.log('Case median deltas'); + printTable( + comparison.cases.map((summary) => ({ + case: summary.name, + createDeltaMs: formatSignedMs( + summary.operations.createTree.medianDeltaMs + ), + createDeltaPct: formatSignedPercent( + summary.operations.createTree.medianDeltaPct + ), + rebuildDeltaMs: formatSignedMs( + summary.operations.rebuildTree.medianDeltaMs + ), + rebuildDeltaPct: formatSignedPercent( + summary.operations.rebuildTree.medianDeltaPct + ), + treeChecksum: summary.treeChecksumMatches ? 'match' : 'mismatch', + createChecksum: summary.createChecksumMatches ? 'match' : 'mismatch', + rebuildChecksum: summary.rebuildChecksumMatches ? 'match' : 'mismatch', + })), + [ + 'case', + 'createDeltaMs', + 'createDeltaPct', + 'rebuildDeltaMs', + 'rebuildDeltaPct', + 'treeChecksum', + 'createChecksum', + 'rebuildChecksum', + ] + ); +} + +function main() { + const config = parseArgs(process.argv.slice(2)); + const selectedCaseConfigs = filterBenchmarkCases( + getFileListToTreeBenchmarkCases(), + config.caseFilters + ); + + if (selectedCaseConfigs.length === 0) { + throw new Error('No benchmark cases matched the provided --case filters.'); + } + + const preparedCases = selectedCaseConfigs.map((caseConfig) => + prepareBenchmarkCase(caseConfig, config.featureProfile) + ); + const samplesByCase = preparedCases.map(() => createOperationSampleStorage()); + const createChecksums = preparedCases.map( + () => undefined as number | undefined + ); + const rebuildChecksums = preparedCases.map( + () => undefined as number | undefined + ); + + const runCaseOperation = ( + caseConfig: PreparedBenchmarkCase, + caseIndex: number, + operation: TreeCorePrimitiveName + ) => { + if (operation === 'createTree') { + let lastCreatedTree: TreeInstance | undefined; + const elapsedMs = measureAverageIterationMs( + config.createIterations, + () => { + const createdTree = createTree(caseConfig.createConfig); + createdTree.setMounted(true); + createdTree.rebuildTree(); + lastCreatedTree = createdTree; + } + ); + + const createdTree = lastCreatedTree; + if (createdTree == null) { + throw new Error(`Missing createTree result for ${caseConfig.name}`); + } + + const checksum = checksumTreeItems(createdTree); + const existingChecksum = createChecksums[caseIndex]; + if (existingChecksum == null) { + createChecksums[caseIndex] = checksum; + } else if (existingChecksum !== checksum) { + throw new Error( + `Non-deterministic createTree checksum for benchmark case ${caseConfig.name}. Expected ${existingChecksum}, received ${checksum}.` + ); + } + + return { elapsedMs }; + } + + const elapsedMs = measureAverageIterationMs( + config.rebuildIterations, + () => { + if (config.rebuildMode === 'expanded-copy') { + caseConfig.rebuildInstance.setConfig((previousConfig) => ({ + ...previousConfig, + state: { + ...(previousConfig.state ?? {}), + expandedItems: [...caseConfig.expandedItemIds], + }, + })); + } + + caseConfig.rebuildInstance.rebuildTree(); + } + ); + const checksum = checksumTreeItems(caseConfig.rebuildInstance); + const existingChecksum = rebuildChecksums[caseIndex]; + if (existingChecksum == null) { + rebuildChecksums[caseIndex] = checksum; + } else if (existingChecksum !== checksum) { + throw new Error( + `Non-deterministic rebuildTree checksum for benchmark case ${caseConfig.name}. Expected ${existingChecksum}, received ${checksum}.` + ); + } + + return { elapsedMs }; + }; + + // Measure each primitive in its own pass so createTree allocation churn does + // not bleed into rebuildTree samples (and vice-versa). + for (const operation of OPERATION_ORDER) { + for (let runIndex = 0; runIndex < config.warmupRuns; runIndex++) { + for ( + let caseOffset = 0; + caseOffset < preparedCases.length; + caseOffset++ + ) { + const caseIndex = (runIndex + caseOffset) % preparedCases.length; + const caseConfig = preparedCases[caseIndex]; + runCaseOperation(caseConfig, caseIndex, operation); + } + } + + for (let runIndex = 0; runIndex < config.runs; runIndex++) { + for ( + let caseOffset = 0; + caseOffset < preparedCases.length; + caseOffset++ + ) { + const caseIndex = (runIndex + caseOffset) % preparedCases.length; + const caseConfig = preparedCases[caseIndex]; + const { elapsedMs } = runCaseOperation( + caseConfig, + caseIndex, + operation + ); + samplesByCase[caseIndex][operation].push(elapsedMs); + } + } + } + + const caseSummaries: CaseSummary[] = preparedCases.map( + (caseConfig, index) => { + const createChecksum = createChecksums[index]; + if (createChecksum == null) { + throw new Error(`Missing createTree checksum for ${caseConfig.name}`); + } + + const rebuildChecksum = rebuildChecksums[index]; + if (rebuildChecksum == null) { + throw new Error(`Missing rebuildTree checksum for ${caseConfig.name}`); + } + + return { + name: caseConfig.name, + source: caseConfig.source, + fileCount: caseConfig.fileCount, + uniqueFolderCount: caseConfig.uniqueFolderCount, + maxDepth: caseConfig.maxDepth, + treeNodeCount: caseConfig.treeNodeCount, + folderNodeCount: caseConfig.folderNodeCount, + expandedItemCount: caseConfig.expandedItemCount, + treeChecksum: caseConfig.treeChecksum, + createChecksum, + rebuildChecksum, + operations: { + createTree: summarizeSamples(samplesByCase[index].createTree), + rebuildTree: summarizeSamples(samplesByCase[index].rebuildTree), + }, + }; + } + ); + + const checksum = caseSummaries.reduce( + (sum, summary) => + sum + + summary.treeChecksum + + summary.createChecksum + + summary.rebuildChecksum, + 0 + ); + const environment = getEnvironment(); + const comparison = + config.comparePath != null + ? buildComparison( + readBenchmarkBaseline(config.comparePath), + caseSummaries + ) + : undefined; + + const output: BenchmarkOutput = { + benchmark: 'treeCorePrimitives', + environment, + config, + checksum, + cases: caseSummaries, + ...(comparison != null && { comparison }), + }; + + if (config.outputJson) { + console.log(JSON.stringify(output, null, 2)); + return; + } + + console.log('tree core primitives benchmark'); + console.log( + `bun=${environment.bunVersion} platform=${environment.platform} arch=${environment.arch}` + ); + console.log( + `cases=${preparedCases.length} runsPerCase=${config.runs} warmupRunsPerCase=${config.warmupRuns}` + ); + console.log( + `createIterationsPerSample=${config.createIterations} rebuildIterationsPerSample=${config.rebuildIterations}` + ); + console.log(`featureProfile=${config.featureProfile}`); + console.log(`rebuildMode=${config.rebuildMode}`); + if (config.caseFilters.length > 0) { + console.log(`filters=${config.caseFilters.join(', ')}`); + } + console.log(`checksum=${checksum}`); + console.log(''); + + printTable( + caseSummaries.map((summary) => ({ + case: summary.name, + source: summary.source, + files: String(summary.fileCount), + folders: String(summary.uniqueFolderCount), + depth: String(summary.maxDepth), + nodes: String(summary.treeNodeCount), + expanded: String(summary.expandedItemCount), + runs: String(summary.operations.createTree.runs), + createMedianMs: formatMs(summary.operations.createTree.medianMs), + createP95Ms: formatMs(summary.operations.createTree.p95Ms), + rebuildMedianMs: formatMs(summary.operations.rebuildTree.medianMs), + rebuildP95Ms: formatMs(summary.operations.rebuildTree.p95Ms), + })), + [ + 'case', + 'source', + 'files', + 'folders', + 'depth', + 'nodes', + 'expanded', + 'runs', + 'createMedianMs', + 'createP95Ms', + 'rebuildMedianMs', + 'rebuildP95Ms', + ] + ); + + if (comparison != null) { + printComparison(comparison); + } +} + +main(); diff --git a/packages/trees/scripts/lib/benchmarkUtils.ts b/packages/trees/scripts/lib/benchmarkUtils.ts new file mode 100644 index 000000000..0d96eca5f --- /dev/null +++ b/packages/trees/scripts/lib/benchmarkUtils.ts @@ -0,0 +1,177 @@ +export interface BenchmarkEnvironment { + bunVersion: string; + platform: string; + arch: string; +} + +export interface TimingSummary { + runs: number; + meanMs: number; + medianMs: number; + p95Ms: number; + minMs: number; + maxMs: number; + stdDevMs: number; +} + +export function parsePositiveInteger(value: string, flagName: string): number { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed <= 0) { + throw new Error( + `Invalid ${flagName} value '${value}'. Expected a positive integer.` + ); + } + return parsed; +} + +export function parseNonNegativeInteger( + value: string, + flagName: string +): number { + const parsed = Number.parseInt(value, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + throw new Error( + `Invalid ${flagName} value '${value}'. Expected a non-negative integer.` + ); + } + return parsed; +} + +export function percentile( + sortedValues: number[], + percentileRank: number +): number { + if (sortedValues.length === 0) { + return 0; + } + + const rank = (sortedValues.length - 1) * percentileRank; + const lowerIndex = Math.floor(rank); + const upperIndex = Math.ceil(rank); + const lower = sortedValues[lowerIndex] ?? sortedValues[0] ?? 0; + const upper = + sortedValues[upperIndex] ?? sortedValues[sortedValues.length - 1] ?? lower; + if (lowerIndex === upperIndex) { + return lower; + } + + const interpolation = rank - lowerIndex; + return lower + (upper - lower) * interpolation; +} + +// Converts raw timing samples into the same summary statistics used in JSON +// output, text tables, and baseline comparisons. +export function summarizeSamples(samples: number[]): TimingSummary { + if (samples.length === 0) { + return { + runs: 0, + meanMs: 0, + medianMs: 0, + p95Ms: 0, + minMs: 0, + maxMs: 0, + stdDevMs: 0, + }; + } + + const sortedSamples = [...samples].sort((left, right) => left - right); + const total = samples.reduce((sum, value) => sum + value, 0); + const mean = total / samples.length; + const variance = + samples.reduce((sum, value) => sum + (value - mean) ** 2, 0) / + samples.length; + + return { + runs: samples.length, + meanMs: mean, + medianMs: percentile(sortedSamples, 0.5), + p95Ms: percentile(sortedSamples, 0.95), + minMs: sortedSamples[0] ?? 0, + maxMs: sortedSamples[sortedSamples.length - 1] ?? 0, + stdDevMs: Math.sqrt(variance), + }; +} + +export function formatMs(value: number): string { + return value.toFixed(3); +} + +export function formatSignedMs(value: number): string { + const prefix = value > 0 ? '+' : ''; + return `${prefix}${value.toFixed(3)}`; +} + +export function formatSignedPercent(value: number): string { + if (!Number.isFinite(value)) { + return value > 0 ? '+inf%' : value < 0 ? '-inf%' : '0.0%'; + } + + const prefix = value > 0 ? '+' : ''; + return `${prefix}${value.toFixed(1)}%`; +} + +export function printTable( + rows: Record[], + headers: string[] +): void { + const widths = headers.map((header) => { + const valueWidth = rows.reduce( + (max, row) => Math.max(max, row[header]?.length ?? 0), + header.length + ); + return valueWidth; + }); + + const formatRow = (row: Record) => + headers + .map((header, index) => (row[header] ?? '').padEnd(widths[index])) + .join(' ') + .trimEnd(); + + const headerRow = Object.fromEntries( + headers.map((header) => [header, header]) + ); + console.log(formatRow(headerRow)); + console.log( + widths + .map((width) => '-'.repeat(width)) + .join(' ') + .trimEnd() + ); + for (const row of rows) { + console.log(formatRow(row)); + } +} + +export function getEnvironment(): BenchmarkEnvironment { + return { + bunVersion: Bun.version, + platform: process.platform, + arch: process.arch, + }; +} + +export function calculateDeltaPercent( + current: number, + baseline: number +): number { + if (baseline === 0) { + return current === 0 ? 0 : Number.POSITIVE_INFINITY; + } + + return ((current - baseline) / baseline) * 100; +} + +// Runs `runIteration` multiple times and reports per-iteration milliseconds. +// This reduces timer jitter for very fast operations while keeping reported +// numbers comparable to single-call timings. +export function measureAverageIterationMs( + iterations: number, + runIteration: () => void +): number { + const startTime = performance.now(); + for (let iteration = 0; iteration < iterations; iteration++) { + runIteration(); + } + return (performance.now() - startTime) / iterations; +} diff --git a/packages/trees/scripts/lib/fileListToTreeBenchmarkData.ts b/packages/trees/scripts/lib/fileListToTreeBenchmarkData.ts index c8e1dbfbd..a0bd0078e 100644 --- a/packages/trees/scripts/lib/fileListToTreeBenchmarkData.ts +++ b/packages/trees/scripts/lib/fileListToTreeBenchmarkData.ts @@ -16,6 +16,9 @@ export interface FileListToTreeBenchmarkCase extends FileListShapeSummary { name: string; source: 'synthetic' | 'fixture'; files: string[]; + // Optional source-of-truth expanded folder paths for realism-sensitive + // workloads (e.g. trees-dev virtualization fixtures). + expandedFolders?: string[]; } const BENCHMARK_FIXTURE_PATH = resolve( @@ -287,12 +290,14 @@ export function describeFileListShape(files: string[]): FileListShapeSummary { function createCase( name: string, source: 'synthetic' | 'fixture', - files: string[] + files: string[], + expandedFolders?: string[] ): FileListToTreeBenchmarkCase { return { name, source, files, + ...(expandedFolders != null && { expandedFolders }), ...describeFileListShape(files), }; } @@ -304,6 +309,8 @@ export function getFileListToTreeBenchmarkCases(): FileListToTreeBenchmarkCase[] return cachedCases; } + const linuxKernelFixture = readLinuxKernelFixture(LINUX_KERNEL_FIXTURE_PATH); + cachedCases = [ createCase('tiny-flat', 'synthetic', buildTinyFlatFiles()), createCase('small-mixed', 'synthetic', buildSmallMixedFiles()), @@ -323,7 +330,8 @@ export function getFileListToTreeBenchmarkCases(): FileListToTreeBenchmarkCase[] createCase( 'fixture-linux-kernel-files', 'fixture', - readLinuxKernelFixture(LINUX_KERNEL_FIXTURE_PATH).files + linuxKernelFixture.files, + linuxKernelFixture.folders ), createCase( 'fixture-pierrejs-repo-snapshot', diff --git a/packages/trees/scripts/lib/treeBenchmarkChecksums.ts b/packages/trees/scripts/lib/treeBenchmarkChecksums.ts new file mode 100644 index 000000000..4bb201252 --- /dev/null +++ b/packages/trees/scripts/lib/treeBenchmarkChecksums.ts @@ -0,0 +1,72 @@ +import type { TreeInstance } from '../../src/core/types/core'; +import type { FileTreeData } from '../../src/types'; + +export function checksumFileTreeData(tree: FileTreeData): number { + let checksum = 0; + + for (const [id, node] of Object.entries(tree)) { + checksum += id.length; + checksum += node.name.length; + checksum += node.path.length; + + if (node.children != null) { + checksum += node.children.direct.length; + for (const child of node.children.direct) { + checksum += child.length; + } + if (node.children.flattened != null) { + checksum += node.children.flattened.length; + for (const child of node.children.flattened) { + checksum += child.length; + } + } + } + + if (node.flattens != null) { + checksum += node.flattens.length; + for (const path of node.flattens) { + checksum += path.length; + } + } + } + + return checksum; +} + +// Computes a deterministic signature of the flattened item list produced by +// rebuildTree so benchmark runs can detect accidental behavior drift. +export function checksumTreeItems(tree: TreeInstance): number { + const items = tree.getItems(); + let checksum = items.length; + + for (const item of items) { + const itemId = item.getId(); + const itemMeta = item.getItemMeta(); + + checksum += itemId.length; + checksum += itemMeta.level; + checksum += itemMeta.index; + checksum += itemMeta.posInSet; + checksum += itemMeta.setSize; + + if (itemMeta.parentId != null) { + checksum += itemMeta.parentId.length; + } + } + + return checksum; +} + +// Captures lightweight state/config signals after createTree so the benchmark +// can verify deterministic construction without measuring rebuild work. +export function checksumCreateTreeState(tree: TreeInstance): number { + const state = tree.getState(); + const config = tree.getConfig(); + + let checksum = config.rootItemId.length; + checksum += state.expandedItems.length; + checksum += state.focusedItem?.length ?? 0; + checksum += Object.keys(tree.getHotkeyPresets()).length; + + return checksum; +} diff --git a/packages/trees/src/core/build-static-instance.ts b/packages/trees/src/core/build-static-instance.ts index 7d1521e95..3de735ad3 100644 --- a/packages/trees/src/core/build-static-instance.ts +++ b/packages/trees/src/core/build-static-instance.ts @@ -13,7 +13,16 @@ export const buildStaticInstance: InstanceBuilder = ( // Loop goes in forward order, each features overwrite previous ones and wraps those in a prev() fn const definition = features[i][instanceType]; if (definition == null) continue featureLoop; - methodLoop: for (const [key, method] of Object.entries(definition)) { + + // Iterate with `for...in` to avoid allocating an `Object.entries` array + // for every instance finalization (hot when building very large trees). + const keyedDefinition = definition as Record< + string, + ((...args: any[]) => unknown) | undefined + >; + methodLoop: for (const key in keyedDefinition) { + if (!Object.hasOwn(keyedDefinition, key)) continue methodLoop; + const method = keyedDefinition[key]; if (method == null) continue methodLoop; const prev = instance[key]; // oxlint-disable-next-line typescript-eslint/no-explicit-any diff --git a/packages/trees/src/features/tree/feature.ts b/packages/trees/src/features/tree/feature.ts index 08860b529..be017dcf0 100644 --- a/packages/trees/src/features/tree/feature.ts +++ b/packages/trees/src/features/tree/feature.ts @@ -31,16 +31,22 @@ export const treeFeature: FeatureImplementation = { const { expandedItems } = tree.getState(); const flatItems: ItemMeta[] = []; const expandedItemsSet = new Set(expandedItems); // TODO support setting state expandedItems as set instead of array + const lineageSet = new Set([rootItemId]); + const lineageStack = [rootItemId]; + // Walks visible tree nodes while reusing a single lineage stack/set for + // circular-reference detection to avoid per-node path array allocations. const recursiveAdd = ( itemId: string, - path: string[], + parentId: string, level: number, setSize: number, posInSet: number ) => { - if (path.includes(itemId)) { - logWarning(`Circular reference for ${path.join('.')}`); + if (lineageSet.has(itemId)) { + logWarning( + `Circular reference for ${[...lineageStack, itemId].join('.')}` + ); return; } @@ -48,30 +54,32 @@ export const treeFeature: FeatureImplementation = { itemId, level, index: flatItems.length, - parentId: path.at(-1) as string, + parentId, setSize, posInSet, }); - if (expandedItemsSet.has(itemId)) { - const children = tree.retrieveChildrenIds(itemId) ?? []; - let i = 0; - for (const childId of children) { - recursiveAdd( - childId, - path.concat(itemId), - level + 1, - children.length, - i++ - ); - } + if (!expandedItemsSet.has(itemId)) { + return; + } + + lineageSet.add(itemId); + lineageStack.push(itemId); + + const children = tree.retrieveChildrenIds(itemId) ?? []; + for (let childIndex = 0; childIndex < children.length; childIndex++) { + const childId = children[childIndex]; + recursiveAdd(childId, itemId, level + 1, children.length, childIndex); } + + lineageStack.pop(); + lineageSet.delete(itemId); }; - const children = tree.retrieveChildrenIds(rootItemId); - let i = 0; - for (const itemId of children) { - recursiveAdd(itemId, [rootItemId], 0, children.length, i++); + const children = tree.retrieveChildrenIds(rootItemId) ?? []; + for (let childIndex = 0; childIndex < children.length; childIndex++) { + const itemId = children[childIndex]; + recursiveAdd(itemId, rootItemId, 0, children.length, childIndex); } return flatItems;