diff --git a/CHANGELOG.md b/CHANGELOG.md index 347449b..886edac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # test-runner - Changelog +## 3.4.0 - 2026-06-23 + +* Only re-run incomplete tests after a test run reset. + * When a run is cancelled and restarted due to lack of progress, results from classes that already completed are kept and only the unfinished classes are re-run, instead of re-running everything. Genuine failures are still re-run sequentially at the end. +* Write a per-reset diagnostic file (`-reset--.json`) alongside the other output, recording the queue item statuses and a results summary at the point of the reset. +* Abort and abandon a run that stalls before it starts processing (e.g. while stuck queued), rather than leaving it in-flight to be re-run. +* Test run status logging now shows a `No progress /` suffix while a run is stalling, and resets log the reuse summary and a `Reset /` count. +* `numberOfResets` in `TestRunSummary` now reports the total resets across all runs in a `Testall` invocation (including missing-test reruns), not just the outer run's count. + ## 3.3.1-beta.0 - 2026-06-14 * Attempt to abort Apex test runs after runner timeout while preserving the original timeout error if aborting fails. diff --git a/package.json b/package.json index 4d310d7..fffd458 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@apexdevtools/test-runner", - "version": "3.3.1-beta.0", + "version": "3.4.0", "description": "Apex parallel test runner with reliability goodness", "author": { "name": "Apex Dev Tools Team", diff --git a/src/command/Testall.ts b/src/command/Testall.ts index 92ffa21..86cc064 100644 --- a/src/command/Testall.ts +++ b/src/command/Testall.ts @@ -20,7 +20,7 @@ import { TestRerun, TestRunSummary, } from '../results/OutputGenerator'; -import { TestRunnerOptions } from '../runner/TestOptions'; +import { getOutputFileBase, TestRunnerOptions } from '../runner/TestOptions'; import { TestRunner } from '../runner/TestRunner'; import { formatTestName, getTestName } from '../results/TestResultUtils'; import { TestResultStore } from '../results/TestResultStore'; @@ -43,8 +43,6 @@ import { retry } from '../runner/Poll'; export interface TestallOptions extends TestRunnerOptions, QueryOptions { maxErrorsForReRun?: number; // Don't re-run if > failed tests (excluding pattern matched tests), default 10 - outputDirBase?: string; // Base for junit and other output files, default 'test-result*' - outputFileName?: string; //File name base disableCoverageReport?: boolean; // if enabled disables coverage collection rerunOption?: RerunOption; // see RerunOption - default 'pattern' } @@ -56,7 +54,6 @@ export enum RerunOption { } const DEFAULT_MAX_ERRORS_FOR_RERUN = 10; -const DEFAULT_OUTPUT_FILE_BASE = 'test-result'; const DEFAULT_RERUN_OPTION = RerunOption.Pattern; export function getMaxErrorsForReRun(options: TestallOptions): number { @@ -65,18 +62,6 @@ export function getMaxErrorsForReRun(options: TestallOptions): number { else return DEFAULT_MAX_ERRORS_FOR_RERUN; } -export function getOutputFileBase(options: TestallOptions): { - fileName: string; - outputDir: string; -} { - if (options.outputDirBase && options.outputFileName) - return { - outputDir: options.outputDirBase, - fileName: options.outputFileName, - }; - else return { outputDir: '', fileName: DEFAULT_OUTPUT_FILE_BASE }; -} - export function getReRunOption(options: TestallOptions): RerunOption { const opt = options.rerunOption; if (opt && Object.values(RerunOption).find(v => v === opt)) return opt; @@ -215,7 +200,6 @@ export class Testall { store ); - store.numberOfResets = result.numberOfResets; } } diff --git a/src/log/BaseLogger.ts b/src/log/BaseLogger.ts index 4ad14c5..1c1e4a6 100644 --- a/src/log/BaseLogger.ts +++ b/src/log/BaseLogger.ts @@ -141,10 +141,34 @@ export abstract class BaseLogger implements Logger { ); } + logRunStuck(testRunId: string, status: string): void { + this.logMessage( + `Test run '${testRunId}' stuck in ${status} with no progress, abandoning this attempt` + ); + } + + logResetCount(resetNumber: number, maxResets: number): void { + this.logMessage(`Reset ${resetNumber}/${maxResets} before abandoning run`); + } + + logRunReset( + reusedTests: number, + completedClasses: number, + remainingTests: number, + pendingClasses: number + ): void { + this.logMessage( + `Reusing ${reusedTests} tests from ${completedClasses} completed classes; ` + + `rerunning ${remainingTests} remaining tests across ${pendingClasses} classes` + ); + } + logStatus( testRunResult: ApexTestRunResult, tests: ApexTestResult[], - elapsedTime: string + elapsedTime: string, + noProgressPolls: number, + noProgressLimit: number ): void { const status = testRunResult.Status; const outcomes = groupByOutcome(tests); @@ -154,8 +178,15 @@ export abstract class BaseLogger implements Logger { const total = testRunResult.MethodsEnqueued; const complete = total > 0 ? Math.floor((completed * 100) / total) : 0; + // While a run is in flight, surface how close it is to a reset so a stall + // is visible without an extra log line per poll. + const noProgress = + noProgressPolls > 0 + ? ` | No progress ${noProgressPolls}/${noProgressLimit}` + : ''; + this.logMessage( - `${elapsedTime} [${status}] Passed: ${passed} | Failed: ${failed} | ${completed}/${total} Complete (${complete}%)` + `${elapsedTime} [${status}] Passed: ${passed} | Failed: ${failed} | ${completed}/${total} Complete (${complete}%)${noProgress}` ); } diff --git a/src/log/Logger.ts b/src/log/Logger.ts index 1455ff3..a72a684 100644 --- a/src/log/Logger.ts +++ b/src/log/Logger.ts @@ -35,10 +35,20 @@ export interface Logger { // Test runner logRunStarted(testRunId: string): void; logNoProgress(testRunId: string): void; + logRunStuck(testRunId: string, status: string): void; + logResetCount(resetNumber: number, maxResets: number): void; + logRunReset( + reusedTests: number, + completedClasses: number, + remainingTests: number, + pendingClasses: number + ): void; logStatus( status: ApexTestRunResult, tests: ApexTestResult[], - elapsedTime: string + elapsedTime: string, + noProgressPolls: number, + noProgressLimit: number ): void; logTestFailures(newResults: ApexTestResult[]): void; diff --git a/src/results/TestResultStore.ts b/src/results/TestResultStore.ts index f77aa97..4f1e9d6 100644 --- a/src/results/TestResultStore.ts +++ b/src/results/TestResultStore.ts @@ -45,7 +45,7 @@ export class TestResultStore { }); this.asyncError = res.error; - this.numberOfResets = res.numberOfResets; + this.numberOfResets += res.numberOfResets; } public saveSyncResult(reruns: TestRerun[]): void { diff --git a/src/runner/TestOptions.ts b/src/runner/TestOptions.ts index 51960d1..c4e2817 100644 --- a/src/runner/TestOptions.ts +++ b/src/runner/TestOptions.ts @@ -34,6 +34,7 @@ export function getCancelPollTimeout(options: CancelTestRunOptions): Duration { else return Duration.minutes(DEFAULT_POLL_TIMEOUT_MINS); } +export const DEFAULT_OUTPUT_FILE_BASE = 'test-result'; export const DEFAULT_STATUS_POLL_INTERVAL_MS = 30000; export const DEFAULT_TEST_RUN_TIMEOUT_MINS = 120; const DEFAULT_MAX_TEST_RUN_RETRIES = 3; @@ -65,6 +66,20 @@ export interface TestRunnerOptions extends CancelTestRunOptions { pollLimitToAssumeHangingTests?: number; // Number polls without test progress before a hang is assumed, default 60 callbacks?: TestRunnerCallbacks; // Callbacks for events in test runner codeCoverage?: boolean; // Collect code coverage data, defaults false + outputDirBase?: string; // Base directory for junit and other output files + outputFileName?: string; // File name base for output files, default 'test-result' +} + +export function getOutputFileBase(options: TestRunnerOptions): { + fileName: string; + outputDir: string; +} { + if (options.outputDirBase && options.outputFileName) + return { + outputDir: options.outputDirBase, + fileName: options.outputFileName, + }; + else return { outputDir: '', fileName: DEFAULT_OUTPUT_FILE_BASE }; } export function getTestRunAborter(options: TestRunnerOptions): TestRunAborter { diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index 9f34fb8..b78ad83 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -18,18 +18,21 @@ import { } from '../model/ApexTestRunResult'; import { getMaxTestRunRetries, + getOutputFileBase, getStatusPollInterval, getTestRunAborter, getTestRunTimeout, getTestRunTimeoutMessage, TestRunnerOptions, } from './TestOptions'; +import path from 'path'; import TestStats from './TestStats'; import { QueryHelper } from '../query/QueryHelper'; -import { ApexTestQueueItem } from '../model/ApexTestQueueItem'; +import { ApexTestQueueItem, QueueItemStatus } from '../model/ApexTestQueueItem'; import { TestError, TestErrorKind } from './TestError'; import { Pollable, poll, retry } from './Poll'; import { ApexTestResult, ApexTestResultFields } from '../model/ApexTestResult'; +import { getTestName, groupByOutcome } from '../results/TestResultUtils'; /** * Parallel unit test runner that includes the ability to cancel & restart a run should it not make sufficient progress @@ -47,6 +50,20 @@ export interface TestRunnerResult { numberOfResets: number; // Track the number of times the test run has been reset due to hanging or cancellation } +// Queue item statuses for tests that have not finished running. After a reset +// these are the classes we re-run; classes that finished keep their results +// (mirrors the statuses the aborter cancels). Note this is only the async +// re-run - failed tests in finished classes are still re-run afterwards by +// Testall.syncRun, which applies the configurable rerun filter and runs them +// sequentially to avoid the row-lock contention that may have caused +// them. +const PENDING_QUEUE_STATUSES: QueueItemStatus[] = [ + 'Holding', + 'Queued', + 'Preparing', + 'Processing', +]; + export interface TestRunner { getTestClasses(): string[]; run(token?: CancellationToken): Promise; @@ -68,6 +85,9 @@ export class AsyncTestRunner implements TestRunner { private readonly _options: TestRunnerOptions; private readonly _testService: TestService; private _stats: TestStats; + // Results from classes that finished before a reset, kept so they are not + // re-run. Keyed by full test name and reset at the start of each run(). + private _completedResults: Map = new Map(); static forClasses( logger: Logger, @@ -118,6 +138,20 @@ export class AsyncTestRunner implements TestRunner { } public async run(token?: CancellationToken): Promise { + this._completedResults = new Map(); + const result = await this.runInternal(token); + return this.mergeCompletedResults(result); + } + + /** + * Runs (or re-runs) the tests. On a restart following a hang, `restartItems` + * carries the subset of tests still to run so we don't re-run ones that have + * already completed. + */ + private async runInternal( + token?: CancellationToken, + restartItems?: TestItem[] + ): Promise { if (this.hasHitMaxNumberOfTestRunRetries()) { throw new TestError( `Max number of test run retries reached, max allowed retries: ${getMaxTestRunRetries( @@ -127,7 +161,9 @@ export class AsyncTestRunner implements TestRunner { ); } - const payload = this.testClassPayload() || (await this.testAllPayload()); + const payload = restartItems + ? this.getSpecifiedTestsPayload(restartItems) + : this.getTestClassPayload() || (await this.getTestAllPayload()); const testRunIdResult = (await retry( () => this._testService.runTestAsynchronous(payload, false, true), @@ -159,9 +195,38 @@ export class AsyncTestRunner implements TestRunner { if (result.run.Status == 'Processing' && this._stats.isTestRunHanging()) { this._logger.logNoProgress(testRunIdResult.testRunId); + + // Only work out what to re-run if we have a reset left. Once retries are + // exhausted the recursive run will abandon (throw), so skip the + // prepareRestart work to avoid a misleading "rerunning" log, a wasted + // queue query and snapshot. + const resetNumber = this._stats.getNumberOfTimesReset() + 1; + const maxResets = getMaxTestRunRetries(this._options) - 1; + let restartItems: TestItem[] | undefined; + if (resetNumber <= maxResets) { + this._logger.logResetCount(resetNumber, maxResets); + restartItems = await this.prepareRestart( + testRunIdResult.testRunId, + result + ); + } + this._stats = this._stats.reset(); await this.abortTestRun(result.run.AsyncApexJobId); - return await this.run(token); + return await this.runInternal(token, restartItems); + } + + if ( + !this.hasTestRunComplete(result.run.Status) && + this._stats.isTestRunHanging() + ) { + // Hung before it started processing (e.g. stuck in Queued). Re-running + // would likely just stall again, and leaving it in-flight would block + // the caller from re-running the missing tests (ALREADY_IN_PROCESS), so + // abort it and give up on this attempt. + this._logger.logRunStuck(testRunIdResult.testRunId, result.run.Status); + await this.abortTestRun(result.run.AsyncApexJobId); + return result; } } catch (err) { // error may already be defined from wait() @@ -179,20 +244,211 @@ export class AsyncTestRunner implements TestRunner { return this._stats.getNumberOfTimesReset() > maxNumberOfResets; } - private testClassPayload(): null | AsyncTestArrayConfiguration { - if (this._testItems.length > 0) { - const config: AsyncTestArrayConfiguration = { - tests: this._testItems, - testLevel: TestLevel.RunSpecifiedTests, - skipCodeCoverage: this.skipCollectCoverage(), - }; - return config; - } else { - return null; + /** + * After a hang, work out which classes still need running so the restart only + * re-runs those. Results from classes that already finished are kept (in + * _completedResults) rather than thrown away and re-run. Returns the test + * items to re-run, or undefined to fall back to a full re-run (e.g. if we + * can't determine progress) so a reset is never worse than before. + */ + private async prepareRestart( + testRunId: string, + result: TestRunnerResult + ): Promise { + try { + const queueItems = await this.getQueueItems(testRunId); + + const pendingClassIds = this.uniqueClassIds( + queueItems.filter(item => PENDING_QUEUE_STATUSES.includes(item.Status)) + ); + const completedClassIds = new Set( + this.uniqueClassIds( + queueItems.filter( + item => !PENDING_QUEUE_STATUSES.includes(item.Status) + ) + ) + ); + + // Keep results from classes that finished; drop the rest so the restart + // re-runs them cleanly. + const completedResults = result.tests.filter(test => + completedClassIds.has(test.ApexClass.Id) + ); + completedResults.forEach(test => + this._completedResults.set(getTestName(test), test) + ); + + // Keep a per-reset diagnostic snapshot of what the org looked like when + // the run stalled, so resets can be investigated after the fact. + this.writeResetSnapshot( + testRunId, + queueItems, + completedClassIds, + pendingClassIds, + result + ); + + if (pendingClassIds.length === 0) { + // Nothing identified to re-run - fall back to a full re-run. + return undefined; + } + + // Tests still to run = those enqueued this attempt that weren't in a + // completed class (i.e. the pending classes' methods). + const remainingTests = + result.run.MethodsEnqueued - completedResults.length; + this._logger.logRunReset( + this._completedResults.size, + completedClassIds.size, + remainingTests, + pendingClassIds.length + ); + + return this.buildRestartItems(pendingClassIds); + } catch (err) { + // Be defensive: a reset should never be worse than a full re-run. + this._logger.logWarning( + `Could not determine completed tests for restart, re-running all: ${ + TestError.wrapError(err).message + }` + ); + return undefined; + } + } + + private async getQueueItems(testRunId: string): Promise { + return this._queryHelper.query( + 'ApexTestQueueItem', + `ParentJobId='${testRunId}'`, + 'Id, ApexClassId, Status' + ); + } + + private uniqueClassIds(queueItems: ApexTestQueueItem[]): string[] { + return Array.from(new Set(queueItems.map(item => item.ApexClassId))); + } + + /** + * Build the restart TestItems for pending classes. For method-specific + * runners (e.g. a missing-test rerun), re-use the original items unchanged + * so a reset doesn't widen a method-filtered run to the whole class. + * For all-local / class-level runs, restart at class granularity. + */ + private buildRestartItems(pendingClassIds: string[]): TestItem[] { + if (this._testItems.some(item => item.testMethods?.length)) { + return this._testItems; + } + return pendingClassIds.map(classId => ({ classId })); + } + + private writeResetSnapshot( + testRunId: string, + queueItems: ApexTestQueueItem[], + completedClassIds: Set, + pendingClassIds: string[], + result: TestRunnerResult + ): void { + const resetNumber = this._stats.getNumberOfTimesReset() + 1; + const outcomes = groupByOutcome(result.tests); + + // Resolve class names from the results we already have (no extra org call). + // Classes that produced no results won't have a name available. + const classNames = new Map(); + result.tests.forEach(test => + classNames.set(test.ApexClass.Id, test.ApexClass.Name) + ); + + const snapshot = { + testRunId, + resetNumber, + completedClasses: completedClassIds.size, + pendingClasses: pendingClassIds.length, + reusedTests: this._completedResults.size, + results: { + total: result.tests.length, + passed: outcomes.Pass.length, + failed: outcomes.Fail.length + outcomes.CompileFail.length, + skipped: outcomes.Skip.length, + }, + // Slimmed down (drop the jsforce attributes) and annotated with the + // class name where we know it. + queueItems: queueItems.map(item => ({ + ApexClassId: item.ApexClassId, + className: classNames.get(item.ApexClassId), + Status: item.Status, + })), + }; + // Write alongside the other output files, using the same dir/name prefix. + const { outputDir, fileName } = getOutputFileBase(this._options); + this._logger.logOutputFile( + path.join( + outputDir, + `${fileName}-reset-${resetNumber}-${testRunId}.json` + ), + JSON.stringify(snapshot, undefined, 2) + ); + } + + /** + * Merge the results kept from completed classes with the final attempt's + * results so the returned result covers every test that ran across attempts. + * The run-level counts (which otherwise only reflect the final restart + * subset) are recomputed from the merged set so the summary stays consistent. + */ + private mergeCompletedResults(result: TestRunnerResult): TestRunnerResult { + if (this._completedResults.size === 0) { + return result; } + + const merged = new Map(this._completedResults); + result.tests.forEach(test => merged.set(getTestName(test), test)); + const tests = Array.from(merged.values()); + + return { + ...result, + tests, + run: this.recomputeRunCounts(result.run, tests), + }; } - private async testAllPayload(): Promise< + private recomputeRunCounts( + run: ApexTestRunResult, + tests: ApexTestResult[] + ): ApexTestRunResult { + const classes = new Set(tests.map(test => test.ApexClass.Id)); + const failed = tests.filter( + test => test.Outcome !== 'Pass' && test.Outcome !== 'Skip' + ).length; + const testTime = tests.reduce((total, test) => total + test.RunTime, 0); + + return { + ...run, + ClassesCompleted: classes.size, + ClassesEnqueued: classes.size, + MethodsCompleted: tests.length, + MethodsEnqueued: tests.length, + MethodsFailed: failed, + TestTime: testTime, + }; + } + + private getTestClassPayload(): null | AsyncTestArrayConfiguration { + return this._testItems.length > 0 + ? this.getSpecifiedTestsPayload(this._testItems) + : null; + } + + private getSpecifiedTestsPayload( + testItems: TestItem[] + ): AsyncTestArrayConfiguration { + return { + tests: testItems, + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: this.skipCollectCoverage(), + }; + } + + private async getTestAllPayload(): Promise< AsyncTestConfiguration | AsyncTestArrayConfiguration > { const payload = await retry( @@ -284,9 +540,20 @@ export class AsyncTestRunner implements TestRunner { results: ApexTestResult[], time: string ): Promise { - this._logger.logStatus(testRunResult, results, time); this._stats = this._stats.update(results.length); + // No-progress count is only meaningful while the run is still in flight. + const noProgressPolls = this.hasTestRunComplete(testRunResult.Status) + ? 0 + : this._stats.getNoProgressPollCount(); + this._logger.logStatus( + testRunResult, + results, + time, + noProgressPolls, + this._stats.getNoProgressPollLimit() + ); + if (this._logger.verbose) { await this.reportQueueItems(testRunResult.AsyncApexJobId); } diff --git a/src/runner/TestStats.ts b/src/runner/TestStats.ts index edadfb7..d8a7600 100644 --- a/src/runner/TestStats.ts +++ b/src/runner/TestStats.ts @@ -66,4 +66,12 @@ export default class TestStats { public getNumberOfTimesReset(): number { return this._numberOfTimesReset; } + + public getNoProgressPollCount(): number { + return this._numberOfTimesUpdatedWithoutChange; + } + + public getNoProgressPollLimit(): number { + return this._updateLimitToAssumeTestHanging; + } } diff --git a/test/command/Testall.spec.ts b/test/command/Testall.spec.ts index a20023d..6871cf7 100644 --- a/test/command/Testall.spec.ts +++ b/test/command/Testall.spec.ts @@ -117,7 +117,7 @@ describe('TestAll', () => { [new MockOutputGenerator()], {} ); - } catch (err) { + } catch { // Ignore } @@ -152,7 +152,7 @@ describe('TestAll', () => { [new MockOutputGenerator()], {} ); - } catch (err) { + } catch { // Ignore } @@ -713,4 +713,53 @@ describe('TestAll', () => { logRegex('Generated reports for 2 tests') ); }); + + it('should accumulate numberOfResets across outer run and missing-test rerun', async () => { + // Outer run has 2 resets; the missing-test rerun has 1 reset. + // The summary should report 3, not just the outer count. + const logger = new CapturingLogger(); + const mockRunResult: ApexTestRunResult = createMockRunResult(); + const classId = 'classId1'; + const className = 'TestClass'; + const mockTestResults = [ + createMockTestResult({ + MethodName: 'method1', + ApexClass: { Id: classId, Name: className, NamespacePrefix: null }, + }), + createMockTestResult({ + MethodName: 'method2', + ApexClass: { Id: classId, Name: className, NamespacePrefix: null }, + }), + ]; + const runner = new MockTestRunner({ + run: mockRunResult, + tests: [mockTestResults[0]], + numberOfResets: 2, + }).addNextResult({ + run: mockRunResult, + tests: [mockTestResults[1]], + numberOfResets: 1, + }); + const testMethods = new MockTestMethodCollector( + logger, + mockConnection, + '', + new Map([[classId, className]]), + new Map>([ + [className, new Set(['method1', 'method2'])], + ]) + ); + + const result = await Testall.run( + logger, + mockConnection, + '', + testMethods, + runner, + [new MockOutputGenerator()], + {} + ); + + expect(result.numberOfResets).to.equal(3); + }); }); diff --git a/test/log/BaseLogger.spec.ts b/test/log/BaseLogger.spec.ts index 723c978..9b3aa72 100644 --- a/test/log/BaseLogger.spec.ts +++ b/test/log/BaseLogger.spec.ts @@ -30,7 +30,9 @@ describe('BaseLogger', () => { logger.logStatus( mockTestRunResult as ApexTestRunResult, mockTestResults as ApexTestResult[], - '' + '', + 0, + 5 ); expect(logger.entries.length).to.equal(1); @@ -41,6 +43,31 @@ describe('BaseLogger', () => { ); }); + it('should append a no-progress count to the status line when stalled', () => { + const mockTestRunResult: Partial = { + Status: 'Processing', + MethodsEnqueued: 6, + }; + const mockTestResults: Partial[] = [{ Outcome: 'Pass' }]; + + const logger = new CapturingLogger(); + logger.logStatus( + mockTestRunResult as ApexTestRunResult, + mockTestResults as ApexTestResult[], + '', + 3, + 5 + ); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex( + '\\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1\\/6 Complete \\(16%\\)' + + ' \\| No progress 3/5' + ) + ); + }); + it('should output file content with filename', () => { const logger = new CapturingLogger('/log/path', true); const content = '{ "records": [] }'; @@ -96,6 +123,42 @@ describe('BaseLogger', () => { expect(logger.files[0][1]).to.equal(content); }); + it('should log a run reset summary', () => { + const logger = new CapturingLogger(); + logger.logRunReset(14000, 320, 6000, 140); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex( + 'Reusing 14000 tests from 320 completed classes; ' + + 'rerunning 6000 remaining tests across 140 classes' + ) + ); + }); + + it('should log the reset count', () => { + const logger = new CapturingLogger(); + logger.logResetCount(1, 2); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex('Reset 1/2 before abandoning run') + ); + }); + + it('should log when a run is abandoned while stuck', () => { + const logger = new CapturingLogger(); + logger.logRunStuck('707xx', 'Queued'); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex( + "Test run '707xx' stuck in Queued with no progress, " + + 'abandoning this attempt' + ) + ); + }); + it('should generate warning messages', () => { const logger = new CapturingLogger('', true); logger.logWarning('A message'); diff --git a/test/runner/TestOptions.spec.ts b/test/runner/TestOptions.spec.ts new file mode 100644 index 0000000..90cd7b4 --- /dev/null +++ b/test/runner/TestOptions.spec.ts @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026, Certinia Inc. All rights reserved. + */ + +import { expect } from 'chai'; +import { getOutputFileBase } from '../../src/runner/TestOptions'; + +describe('TestOptions', () => { + describe('getOutputFileBase', () => { + it('should use the supplied dir and file name when both are set', () => { + expect( + getOutputFileBase({ + outputDirBase: 'test-results/apex', + outputFileName: 'reset', + }) + ).to.deep.equal({ outputDir: 'test-results/apex', fileName: 'reset' }); + }); + + it('should default when neither is set', () => { + expect(getOutputFileBase({})).to.deep.equal({ + outputDir: '', + fileName: 'test-result', + }); + }); + + it('should default when only the dir is set', () => { + expect( + getOutputFileBase({ outputDirBase: 'test-results/apex' }) + ).to.deep.equal({ outputDir: '', fileName: 'test-result' }); + }); + + it('should default when only the file name is set', () => { + expect(getOutputFileBase({ outputFileName: 'jim' })).to.deep.equal({ + outputDir: '', + fileName: 'test-result', + }); + }); + }); +}); diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index de6b051..341516d 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -265,6 +265,49 @@ describe('TestRunner', () => { expect(error.kind).to.equal(TestErrorKind.Timeout); }); + it('should abort and abandon without re-running when retries are exhausted', async () => { + // One retry allowed. The run stalls, exhausting it - so we abort and + // abandon without pretending to re-run (no reset/reusing logs, no extra + // queue query). + qhStub.query + .withArgs('ApexTestResult', match.any, match.any) + .resolves([mockTestResult[0]]); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + ]); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 1, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } + ); + + // Exhausting retries surfaces as a result error (caught for partial + // reporting), not a throw. + const testRunResult = await runner.run(); + expect(testRunResult.error).to.be.instanceof(TestError); + expect(testRunResult.error?.message).to.equal( + 'Max number of test run retries reached, max allowed retries: 1' + ); + // The stalled run was aborted, and not re-run + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledOnce).to.be.true; + // No queue query, no reset/reusing logs - we did not pretend to re-run + expect(qhStub.query.calledWith('ApexTestQueueItem', match.any, match.any)) + .to.be.false; + expect(logger.entries.some(e => /Reusing/.test(e))).to.be.false; + expect(logger.entries.some(e => /Reset \d/.test(e))).to.be.false; + }); + it('should throw on timeout if no results found', async () => { qhStub.query.resolves([]); @@ -422,7 +465,24 @@ describe('TestRunner', () => { expect(error.kind).to.equal(TestErrorKind.Timeout); }); - it('should cancel and restart after no progress detected', async () => { + it('should cancel and re-run only incomplete classes after no progress detected', async () => { + // Class1 finishes (its pass is kept); Class3 is still running when the run + // stalls, so only Class3 should be re-run after the reset. + qhStub.query + .withArgs('ApexTestResult', match.any, match.any) + .onCall(0) + .resolves([mockTestResult[0]]) // Class1 pass + .onCall(1) + .resolves([mockTestResult[0]]) // no progress -> hang + .onCall(2) + .resolves([mockTestResult[1]]); // re-run produces Class3 result + const queueItems = [ + { Id: 'q1', ApexClassId: 'Class1', Status: 'Completed' }, + { Id: 'q3', ApexClassId: 'Class3', Status: 'Processing' }, + ]; + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .resolves(queueItems); setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ { Status: 'Processing' }, { Status: 'Processing' }, @@ -459,30 +519,52 @@ describe('TestRunner', () => { expect(mockAborter.calls).to.equal(1); expect(testServiceAsyncStub.calledTwice).to.be.true; + // The re-run only asks for the class that had not completed + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [{ classId: 'Class3' }], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); expect(testRunResult.numberOfResets).to.equal(1); + // Kept Class1's result and added Class3's from the re-run + expect(testRunResult.tests.map(t => t.Id).sort()).to.deep.equal([ + 'test1', + 'test2', + ]); + // Run-level counts reflect the merged set, not just the re-run subset + expect(testRunResult.run.MethodsCompleted).to.equal(2); + expect(testRunResult.run.MethodsEnqueued).to.equal(2); + expect(testRunResult.run.MethodsFailed).to.equal(1); + expect(testRunResult.run.ClassesCompleted).to.equal(2); + expect(testRunResult.run.TestTime).to.equal(30); // 10 + 20 expect(logger.entries.length).to.equal(10); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) ); expect(logger.entries[1]).to.match( logRegex( - `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 1 \\| 2/2 Complete \\(100%\\)` + `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1/2 Complete \\(50%\\)` ) ); - expect(logger.entries[2]).to.match(logRegex("Failing tests in 'Class3':")); - expect(logger.entries[3]).to.match( - logRegex('\\* Method2 - Exception: Test Failed') + expect(logger.entries[2]).to.match( + logRegex( + `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1/2 Complete \\(50%\\) \\| No progress 1/1` + ) ); - expect(logger.entries[4]).to.match( + expect(logger.entries[3]).to.match( logRegex( - `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 1 \\| 2/2 Complete \\(100%\\)` + `Test run '${testRunId}' was not progressing, cancelling and retrying...` ) ); + expect(logger.entries[4]).to.match( + logRegex('Reset 1/1 before abandoning run') + ); expect(logger.entries[5]).to.match( logRegex( - `Test run '${testRunId}' was not progressing, cancelling and retrying...` + 'Reusing 1 tests from 1 completed classes; ' + + 'rerunning 1 remaining tests across 1 classes' ) ); expect(logger.entries[6]).to.match( @@ -490,9 +572,250 @@ describe('TestRunner', () => { ); expect(logger.entries[7]).to.match( logRegex( - `${timeFormat} \\[Completed\\] Passed: 1 \\| Failed: 1 \\| 2/2 Complete \\(100%\\)` + `${timeFormat} \\[Completed\\] Passed: 0 \\| Failed: 1 \\| 1/2 Complete \\(50%\\)` + ) + ); + expect(logger.entries[8]).to.match(logRegex("Failing tests in 'Class3':")); + expect(logger.entries[9]).to.match( + logRegex('\\* Method2 - Exception: Test Failed') + ); + // A per-reset diagnostic snapshot is written + expect(logger.files.length).to.equal(1); + expect(logger.files[0][0]).to.equal( + `${process.cwd()}/test-result-reset-1-${testRunId}.json` + ); + const snapshot = JSON.parse(logger.files[0][1]) as { + resetNumber: number; + completedClasses: number; + pendingClasses: number; + reusedTests: number; + queueItems: unknown[]; + }; + expect(snapshot.resetNumber).to.equal(1); + expect(snapshot.completedClasses).to.equal(1); + expect(snapshot.pendingClasses).to.equal(1); + expect(snapshot.reusedTests).to.equal(1); + expect(snapshot.queueItems).to.have.length(2); + }); + + it('should narrow restart to originally-requested methods for method-specific runners', async () => { + // The runner was created with specific methods (e.g. a missing-test rerun). + // Class3 is still pending. The restart should only re-run the originally + // requested methods of Class3, not widen to the whole class. + qhStub.query + .withArgs('ApexTestResult', match.any, match.any) + .onCall(0) + .resolves([mockTestResult[0]]) // Class1 pass + .onCall(1) + .resolves([mockTestResult[0]]) // no progress → hang + .onCall(2) + .resolves([mockTestResult[1]]); // re-run produces Class3 result + const queueItems = [ + { Id: 'q1', ApexClassId: 'Class1', Status: 'Completed' }, + { Id: 'q3', ApexClassId: 'Class3', Status: 'Processing' }, + ]; + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .resolves(queueItems); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + { Status: 'Completed' }, + ]); + setupExecuteAnonymous( + sandbox.stub(ExecuteService.prototype, 'connectionRequest'), + { + column: -1, + line: -1, + compiled: 'true', + compileProblem: '', + exceptionMessage: '', + exceptionStackTrace: '', + success: 'true', + } + ); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + // Method-specific runner — only Method2 of Class3 was requested + const runner = new AsyncTestRunner(logger, mockConnection, [ + { className: 'Class1' }, + { classId: 'Class3', testMethods: ['Method2'] }, + ], { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + }); + + await runner.run(); + + // Restart uses original _testItems — method scope is preserved, not widened to whole class + expect(testServiceAsyncStub.calledTwice).to.be.true; + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [ + { className: 'Class1' }, + { classId: 'Class3', testMethods: ['Method2'] }, + ], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); + }); + + it('should fall back to a full re-run when no incomplete classes can be identified', async () => { + // No queue items returned -> we cannot tell what is incomplete, so re-run + // everything rather than risk dropping tests. + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .resolves([]); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + { Status: 'Completed' }, + ]); + setupExecuteAnonymous( + sandbox.stub(ExecuteService.prototype, 'connectionRequest'), + { + column: -1, + line: -1, + compiled: 'true', + compileProblem: '', + exceptionMessage: '', + exceptionStackTrace: '', + success: 'true', + } + ); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } + ); + + const testRunResult = await runner.run(); + + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledTwice).to.be.true; + // Re-run uses the original full payload, not a pending subset + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [{ className: 'TestSample', namespace: undefined }], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); + expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(1); + // No reset reuse summary was logged + expect(logger.entries.some(e => /Reusing/.test(e))).to.be.false; + }); + + it('should fall back to a full re-run when the queue cannot be queried', async () => { + // The query for incomplete classes fails, so we must not drop tests - fall + // back to re-running everything and warn. + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .rejects(new Error('query boom')); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + { Status: 'Completed' }, + ]); + setupExecuteAnonymous( + sandbox.stub(ExecuteService.prototype, 'connectionRequest'), + { + column: -1, + line: -1, + compiled: 'true', + compileProblem: '', + exceptionMessage: '', + exceptionStackTrace: '', + success: 'true', + } + ); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } + ); + + const testRunResult = await runner.run(); + + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledTwice).to.be.true; + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [{ className: 'TestSample', namespace: undefined }], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); + expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(1); + expect(logger.entries.some(e => /Reusing/.test(e))).to.be.false; + expect( + logger.entries.some(e => + logRegex( + 'Warning: Could not determine completed tests for restart, ' + + 're-running all: query boom' + ).test(e) ) + ).to.be.true; + // No snapshot written when we could not inspect the queue + expect(logger.files.length).to.equal(0); + }); + + it('should abort and give up when a run is stuck before processing', async () => { + // Run never leaves 'Queued' and makes no progress. It should be aborted and + // abandoned (not reset/retried) so its tests aren't left enqueued for the + // caller's missing-test re-run to collide with. + qhStub.query.withArgs('ApexTestResult', match.any, match.any).resolves([]); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Queued' }, + { Status: 'Queued' }, + ]); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } ); + + const testRunResult = await runner.run(); + + // Aborted once, and not restarted + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledOnce).to.be.true; + expect(testRunResult.run.Status).to.equal('Queued'); + expect(testRunResult.numberOfResets).to.equal(0); + expect( + logger.entries.some(e => + logRegex( + `Test run '${testRunId}' stuck in Queued with no progress, ` + + 'abandoning this attempt' + ).test(e) + ) + ).to.be.true; }); it('should not cancel if progress detected', async () => {