From 1b75418632db912b0126512b549882bcaea1abca Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Mon, 15 Jun 2026 13:54:48 -0400 Subject: [PATCH 01/12] Add logging method for runReset info --- src/log/BaseLogger.ts | 12 ++++++++++++ src/log/Logger.ts | 6 ++++++ test/log/BaseLogger.spec.ts | 13 +++++++++++++ 3 files changed, 31 insertions(+) diff --git a/src/log/BaseLogger.ts b/src/log/BaseLogger.ts index ef8c80f..ae71dfe 100644 --- a/src/log/BaseLogger.ts +++ b/src/log/BaseLogger.ts @@ -141,6 +141,18 @@ export abstract class BaseLogger implements Logger { ); } + logRunReset( + reusedTests: number, + completedClasses: number, + remainingTests: number, + pendingClasses: number + ): void { + this.logMessage( + `Reusing ${reusedTests} tests from ${completedClasses} completed classes; ` + + `rerunning ${remainingTests} remaining tests across ${pendingClasses} classes` + ); + } + logStatus( testRunResult: ApexTestRunResult, tests: ApexTestResult[], diff --git a/src/log/Logger.ts b/src/log/Logger.ts index 1455ff3..0333cf0 100644 --- a/src/log/Logger.ts +++ b/src/log/Logger.ts @@ -35,6 +35,12 @@ export interface Logger { // Test runner logRunStarted(testRunId: string): void; logNoProgress(testRunId: string): void; + logRunReset( + reusedTests: number, + completedClasses: number, + remainingTests: number, + pendingClasses: number + ): void; logStatus( status: ApexTestRunResult, tests: ApexTestResult[], diff --git a/test/log/BaseLogger.spec.ts b/test/log/BaseLogger.spec.ts index 723c978..1d4a571 100644 --- a/test/log/BaseLogger.spec.ts +++ b/test/log/BaseLogger.spec.ts @@ -96,6 +96,19 @@ describe('BaseLogger', () => { expect(logger.files[0][1]).to.equal(content); }); + it('should log a run reset summary', () => { + const logger = new CapturingLogger(); + logger.logRunReset(14000, 320, 6000, 140); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex( + 'Reusing 14000 tests from 320 completed classes; ' + + 'rerunning 6000 remaining tests across 140 classes' + ) + ); + }); + it('should generate warning messages', () => { const logger = new CapturingLogger('', true); logger.logWarning('A message'); From baeb73b574b215b3158a2b32dc95f40e729d8b1a Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Mon, 15 Jun 2026 14:13:53 -0400 Subject: [PATCH 02/12] Refactor to allow for specifying tests to rerun after a reset --- src/runner/TestRunner.ts | 45 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index 50b88c7..363a81d 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -118,6 +118,18 @@ export class AsyncTestRunner implements TestRunner { } public async run(token?: CancellationToken): Promise { + return this.runInternal(token); + } + + /** + * Runs (or re-runs) the tests. On a restart following a hang, `restartItems` + * carries the subset of tests still to run so we don't re-run ones that have + * already completed. + */ + private async runInternal( + token?: CancellationToken, + restartItems?: TestItem[] + ): Promise { if (this.hasHitMaxNumberOfTestRunRetries()) { throw new TestError( `Max number of test run retries reached, max allowed retries: ${getMaxTestRunRetries( @@ -127,7 +139,9 @@ export class AsyncTestRunner implements TestRunner { ); } - const payload = this.testClassPayload() || (await this.testAllPayload()); + const payload = restartItems + ? this.getSpecifiedTestsPayload(restartItems) + : this.getTestClassPayload() || (await this.getTestAllPayload()); const testRunIdResult = (await retry( () => this._testService.runTestAsynchronous(payload, false, true), @@ -156,7 +170,7 @@ export class AsyncTestRunner implements TestRunner { this._logger.logNoProgress(testRunIdResult.testRunId); this._stats = this._stats.reset(); await this.abortTestRun(result.run.AsyncApexJobId); - return await this.run(token); + return await this.runInternal(token); } } catch (err) { // error may already be defined from wait() @@ -174,20 +188,23 @@ export class AsyncTestRunner implements TestRunner { return this._stats.getNumberOfTimesReset() > maxNumberOfResets; } - private testClassPayload(): null | AsyncTestArrayConfiguration { - if (this._testItems.length > 0) { - const config: AsyncTestArrayConfiguration = { - tests: this._testItems, - testLevel: TestLevel.RunSpecifiedTests, - skipCodeCoverage: this.skipCollectCoverage(), - }; - return config; - } else { - return null; - } + private getTestClassPayload(): null | AsyncTestArrayConfiguration { + return this._testItems.length > 0 + ? this.getSpecifiedTestsPayload(this._testItems) + : null; + } + + private getSpecifiedTestsPayload( + testItems: TestItem[] + ): AsyncTestArrayConfiguration { + return { + tests: testItems, + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: this.skipCollectCoverage(), + }; } - private async testAllPayload(): Promise< + private async getTestAllPayload(): Promise< AsyncTestConfiguration | AsyncTestArrayConfiguration > { const payload = await retry( From bad3f439139c557e7201df294cfed4f6682dce81 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Mon, 15 Jun 2026 14:49:27 -0400 Subject: [PATCH 03/12] Find passed test classes and pass exclude from the next test run --- src/runner/TestRunner.ts | 114 ++++++++++++++++++++++++++++++++- test/runner/TestRunner.spec.ts | 114 +++++++++++++++++++++++++++++---- 2 files changed, 212 insertions(+), 16 deletions(-) diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index 363a81d..ca1fa55 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -26,10 +26,11 @@ import { } from './TestOptions'; import TestStats from './TestStats'; import { QueryHelper } from '../query/QueryHelper'; -import { ApexTestQueueItem } from '../model/ApexTestQueueItem'; +import { ApexTestQueueItem, QueueItemStatus } from '../model/ApexTestQueueItem'; import { TestError, TestErrorKind } from './TestError'; import { Pollable, poll, retry } from './Poll'; import { ApexTestResult, ApexTestResultFields } from '../model/ApexTestResult'; +import { getTestName } from '../results/TestResultUtils'; /** * Parallel unit test runner that includes the ability to cancel & restart a run should it not make sufficient progress @@ -47,6 +48,20 @@ export interface TestRunnerResult { numberOfResets: number; // Track the number of times the test run has been reset due to hanging or cancellation } +// Queue item statuses for tests that have not finished running. After a reset +// these are the classes we re-run; classes that finished keep their results +// (mirrors the statuses the aborter cancels). Note this is only the async +// re-run - failed tests in finished classes are still re-run afterwards by +// Testall.syncRun, which applies the configurable rerun filter and runs them +// sequentially to avoid the row-lock contention that may have caused +// them. +const PENDING_QUEUE_STATUSES: QueueItemStatus[] = [ + 'Holding', + 'Queued', + 'Preparing', + 'Processing', +]; + export interface TestRunner { getTestClasses(): string[]; run(token?: CancellationToken): Promise; @@ -68,6 +83,9 @@ export class AsyncTestRunner implements TestRunner { private readonly _options: TestRunnerOptions; private readonly _testService: TestService; private _stats: TestStats; + // Results from classes that finished before a reset, kept so they are not + // re-run. Keyed by full test name and reset at the start of each run(). + private _completedResults: Map = new Map(); static forClasses( logger: Logger, @@ -118,7 +136,9 @@ export class AsyncTestRunner implements TestRunner { } public async run(token?: CancellationToken): Promise { - return this.runInternal(token); + this._completedResults = new Map(); + const result = await this.runInternal(token); + return this.mergeCompletedResults(result); } /** @@ -168,9 +188,13 @@ export class AsyncTestRunner implements TestRunner { if (result.run.Status == 'Processing' && this._stats.isTestRunHanging()) { this._logger.logNoProgress(testRunIdResult.testRunId); + const restartItems = await this.prepareRestart( + testRunIdResult.testRunId, + result + ); this._stats = this._stats.reset(); await this.abortTestRun(result.run.AsyncApexJobId); - return await this.runInternal(token); + return await this.runInternal(token, restartItems); } } catch (err) { // error may already be defined from wait() @@ -188,6 +212,90 @@ export class AsyncTestRunner implements TestRunner { return this._stats.getNumberOfTimesReset() > maxNumberOfResets; } + /** + * After a hang, work out which classes still need running so the restart only + * re-runs those. Results from classes that already finished are kept (in + * _completedResults) rather than thrown away and re-run. Returns the test + * items to re-run, or undefined to fall back to a full re-run (e.g. if we + * can't determine progress) so a reset is never worse than before. + */ + private async prepareRestart( + testRunId: string, + result: TestRunnerResult + ): Promise { + try { + const queueItems = await this.getQueueItems(testRunId); + + const pendingClassIds = this.uniqueClassIds( + queueItems.filter(item => PENDING_QUEUE_STATUSES.includes(item.Status)) + ); + const completedClassIds = new Set( + this.uniqueClassIds( + queueItems.filter( + item => !PENDING_QUEUE_STATUSES.includes(item.Status) + ) + ) + ); + + // Keep results from classes that finished; drop the rest so the restart + // re-runs them cleanly. + result.tests + .filter(test => completedClassIds.has(test.ApexClass.Id)) + .forEach(test => this._completedResults.set(getTestName(test), test)); + + if (pendingClassIds.length === 0) { + // Nothing identified to re-run - fall back to a full re-run. + return undefined; + } + + const remainingTests = + result.run.MethodsEnqueued - result.run.MethodsCompleted; + this._logger.logRunReset( + this._completedResults.size, + completedClassIds.size, + remainingTests, + pendingClassIds.length + ); + + return pendingClassIds.map(classId => ({ classId })); + } catch (err) { + // Be defensive: a reset should never be worse than a full re-run. + this._logger.logWarning( + `Could not determine completed tests for restart, re-running all: ${ + TestError.wrapError(err).message + }` + ); + return undefined; + } + } + + private async getQueueItems(testRunId: string): Promise { + return this._queryHelper.query( + 'ApexTestQueueItem', + `ParentJobId='${testRunId}'`, + 'Id, ApexClassId, Status' + ); + } + + private uniqueClassIds(queueItems: ApexTestQueueItem[]): string[] { + return Array.from(new Set(queueItems.map(item => item.ApexClassId))); + } + + /** + * Merge the results kept from completed classes with the final attempt's + * results so the returned result covers every test that ran across attempts. + */ + private mergeCompletedResults(result: TestRunnerResult): TestRunnerResult { + if (this._completedResults.size === 0) { + return result; + } + + const merged = new Map(this._completedResults); + result.tests.forEach(test => merged.set(getTestName(test), test)); + + return { ...result, tests: Array.from(merged.values()) }; + } + private getTestClassPayload(): null | AsyncTestArrayConfiguration { return this._testItems.length > 0 ? this.getSpecifiedTestsPayload(this._testItems) diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 0816e06..3d5a1ec 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -385,7 +385,24 @@ describe('TestRunner', () => { expect(error.kind).to.equal(TestErrorKind.Timeout); }); - it('should cancel and restart after no progress detected', async () => { + it('should cancel and re-run only incomplete classes after no progress detected', async () => { + // Class1 finishes (its pass is kept); Class3 is still running when the run + // stalls, so only Class3 should be re-run after the reset. + qhStub.query + .withArgs('ApexTestResult', match.any, match.any) + .onCall(0) + .resolves([mockTestResult[0]]) // Class1 pass + .onCall(1) + .resolves([mockTestResult[0]]) // no progress -> hang + .onCall(2) + .resolves([mockTestResult[1]]); // re-run produces Class3 result + const queueItems = [ + { Id: 'q1', ApexClassId: 'Class1', Status: 'Completed' }, + { Id: 'q3', ApexClassId: 'Class3', Status: 'Processing' }, + ]; + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .resolves(queueItems); setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ { Status: 'Processing' }, { Status: 'Processing' }, @@ -422,40 +439,111 @@ describe('TestRunner', () => { expect(mockAborter.calls).to.equal(1); expect(testServiceAsyncStub.calledTwice).to.be.true; + // The re-run only asks for the class that had not completed + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [{ classId: 'Class3' }], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); expect(testRunResult.run.AsyncApexJobId).to.equal(testRunId); expect(testRunResult.run.Status).to.equal('Completed'); expect(testRunResult.numberOfResets).to.equal(1); - expect(logger.entries.length).to.equal(10); + // Kept Class1's result and added Class3's from the re-run + expect(testRunResult.tests.map(t => t.Id).sort()).to.deep.equal([ + 'test1', + 'test2', + ]); + expect(logger.entries.length).to.equal(9); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) ); expect(logger.entries[1]).to.match( logRegex( - `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 1 \\| 2/2 Complete \\(100%\\)` + `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1/2 Complete \\(50%\\)` ) ); - expect(logger.entries[2]).to.match(logRegex("Failing tests in 'Class3':")); - expect(logger.entries[3]).to.match( - logRegex('\\* Method2 - Exception: Test Failed') - ); - expect(logger.entries[4]).to.match( + expect(logger.entries[2]).to.match( logRegex( - `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 1 \\| 2/2 Complete \\(100%\\)` + `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1/2 Complete \\(50%\\)` ) ); - expect(logger.entries[5]).to.match( + expect(logger.entries[3]).to.match( logRegex( `Test run '${testRunId}' was not progressing, cancelling and retrying...` ) ); - expect(logger.entries[6]).to.match( + expect(logger.entries[4]).to.match( + logRegex( + 'Reusing 1 tests from 1 completed classes; ' + + 'rerunning 2 remaining tests across 1 classes' + ) + ); + expect(logger.entries[5]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) ); - expect(logger.entries[7]).to.match( + expect(logger.entries[6]).to.match( logRegex( - `${timeFormat} \\[Completed\\] Passed: 1 \\| Failed: 1 \\| 2/2 Complete \\(100%\\)` + `${timeFormat} \\[Completed\\] Passed: 0 \\| Failed: 1 \\| 1/2 Complete \\(50%\\)` ) ); + expect(logger.entries[7]).to.match(logRegex("Failing tests in 'Class3':")); + expect(logger.entries[8]).to.match( + logRegex('\\* Method2 - Exception: Test Failed') + ); + }); + + it('should fall back to a full re-run when no incomplete classes can be identified', async () => { + // No queue items returned -> we cannot tell what is incomplete, so re-run + // everything rather than risk dropping tests. + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .resolves([]); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + { Status: 'Completed' }, + ]); + setupExecuteAnonymous( + sandbox.stub(ExecuteService.prototype, 'connectionRequest'), + { + column: -1, + line: -1, + compiled: 'true', + compileProblem: '', + exceptionMessage: '', + exceptionStackTrace: '', + success: 'true', + } + ); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } + ); + + const testRunResult = await runner.run(); + + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledTwice).to.be.true; + // Re-run uses the original full payload, not a pending subset + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [{ className: 'TestSample', namespace: undefined }], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); + expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(1); + // No reset reuse summary was logged + expect(logger.entries.some(e => /Reusing/.test(e))).to.be.false; }); it('should not cancel if progress detected', async () => { From 1f808b9b43dd1bedc5913835e73ed11416589f5f Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Mon, 15 Jun 2026 14:58:01 -0400 Subject: [PATCH 04/12] Updated mergeCompletedResults to merge results from multiple attempts when there is a reset --- src/runner/TestRunner.ts | 30 +++++++++++++++++++++++++++++- test/runner/TestRunner.spec.ts | 6 ++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index ca1fa55..712b045 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -284,6 +284,8 @@ export class AsyncTestRunner implements TestRunner { /** * Merge the results kept from completed classes with the final attempt's * results so the returned result covers every test that ran across attempts. + * The run-level counts (which otherwise only reflect the final restart + * subset) are recomputed from the merged set so the summary stays consistent. */ private mergeCompletedResults(result: TestRunnerResult): TestRunnerResult { if (this._completedResults.size === 0) { @@ -292,8 +294,34 @@ export class AsyncTestRunner implements TestRunner { const merged = new Map(this._completedResults); result.tests.forEach(test => merged.set(getTestName(test), test)); + const tests = Array.from(merged.values()); - return { ...result, tests: Array.from(merged.values()) }; + return { + ...result, + tests, + run: this.recomputeRunCounts(result.run, tests), + }; + } + + private recomputeRunCounts( + run: ApexTestRunResult, + tests: ApexTestResult[] + ): ApexTestRunResult { + const classes = new Set(tests.map(test => test.ApexClass.Id)); + const failed = tests.filter( + test => test.Outcome !== 'Pass' && test.Outcome !== 'Skip' + ).length; + const testTime = tests.reduce((total, test) => total + test.RunTime, 0); + + return { + ...run, + ClassesCompleted: classes.size, + ClassesEnqueued: classes.size, + MethodsCompleted: tests.length, + MethodsEnqueued: tests.length, + MethodsFailed: failed, + TestTime: testTime, + }; } private getTestClassPayload(): null | AsyncTestArrayConfiguration { diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 3d5a1ec..40a6a60 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -453,6 +453,12 @@ describe('TestRunner', () => { 'test1', 'test2', ]); + // Run-level counts reflect the merged set, not just the re-run subset + expect(testRunResult.run.MethodsCompleted).to.equal(2); + expect(testRunResult.run.MethodsEnqueued).to.equal(2); + expect(testRunResult.run.MethodsFailed).to.equal(1); + expect(testRunResult.run.ClassesCompleted).to.equal(2); + expect(testRunResult.run.TestTime).to.equal(30); // 10 + 20 expect(logger.entries.length).to.equal(9); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) From adfe3279aa768f849af0dba247a19a5c605b21f0 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Mon, 15 Jun 2026 15:14:29 -0400 Subject: [PATCH 05/12] Write diagnostic file when there is a reset --- src/runner/TestRunner.ts | 41 +++++++++++++++++++++++++++++++++- test/runner/TestRunner.spec.ts | 17 ++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index 712b045..f9e15a4 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -30,7 +30,7 @@ import { ApexTestQueueItem, QueueItemStatus } from '../model/ApexTestQueueItem'; import { TestError, TestErrorKind } from './TestError'; import { Pollable, poll, retry } from './Poll'; import { ApexTestResult, ApexTestResultFields } from '../model/ApexTestResult'; -import { getTestName } from '../results/TestResultUtils'; +import { getTestName, groupByOutcome } from '../results/TestResultUtils'; /** * Parallel unit test runner that includes the ability to cancel & restart a run should it not make sufficient progress @@ -243,6 +243,16 @@ export class AsyncTestRunner implements TestRunner { .filter(test => completedClassIds.has(test.ApexClass.Id)) .forEach(test => this._completedResults.set(getTestName(test), test)); + // Keep a per-reset diagnostic snapshot of what the org looked like when + // the run stalled, so resets can be investigated after the fact. + this.writeResetSnapshot( + testRunId, + queueItems, + completedClassIds, + pendingClassIds, + result + ); + if (pendingClassIds.length === 0) { // Nothing identified to re-run - fall back to a full re-run. return undefined; @@ -281,6 +291,35 @@ export class AsyncTestRunner implements TestRunner { return Array.from(new Set(queueItems.map(item => item.ApexClassId))); } + private writeResetSnapshot( + testRunId: string, + queueItems: ApexTestQueueItem[], + completedClassIds: Set, + pendingClassIds: string[], + result: TestRunnerResult + ): void { + const resetNumber = this._stats.getNumberOfTimesReset() + 1; + const outcomes = groupByOutcome(result.tests); + const snapshot = { + testRunId, + resetNumber, + completedClasses: completedClassIds.size, + pendingClasses: pendingClassIds.length, + reusedTests: this._completedResults.size, + results: { + total: result.tests.length, + passed: outcomes.Pass.length, + failed: outcomes.Fail.length + outcomes.CompileFail.length, + skipped: outcomes.Skip.length, + }, + queueItems, + }; + this._logger.logOutputFile( + `reset-${resetNumber}-${testRunId}.json`, + JSON.stringify(snapshot, undefined, 2) + ); + } + /** * Merge the results kept from completed classes with the final attempt's * results so the returned result covers every test that ran across attempts. diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 40a6a60..1fa719e 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -496,6 +496,23 @@ describe('TestRunner', () => { expect(logger.entries[8]).to.match( logRegex('\\* Method2 - Exception: Test Failed') ); + // A per-reset diagnostic snapshot is written + expect(logger.files.length).to.equal(1); + expect(logger.files[0][0]).to.equal( + `${process.cwd()}/reset-1-${testRunId}.json` + ); + const snapshot = JSON.parse(logger.files[0][1]) as { + resetNumber: number; + completedClasses: number; + pendingClasses: number; + reusedTests: number; + queueItems: unknown[]; + }; + expect(snapshot.resetNumber).to.equal(1); + expect(snapshot.completedClasses).to.equal(1); + expect(snapshot.pendingClasses).to.equal(1); + expect(snapshot.reusedTests).to.equal(1); + expect(snapshot.queueItems).to.have.length(2); }); it('should fall back to a full re-run when no incomplete classes can be identified', async () => { From b9455acbe203e69b33e2a7bfb4c5b3f1ca8c0d48 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Mon, 15 Jun 2026 15:19:59 -0400 Subject: [PATCH 06/12] Additional unit test for when the query fails --- test/runner/TestRunner.spec.ts | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 1fa719e..13d5976 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -569,6 +569,68 @@ describe('TestRunner', () => { expect(logger.entries.some(e => /Reusing/.test(e))).to.be.false; }); + it('should fall back to a full re-run when the queue cannot be queried', async () => { + // The query for incomplete classes fails, so we must not drop tests - fall + // back to re-running everything and warn. + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .rejects(new Error('query boom')); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + { Status: 'Completed' }, + ]); + setupExecuteAnonymous( + sandbox.stub(ExecuteService.prototype, 'connectionRequest'), + { + column: -1, + line: -1, + compiled: 'true', + compileProblem: '', + exceptionMessage: '', + exceptionStackTrace: '', + success: 'true', + } + ); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } + ); + + const testRunResult = await runner.run(); + + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledTwice).to.be.true; + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [{ className: 'TestSample', namespace: undefined }], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); + expect(testRunResult.run.Status).to.equal('Completed'); + expect(testRunResult.numberOfResets).to.equal(1); + expect(logger.entries.some(e => /Reusing/.test(e))).to.be.false; + expect( + logger.entries.some(e => + logRegex( + 'Warning: Could not determine completed tests for restart, ' + + 're-running all: query boom' + ).test(e) + ) + ).to.be.true; + // No snapshot written when we could not inspect the queue + expect(logger.files.length).to.equal(0); + }); + it('should not cancel if progress detected', async () => { const finalResults: ApexTestResult[] = [ ...mockTestResult, From 9decae056dc2bf8aaa1bcd365f8874bf42a01535 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Tue, 16 Jun 2026 16:38:42 -0400 Subject: [PATCH 07/12] Add logging about stalls to the usual 'processing' line --- src/log/BaseLogger.ts | 13 +++++++++++-- src/log/Logger.ts | 4 +++- src/runner/TestRunner.ts | 13 ++++++++++++- src/runner/TestStats.ts | 8 ++++++++ test/log/BaseLogger.spec.ts | 29 ++++++++++++++++++++++++++++- test/runner/TestRunner.spec.ts | 2 +- 6 files changed, 63 insertions(+), 6 deletions(-) diff --git a/src/log/BaseLogger.ts b/src/log/BaseLogger.ts index ae71dfe..4cd4d7e 100644 --- a/src/log/BaseLogger.ts +++ b/src/log/BaseLogger.ts @@ -156,7 +156,9 @@ export abstract class BaseLogger implements Logger { logStatus( testRunResult: ApexTestRunResult, tests: ApexTestResult[], - elapsedTime: string + elapsedTime: string, + noProgressPolls: number, + noProgressLimit: number ): void { const status = testRunResult.Status; const outcomes = groupByOutcome(tests); @@ -166,8 +168,15 @@ export abstract class BaseLogger implements Logger { const total = testRunResult.MethodsEnqueued; const complete = total > 0 ? Math.floor((completed * 100) / total) : 0; + // While a run is in flight, surface how close it is to a reset so a stall + // is visible without an extra log line per poll. + const noProgress = + noProgressPolls > 0 + ? ` | No progress ${noProgressPolls}/${noProgressLimit}` + : ''; + this.logMessage( - `${elapsedTime} [${status}] Passed: ${passed} | Failed: ${failed} | ${completed}/${total} Complete (${complete}%)` + `${elapsedTime} [${status}] Passed: ${passed} | Failed: ${failed} | ${completed}/${total} Complete (${complete}%)${noProgress}` ); } diff --git a/src/log/Logger.ts b/src/log/Logger.ts index 0333cf0..af65820 100644 --- a/src/log/Logger.ts +++ b/src/log/Logger.ts @@ -44,7 +44,9 @@ export interface Logger { logStatus( status: ApexTestRunResult, tests: ApexTestResult[], - elapsedTime: string + elapsedTime: string, + noProgressPolls: number, + noProgressLimit: number ): void; logTestFailures(newResults: ApexTestResult[]): void; diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index f9e15a4..e3e3f75 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -471,9 +471,20 @@ export class AsyncTestRunner implements TestRunner { results: ApexTestResult[], time: string ): Promise { - this._logger.logStatus(testRunResult, results, time); this._stats = this._stats.update(results.length); + // No-progress count is only meaningful while the run is still in flight. + const noProgressPolls = this.hasTestRunComplete(testRunResult.Status) + ? 0 + : this._stats.getNoProgressPollCount(); + this._logger.logStatus( + testRunResult, + results, + time, + noProgressPolls, + this._stats.getNoProgressPollLimit() + ); + if (this._logger.verbose) { await this.reportQueueItems(testRunResult.AsyncApexJobId); } diff --git a/src/runner/TestStats.ts b/src/runner/TestStats.ts index edadfb7..d8a7600 100644 --- a/src/runner/TestStats.ts +++ b/src/runner/TestStats.ts @@ -66,4 +66,12 @@ export default class TestStats { public getNumberOfTimesReset(): number { return this._numberOfTimesReset; } + + public getNoProgressPollCount(): number { + return this._numberOfTimesUpdatedWithoutChange; + } + + public getNoProgressPollLimit(): number { + return this._updateLimitToAssumeTestHanging; + } } diff --git a/test/log/BaseLogger.spec.ts b/test/log/BaseLogger.spec.ts index 1d4a571..bc4eb71 100644 --- a/test/log/BaseLogger.spec.ts +++ b/test/log/BaseLogger.spec.ts @@ -30,7 +30,9 @@ describe('BaseLogger', () => { logger.logStatus( mockTestRunResult as ApexTestRunResult, mockTestResults as ApexTestResult[], - '' + '', + 0, + 5 ); expect(logger.entries.length).to.equal(1); @@ -41,6 +43,31 @@ describe('BaseLogger', () => { ); }); + it('should append a no-progress count to the status line when stalled', () => { + const mockTestRunResult: Partial = { + Status: 'Processing', + MethodsEnqueued: 6, + }; + const mockTestResults: Partial[] = [{ Outcome: 'Pass' }]; + + const logger = new CapturingLogger(); + logger.logStatus( + mockTestRunResult as ApexTestRunResult, + mockTestResults as ApexTestResult[], + '', + 3, + 5 + ); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex( + '\\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1\\/6 Complete \\(16%\\)' + + ' \\| No progress 3/5' + ) + ); + }); + it('should output file content with filename', () => { const logger = new CapturingLogger('/log/path', true); const content = '{ "records": [] }'; diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 13d5976..912aa6c 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -470,7 +470,7 @@ describe('TestRunner', () => { ); expect(logger.entries[2]).to.match( logRegex( - `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1/2 Complete \\(50%\\)` + `${timeFormat} \\[Processing\\] Passed: 1 \\| Failed: 0 \\| 1/2 Complete \\(50%\\) \\| No progress 1/1` ) ); expect(logger.entries[3]).to.match( From 6b37b319e798259ae0c503e869d0079329682f19 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Wed, 17 Jun 2026 14:09:24 -0400 Subject: [PATCH 08/12] Abort runs that get stuck before making any progress --- src/log/BaseLogger.ts | 6 +++++ src/log/Logger.ts | 1 + src/runner/TestRunner.ts | 42 +++++++++++++++++++++++++++++---- test/log/BaseLogger.spec.ts | 13 ++++++++++ test/runner/TestRunner.spec.ts | 43 +++++++++++++++++++++++++++++++++- 5 files changed, 99 insertions(+), 6 deletions(-) diff --git a/src/log/BaseLogger.ts b/src/log/BaseLogger.ts index 4cd4d7e..8759993 100644 --- a/src/log/BaseLogger.ts +++ b/src/log/BaseLogger.ts @@ -141,6 +141,12 @@ export abstract class BaseLogger implements Logger { ); } + logRunStuck(testRunId: string, status: string): void { + this.logMessage( + `Test run '${testRunId}' stuck in ${status} with no progress, abandoning this attempt` + ); + } + logRunReset( reusedTests: number, completedClasses: number, diff --git a/src/log/Logger.ts b/src/log/Logger.ts index af65820..66146ef 100644 --- a/src/log/Logger.ts +++ b/src/log/Logger.ts @@ -35,6 +35,7 @@ export interface Logger { // Test runner logRunStarted(testRunId: string): void; logNoProgress(testRunId: string): void; + logRunStuck(testRunId: string, status: string): void; logRunReset( reusedTests: number, completedClasses: number, diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index e3e3f75..087a086 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -196,6 +196,19 @@ export class AsyncTestRunner implements TestRunner { await this.abortTestRun(result.run.AsyncApexJobId); return await this.runInternal(token, restartItems); } + + if ( + !this.hasTestRunComplete(result.run.Status) && + this._stats.isTestRunHanging() + ) { + // Hung before it started processing (e.g. stuck in Queued). Re-running + // would likely just stall again, and leaving it in-flight would block + // the caller from re-running the missing tests (ALREADY_IN_PROCESS), so + // abort it and give up on this attempt. + this._logger.logRunStuck(testRunIdResult.testRunId, result.run.Status); + await this.abortTestRun(result.run.AsyncApexJobId); + return result; + } } catch (err) { // error may already be defined from wait() if (!result.error) { @@ -239,9 +252,12 @@ export class AsyncTestRunner implements TestRunner { // Keep results from classes that finished; drop the rest so the restart // re-runs them cleanly. - result.tests - .filter(test => completedClassIds.has(test.ApexClass.Id)) - .forEach(test => this._completedResults.set(getTestName(test), test)); + const completedResults = result.tests.filter(test => + completedClassIds.has(test.ApexClass.Id) + ); + completedResults.forEach(test => + this._completedResults.set(getTestName(test), test) + ); // Keep a per-reset diagnostic snapshot of what the org looked like when // the run stalled, so resets can be investigated after the fact. @@ -258,8 +274,10 @@ export class AsyncTestRunner implements TestRunner { return undefined; } + // Tests still to run = those enqueued this attempt that weren't in a + // completed class (i.e. the pending classes' methods). const remainingTests = - result.run.MethodsEnqueued - result.run.MethodsCompleted; + result.run.MethodsEnqueued - completedResults.length; this._logger.logRunReset( this._completedResults.size, completedClassIds.size, @@ -300,6 +318,14 @@ export class AsyncTestRunner implements TestRunner { ): void { const resetNumber = this._stats.getNumberOfTimesReset() + 1; const outcomes = groupByOutcome(result.tests); + + // Resolve class names from the results we already have (no extra org call). + // Classes that produced no results won't have a name available. + const classNames = new Map(); + result.tests.forEach(test => + classNames.set(test.ApexClass.Id, test.ApexClass.Name) + ); + const snapshot = { testRunId, resetNumber, @@ -312,7 +338,13 @@ export class AsyncTestRunner implements TestRunner { failed: outcomes.Fail.length + outcomes.CompileFail.length, skipped: outcomes.Skip.length, }, - queueItems, + // Slimmed down (drop the jsforce attributes) and annotated with the + // class name where we know it. + queueItems: queueItems.map(item => ({ + ApexClassId: item.ApexClassId, + className: classNames.get(item.ApexClassId), + Status: item.Status, + })), }; this._logger.logOutputFile( `reset-${resetNumber}-${testRunId}.json`, diff --git a/test/log/BaseLogger.spec.ts b/test/log/BaseLogger.spec.ts index bc4eb71..4a250a8 100644 --- a/test/log/BaseLogger.spec.ts +++ b/test/log/BaseLogger.spec.ts @@ -136,6 +136,19 @@ describe('BaseLogger', () => { ); }); + it('should log when a run is abandoned while stuck', () => { + const logger = new CapturingLogger(); + logger.logRunStuck('707xx', 'Queued'); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex( + "Test run '707xx' stuck in Queued with no progress, " + + 'abandoning this attempt' + ) + ); + }); + it('should generate warning messages', () => { const logger = new CapturingLogger('', true); logger.logWarning('A message'); diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 912aa6c..9d48399 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -481,7 +481,7 @@ describe('TestRunner', () => { expect(logger.entries[4]).to.match( logRegex( 'Reusing 1 tests from 1 completed classes; ' + - 'rerunning 2 remaining tests across 1 classes' + 'rerunning 1 remaining tests across 1 classes' ) ); expect(logger.entries[5]).to.match( @@ -631,6 +631,47 @@ describe('TestRunner', () => { expect(logger.files.length).to.equal(0); }); + it('should abort and give up when a run is stuck before processing', async () => { + // Run never leaves 'Queued' and makes no progress. It should be aborted and + // abandoned (not reset/retried) so its tests aren't left enqueued for the + // caller's missing-test re-run to collide with. + qhStub.query.withArgs('ApexTestResult', match.any, match.any).resolves([]); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Queued' }, + { Status: 'Queued' }, + ]); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } + ); + + const testRunResult = await runner.run(); + + // Aborted once, and not restarted + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledOnce).to.be.true; + expect(testRunResult.run.Status).to.equal('Queued'); + expect(testRunResult.numberOfResets).to.equal(0); + expect( + logger.entries.some(e => + logRegex( + `Test run '${testRunId}' stuck in Queued with no progress, ` + + 'abandoning this attempt' + ).test(e) + ) + ).to.be.true; + }); + it('should not cancel if progress detected', async () => { const finalResults: ApexTestResult[] = [ ...mockTestResult, From d613352dfcbb03bcb7405b2be8b823429bf54203 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Wed, 17 Jun 2026 16:13:51 -0400 Subject: [PATCH 09/12] Improve output file handling and logging for test run resets --- src/command/Testall.ts | 17 +--------- src/log/BaseLogger.ts | 4 +++ src/log/Logger.ts | 1 + src/runner/TestOptions.ts | 15 +++++++++ src/runner/TestRunner.ts | 29 ++++++++++++++--- test/log/BaseLogger.spec.ts | 10 ++++++ test/runner/TestOptions.spec.ts | 39 ++++++++++++++++++++++ test/runner/TestRunner.spec.ts | 58 +++++++++++++++++++++++++++++---- 8 files changed, 146 insertions(+), 27 deletions(-) create mode 100644 test/runner/TestOptions.spec.ts diff --git a/src/command/Testall.ts b/src/command/Testall.ts index 7ead67c..a6d00a1 100644 --- a/src/command/Testall.ts +++ b/src/command/Testall.ts @@ -20,7 +20,7 @@ import { TestRerun, TestRunSummary, } from '../results/OutputGenerator'; -import { TestRunnerOptions } from '../runner/TestOptions'; +import { getOutputFileBase, TestRunnerOptions } from '../runner/TestOptions'; import { TestRunner } from '../runner/TestRunner'; import { formatTestName, getTestName } from '../results/TestResultUtils'; import { TestResultStore } from '../results/TestResultStore'; @@ -43,8 +43,6 @@ import { retry } from '../runner/Poll'; export interface TestallOptions extends TestRunnerOptions, QueryOptions { maxErrorsForReRun?: number; // Don't re-run if > failed tests (excluding pattern matched tests), default 10 - outputDirBase?: string; // Base for junit and other output files, default 'test-result*' - outputFileName?: string; //File name base disableCoverageReport?: boolean; // if enabled disables coverage collection rerunOption?: RerunOption; // see RerunOption - default 'pattern' } @@ -56,7 +54,6 @@ export enum RerunOption { } const DEFAULT_MAX_ERRORS_FOR_RERUN = 10; -const DEFAULT_OUTPUT_FILE_BASE = 'test-result'; const DEFAULT_RERUN_OPTION = RerunOption.Pattern; export function getMaxErrorsForReRun(options: TestallOptions): number { @@ -65,18 +62,6 @@ export function getMaxErrorsForReRun(options: TestallOptions): number { else return DEFAULT_MAX_ERRORS_FOR_RERUN; } -export function getOutputFileBase(options: TestallOptions): { - fileName: string; - outputDir: string; -} { - if (options.outputDirBase && options.outputFileName) - return { - outputDir: options.outputDirBase, - fileName: options.outputFileName, - }; - else return { outputDir: '', fileName: DEFAULT_OUTPUT_FILE_BASE }; -} - export function getReRunOption(options: TestallOptions): RerunOption { const opt = options.rerunOption; if (opt && Object.values(RerunOption).find(v => v === opt)) return opt; diff --git a/src/log/BaseLogger.ts b/src/log/BaseLogger.ts index 8759993..850f89e 100644 --- a/src/log/BaseLogger.ts +++ b/src/log/BaseLogger.ts @@ -147,6 +147,10 @@ export abstract class BaseLogger implements Logger { ); } + logResetCount(resetNumber: number, maxResets: number): void { + this.logMessage(`Reset ${resetNumber}/${maxResets} before abandoning run`); + } + logRunReset( reusedTests: number, completedClasses: number, diff --git a/src/log/Logger.ts b/src/log/Logger.ts index 66146ef..a72a684 100644 --- a/src/log/Logger.ts +++ b/src/log/Logger.ts @@ -36,6 +36,7 @@ export interface Logger { logRunStarted(testRunId: string): void; logNoProgress(testRunId: string): void; logRunStuck(testRunId: string, status: string): void; + logResetCount(resetNumber: number, maxResets: number): void; logRunReset( reusedTests: number, completedClasses: number, diff --git a/src/runner/TestOptions.ts b/src/runner/TestOptions.ts index 51960d1..c4e2817 100644 --- a/src/runner/TestOptions.ts +++ b/src/runner/TestOptions.ts @@ -34,6 +34,7 @@ export function getCancelPollTimeout(options: CancelTestRunOptions): Duration { else return Duration.minutes(DEFAULT_POLL_TIMEOUT_MINS); } +export const DEFAULT_OUTPUT_FILE_BASE = 'test-result'; export const DEFAULT_STATUS_POLL_INTERVAL_MS = 30000; export const DEFAULT_TEST_RUN_TIMEOUT_MINS = 120; const DEFAULT_MAX_TEST_RUN_RETRIES = 3; @@ -65,6 +66,20 @@ export interface TestRunnerOptions extends CancelTestRunOptions { pollLimitToAssumeHangingTests?: number; // Number polls without test progress before a hang is assumed, default 60 callbacks?: TestRunnerCallbacks; // Callbacks for events in test runner codeCoverage?: boolean; // Collect code coverage data, defaults false + outputDirBase?: string; // Base directory for junit and other output files + outputFileName?: string; // File name base for output files, default 'test-result' +} + +export function getOutputFileBase(options: TestRunnerOptions): { + fileName: string; + outputDir: string; +} { + if (options.outputDirBase && options.outputFileName) + return { + outputDir: options.outputDirBase, + fileName: options.outputFileName, + }; + else return { outputDir: '', fileName: DEFAULT_OUTPUT_FILE_BASE }; } export function getTestRunAborter(options: TestRunnerOptions): TestRunAborter { diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index 087a086..0b611f9 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -18,12 +18,14 @@ import { } from '../model/ApexTestRunResult'; import { getMaxTestRunRetries, + getOutputFileBase, getStatusPollInterval, getTestRunAborter, getTestRunTimeout, getTestRunTimeoutMessage, TestRunnerOptions, } from './TestOptions'; +import path from 'path'; import TestStats from './TestStats'; import { QueryHelper } from '../query/QueryHelper'; import { ApexTestQueueItem, QueueItemStatus } from '../model/ApexTestQueueItem'; @@ -188,10 +190,22 @@ export class AsyncTestRunner implements TestRunner { if (result.run.Status == 'Processing' && this._stats.isTestRunHanging()) { this._logger.logNoProgress(testRunIdResult.testRunId); - const restartItems = await this.prepareRestart( - testRunIdResult.testRunId, - result - ); + + // Only work out what to re-run if we have a reset left. Once retries are + // exhausted the recursive run will abandon (throw), so skip the + // prepareRestart work to avoid a misleading "rerunning" log, a wasted + // queue query and snapshot. + const resetNumber = this._stats.getNumberOfTimesReset() + 1; + const maxResets = getMaxTestRunRetries(this._options) - 1; + let restartItems: TestItem[] | undefined; + if (resetNumber <= maxResets) { + this._logger.logResetCount(resetNumber, maxResets); + restartItems = await this.prepareRestart( + testRunIdResult.testRunId, + result + ); + } + this._stats = this._stats.reset(); await this.abortTestRun(result.run.AsyncApexJobId); return await this.runInternal(token, restartItems); @@ -346,8 +360,13 @@ export class AsyncTestRunner implements TestRunner { Status: item.Status, })), }; + // Write alongside the other output files, using the same dir/name prefix. + const { outputDir, fileName } = getOutputFileBase(this._options); this._logger.logOutputFile( - `reset-${resetNumber}-${testRunId}.json`, + path.join( + outputDir, + `${fileName}-reset-${resetNumber}-${testRunId}.json` + ), JSON.stringify(snapshot, undefined, 2) ); } diff --git a/test/log/BaseLogger.spec.ts b/test/log/BaseLogger.spec.ts index 4a250a8..9b3aa72 100644 --- a/test/log/BaseLogger.spec.ts +++ b/test/log/BaseLogger.spec.ts @@ -136,6 +136,16 @@ describe('BaseLogger', () => { ); }); + it('should log the reset count', () => { + const logger = new CapturingLogger(); + logger.logResetCount(1, 2); + + expect(logger.entries.length).to.equal(1); + expect(logger.entries[0]).to.match( + logRegex('Reset 1/2 before abandoning run') + ); + }); + it('should log when a run is abandoned while stuck', () => { const logger = new CapturingLogger(); logger.logRunStuck('707xx', 'Queued'); diff --git a/test/runner/TestOptions.spec.ts b/test/runner/TestOptions.spec.ts new file mode 100644 index 0000000..90cd7b4 --- /dev/null +++ b/test/runner/TestOptions.spec.ts @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2026, Certinia Inc. All rights reserved. + */ + +import { expect } from 'chai'; +import { getOutputFileBase } from '../../src/runner/TestOptions'; + +describe('TestOptions', () => { + describe('getOutputFileBase', () => { + it('should use the supplied dir and file name when both are set', () => { + expect( + getOutputFileBase({ + outputDirBase: 'test-results/apex', + outputFileName: 'reset', + }) + ).to.deep.equal({ outputDir: 'test-results/apex', fileName: 'reset' }); + }); + + it('should default when neither is set', () => { + expect(getOutputFileBase({})).to.deep.equal({ + outputDir: '', + fileName: 'test-result', + }); + }); + + it('should default when only the dir is set', () => { + expect( + getOutputFileBase({ outputDirBase: 'test-results/apex' }) + ).to.deep.equal({ outputDir: '', fileName: 'test-result' }); + }); + + it('should default when only the file name is set', () => { + expect(getOutputFileBase({ outputFileName: 'jim' })).to.deep.equal({ + outputDir: '', + fileName: 'test-result', + }); + }); + }); +}); diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 9d48399..3157a35 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -265,6 +265,49 @@ describe('TestRunner', () => { expect(error.kind).to.equal(TestErrorKind.Timeout); }); + it('should abort and abandon without re-running when retries are exhausted', async () => { + // One retry allowed. The run stalls, exhausting it - so we abort and + // abandon without pretending to re-run (no reset/reusing logs, no extra + // queue query). + qhStub.query + .withArgs('ApexTestResult', match.any, match.any) + .resolves([mockTestResult[0]]); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + ]); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + const runner = AsyncTestRunner.forClasses( + logger, + mockConnection, + '', + ['TestSample'], + { + maxTestRunRetries: 1, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + } + ); + + // Exhausting retries surfaces as a result error (caught for partial + // reporting), not a throw. + const testRunResult = await runner.run(); + expect(testRunResult.error).to.be.instanceof(TestError); + expect(testRunResult.error?.message).to.equal( + 'Max number of test run retries reached, max allowed retries: 1' + ); + // The stalled run was aborted, and not re-run + expect(mockAborter.calls).to.equal(1); + expect(testServiceAsyncStub.calledOnce).to.be.true; + // No queue query, no reset/reusing logs - we did not pretend to re-run + expect(qhStub.query.calledWith('ApexTestQueueItem', match.any, match.any)) + .to.be.false; + expect(logger.entries.some(e => /Reusing/.test(e))).to.be.false; + expect(logger.entries.some(e => /Reset \d/.test(e))).to.be.false; + }); + it('should throw on timeout if no results found', async () => { qhStub.query.resolves([]); @@ -459,7 +502,7 @@ describe('TestRunner', () => { expect(testRunResult.run.MethodsFailed).to.equal(1); expect(testRunResult.run.ClassesCompleted).to.equal(2); expect(testRunResult.run.TestTime).to.equal(30); // 10 + 20 - expect(logger.entries.length).to.equal(9); + expect(logger.entries.length).to.equal(10); expect(logger.entries[0]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) ); @@ -479,27 +522,30 @@ describe('TestRunner', () => { ) ); expect(logger.entries[4]).to.match( + logRegex('Reset 1/1 before abandoning run') + ); + expect(logger.entries[5]).to.match( logRegex( 'Reusing 1 tests from 1 completed classes; ' + 'rerunning 1 remaining tests across 1 classes' ) ); - expect(logger.entries[5]).to.match( + expect(logger.entries[6]).to.match( logRegex(`Test run started with AsyncApexJob Id: ${testRunId}`) ); - expect(logger.entries[6]).to.match( + expect(logger.entries[7]).to.match( logRegex( `${timeFormat} \\[Completed\\] Passed: 0 \\| Failed: 1 \\| 1/2 Complete \\(50%\\)` ) ); - expect(logger.entries[7]).to.match(logRegex("Failing tests in 'Class3':")); - expect(logger.entries[8]).to.match( + expect(logger.entries[8]).to.match(logRegex("Failing tests in 'Class3':")); + expect(logger.entries[9]).to.match( logRegex('\\* Method2 - Exception: Test Failed') ); // A per-reset diagnostic snapshot is written expect(logger.files.length).to.equal(1); expect(logger.files[0][0]).to.equal( - `${process.cwd()}/reset-1-${testRunId}.json` + `${process.cwd()}/test-result-reset-1-${testRunId}.json` ); const snapshot = JSON.parse(logger.files[0][1]) as { resetNumber: number; From a5ac2b2f305e6f3e7f8607fb4f005053f74d1799 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Wed, 17 Jun 2026 16:57:15 -0400 Subject: [PATCH 10/12] Update changelog and version number --- CHANGELOG.md | 8 ++++++++ package.json | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d23400..b744cfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # test-runner - Changelog +## 3.4.0 - 2026-06-17 + +* Only re-run incomplete tests after a test run reset. + * When a run is cancelled and restarted due to lack of progress, results from classes that already completed are kept and only the unfinished classes are re-run, instead of re-running everything. Genuine failures are still re-run sequentially at the end. +* Write a per-reset diagnostic file (`-reset--.json`) alongside the other output, recording the queue item statuses and a results summary at the point of the reset. +* Abort and abandon a run that stalls before it starts processing (e.g. while stuck queued), rather than leaving it in-flight to be re-run. +* Test run status logging now shows a `No progress /` suffix while a run is stalling, and resets log the reuse summary and a `Reset /` count. + ## 3.3.0 - 2026-02-25 * Add `numberOfResets` to `TestRunnerResult`, tracking how many times a test run was cancelled and restarted due to lack of progress. diff --git a/package.json b/package.json index baa4da1..45cecf4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@apexdevtools/test-runner", - "version": "3.3.0", + "version": "3.4.0", "description": "Apex parallel test runner with reliability goodness", "author": { "name": "Apex Dev Tools Team", From 701388e4e83ab5921b51aaa0e38a418cfe1021dd Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Tue, 23 Jun 2026 13:14:22 -0400 Subject: [PATCH 11/12] Fix a bug when running specific tests, so we don't re-run more tests than we've been asked for --- src/runner/TestRunner.ts | 15 +++++++- test/runner/TestRunner.spec.ts | 63 ++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/src/runner/TestRunner.ts b/src/runner/TestRunner.ts index 8ff78f7..b78ad83 100644 --- a/src/runner/TestRunner.ts +++ b/src/runner/TestRunner.ts @@ -304,7 +304,7 @@ export class AsyncTestRunner implements TestRunner { pendingClassIds.length ); - return pendingClassIds.map(classId => ({ classId })); + return this.buildRestartItems(pendingClassIds); } catch (err) { // Be defensive: a reset should never be worse than a full re-run. this._logger.logWarning( @@ -328,6 +328,19 @@ export class AsyncTestRunner implements TestRunner { return Array.from(new Set(queueItems.map(item => item.ApexClassId))); } + /** + * Build the restart TestItems for pending classes. For method-specific + * runners (e.g. a missing-test rerun), re-use the original items unchanged + * so a reset doesn't widen a method-filtered run to the whole class. + * For all-local / class-level runs, restart at class granularity. + */ + private buildRestartItems(pendingClassIds: string[]): TestItem[] { + if (this._testItems.some(item => item.testMethods?.length)) { + return this._testItems; + } + return pendingClassIds.map(classId => ({ classId })); + } + private writeResetSnapshot( testRunId: string, queueItems: ApexTestQueueItem[], diff --git a/test/runner/TestRunner.spec.ts b/test/runner/TestRunner.spec.ts index 98c17ce..341516d 100644 --- a/test/runner/TestRunner.spec.ts +++ b/test/runner/TestRunner.spec.ts @@ -598,6 +598,69 @@ describe('TestRunner', () => { expect(snapshot.queueItems).to.have.length(2); }); + it('should narrow restart to originally-requested methods for method-specific runners', async () => { + // The runner was created with specific methods (e.g. a missing-test rerun). + // Class3 is still pending. The restart should only re-run the originally + // requested methods of Class3, not widen to the whole class. + qhStub.query + .withArgs('ApexTestResult', match.any, match.any) + .onCall(0) + .resolves([mockTestResult[0]]) // Class1 pass + .onCall(1) + .resolves([mockTestResult[0]]) // no progress → hang + .onCall(2) + .resolves([mockTestResult[1]]); // re-run produces Class3 result + const queueItems = [ + { Id: 'q1', ApexClassId: 'Class1', Status: 'Completed' }, + { Id: 'q3', ApexClassId: 'Class3', Status: 'Processing' }, + ]; + qhStub.query + .withArgs('ApexTestQueueItem', match.any, match.any) + .resolves(queueItems); + setupMultipleQueryApexTestResults(qhStub, mockTestResult, [ + { Status: 'Processing' }, + { Status: 'Processing' }, + { Status: 'Completed' }, + ]); + setupExecuteAnonymous( + sandbox.stub(ExecuteService.prototype, 'connectionRequest'), + { + column: -1, + line: -1, + compiled: 'true', + compileProblem: '', + exceptionMessage: '', + exceptionStackTrace: '', + success: 'true', + } + ); + + const logger = new CapturingLogger(); + const mockAborter = new MockAborter(); + // Method-specific runner — only Method2 of Class3 was requested + const runner = new AsyncTestRunner(logger, mockConnection, [ + { className: 'Class1' }, + { classId: 'Class3', testMethods: ['Method2'] }, + ], { + maxTestRunRetries: 2, + pollLimitToAssumeHangingTests: 1, + aborter: mockAborter, + }); + + await runner.run(); + + // Restart uses original _testItems — method scope is preserved, not widened to whole class + expect(testServiceAsyncStub.calledTwice).to.be.true; + expect(testServiceAsyncStub.args[1][0]).to.deep.equal({ + tests: [ + { className: 'Class1' }, + { classId: 'Class3', testMethods: ['Method2'] }, + ], + testLevel: TestLevel.RunSpecifiedTests, + skipCodeCoverage: true, + }); + }); + it('should fall back to a full re-run when no incomplete classes can be identified', async () => { // No queue items returned -> we cannot tell what is incomplete, so re-run // everything rather than risk dropping tests. From 0bb28b909537149628105a5ff1cdfb66a1c3ce23 Mon Sep 17 00:00:00 2001 From: Jim Woodward Date: Tue, 23 Jun 2026 13:39:25 -0400 Subject: [PATCH 12/12] Include resets from the missing-test run in numberOfResets --- CHANGELOG.md | 3 +- src/command/Testall.ts | 1 - src/results/TestResultStore.ts | 2 +- test/command/Testall.spec.ts | 53 ++++++++++++++++++++++++++++++++-- 4 files changed, 54 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa30048..886edac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,13 @@ # test-runner - Changelog -## 3.4.0 - 2026-06-17 +## 3.4.0 - 2026-06-23 * Only re-run incomplete tests after a test run reset. * When a run is cancelled and restarted due to lack of progress, results from classes that already completed are kept and only the unfinished classes are re-run, instead of re-running everything. Genuine failures are still re-run sequentially at the end. * Write a per-reset diagnostic file (`-reset--.json`) alongside the other output, recording the queue item statuses and a results summary at the point of the reset. * Abort and abandon a run that stalls before it starts processing (e.g. while stuck queued), rather than leaving it in-flight to be re-run. * Test run status logging now shows a `No progress /` suffix while a run is stalling, and resets log the reuse summary and a `Reset /` count. +* `numberOfResets` in `TestRunSummary` now reports the total resets across all runs in a `Testall` invocation (including missing-test reruns), not just the outer run's count. ## 3.3.1-beta.0 - 2026-06-14 diff --git a/src/command/Testall.ts b/src/command/Testall.ts index 4fb20aa..86cc064 100644 --- a/src/command/Testall.ts +++ b/src/command/Testall.ts @@ -200,7 +200,6 @@ export class Testall { store ); - store.numberOfResets = result.numberOfResets; } } diff --git a/src/results/TestResultStore.ts b/src/results/TestResultStore.ts index f77aa97..4f1e9d6 100644 --- a/src/results/TestResultStore.ts +++ b/src/results/TestResultStore.ts @@ -45,7 +45,7 @@ export class TestResultStore { }); this.asyncError = res.error; - this.numberOfResets = res.numberOfResets; + this.numberOfResets += res.numberOfResets; } public saveSyncResult(reruns: TestRerun[]): void { diff --git a/test/command/Testall.spec.ts b/test/command/Testall.spec.ts index a20023d..6871cf7 100644 --- a/test/command/Testall.spec.ts +++ b/test/command/Testall.spec.ts @@ -117,7 +117,7 @@ describe('TestAll', () => { [new MockOutputGenerator()], {} ); - } catch (err) { + } catch { // Ignore } @@ -152,7 +152,7 @@ describe('TestAll', () => { [new MockOutputGenerator()], {} ); - } catch (err) { + } catch { // Ignore } @@ -713,4 +713,53 @@ describe('TestAll', () => { logRegex('Generated reports for 2 tests') ); }); + + it('should accumulate numberOfResets across outer run and missing-test rerun', async () => { + // Outer run has 2 resets; the missing-test rerun has 1 reset. + // The summary should report 3, not just the outer count. + const logger = new CapturingLogger(); + const mockRunResult: ApexTestRunResult = createMockRunResult(); + const classId = 'classId1'; + const className = 'TestClass'; + const mockTestResults = [ + createMockTestResult({ + MethodName: 'method1', + ApexClass: { Id: classId, Name: className, NamespacePrefix: null }, + }), + createMockTestResult({ + MethodName: 'method2', + ApexClass: { Id: classId, Name: className, NamespacePrefix: null }, + }), + ]; + const runner = new MockTestRunner({ + run: mockRunResult, + tests: [mockTestResults[0]], + numberOfResets: 2, + }).addNextResult({ + run: mockRunResult, + tests: [mockTestResults[1]], + numberOfResets: 1, + }); + const testMethods = new MockTestMethodCollector( + logger, + mockConnection, + '', + new Map([[classId, className]]), + new Map>([ + [className, new Set(['method1', 'method2'])], + ]) + ); + + const result = await Testall.run( + logger, + mockConnection, + '', + testMethods, + runner, + [new MockOutputGenerator()], + {} + ); + + expect(result.numberOfResets).to.equal(3); + }); });