From 10aeba41edb78fbd65f1b8d2767a621f0ffdd0f2 Mon Sep 17 00:00:00 2001 From: Satvik Choudhary Date: Tue, 10 Feb 2026 14:23:37 +0530 Subject: [PATCH 1/7] Add Allure upload command support Introduce a directory-based Allure parser and wire it into the upload pipeline so users can upload Allure 2 results with TMS-link or marker-based matching. Extend CLI, tests, and docs for allure-upload, support force-aware skipping of invalid Allure files, and add AGENTS.md as a symlink to CLAUDE.md. Co-authored-by: Cursor --- AGENTS.md | 1 + CLAUDE.md | 10 +- README.md | 37 ++- src/commands/main.ts | 1 + src/commands/resultUpload.ts | 43 ++- src/tests/allure-parsing.spec.ts | 271 ++++++++++++++++++ .../allure/empty-tsuite/001-result.json | 13 + .../allure/empty-tsuite/002-container.json | 8 + .../allure/invalid-results/001-result.json | 7 + .../allure/invalid-results/003-result.json | 6 + .../allure/matching-tcases/001-result.json | 27 ++ .../allure/matching-tcases/002-result.json | 28 ++ .../allure/matching-tcases/003-result.json | 21 ++ .../allure/matching-tcases/004-result.json | 30 ++ .../allure/matching-tcases/005-result.json | 24 ++ .../allure/matching-tcases/006-container.json | 8 + .../matching-tcases/legacy-testsuite.xml | 1 + .../matching-tcases/shared-attachment.txt | 1 + .../missing-attachments/001-result.json | 14 + .../missing-attachments/002-result.json | 14 + .../missing-attachments/003-result.json | 14 + .../missing-attachments/004-result.json | 17 ++ .../missing-attachments/005-result.json | 14 + .../existing-attachment.txt | 1 + .../allure/missing-tcases/001-result.json | 13 + .../allure/missing-tcases/002-result.json | 13 + .../allure/missing-tcases/003-result.json | 13 + .../allure/missing-tcases/004-result.json | 16 ++ .../allure/missing-tcases/005-result.json | 13 + .../allure/without-markers/001-result.json | 8 + .../allure/without-markers/002-result.json | 11 + .../allure/without-markers/003-result.json | 8 + src/tests/result-upload.spec.ts | 95 ++++-- .../ResultUploadCommandHandler.ts | 12 +- src/utils/result-upload/ResultUploader.ts | 32 +++ src/utils/result-upload/allureParser.ts | 233 +++++++++++++++ 36 files changed, 1025 insertions(+), 53 deletions(-) create mode 120000 AGENTS.md create mode 100644 src/tests/allure-parsing.spec.ts create mode 100644 src/tests/fixtures/allure/empty-tsuite/001-result.json create mode 100644 src/tests/fixtures/allure/empty-tsuite/002-container.json create mode 100644 src/tests/fixtures/allure/invalid-results/001-result.json create mode 100644 src/tests/fixtures/allure/invalid-results/003-result.json create mode 100644 src/tests/fixtures/allure/matching-tcases/001-result.json create mode 100644 src/tests/fixtures/allure/matching-tcases/002-result.json create mode 100644 src/tests/fixtures/allure/matching-tcases/003-result.json create mode 100644 src/tests/fixtures/allure/matching-tcases/004-result.json create mode 100644 src/tests/fixtures/allure/matching-tcases/005-result.json create mode 100644 src/tests/fixtures/allure/matching-tcases/006-container.json create mode 100644 src/tests/fixtures/allure/matching-tcases/legacy-testsuite.xml create mode 100644 src/tests/fixtures/allure/matching-tcases/shared-attachment.txt create mode 100644 src/tests/fixtures/allure/missing-attachments/001-result.json create mode 100644 src/tests/fixtures/allure/missing-attachments/002-result.json create mode 100644 src/tests/fixtures/allure/missing-attachments/003-result.json create mode 100644 src/tests/fixtures/allure/missing-attachments/004-result.json create mode 100644 src/tests/fixtures/allure/missing-attachments/005-result.json create mode 100644 src/tests/fixtures/allure/missing-attachments/existing-attachment.txt create mode 100644 src/tests/fixtures/allure/missing-tcases/001-result.json create mode 100644 src/tests/fixtures/allure/missing-tcases/002-result.json create mode 100644 src/tests/fixtures/allure/missing-tcases/003-result.json create mode 100644 src/tests/fixtures/allure/missing-tcases/004-result.json create mode 100644 src/tests/fixtures/allure/missing-tcases/005-result.json create mode 100644 src/tests/fixtures/allure/without-markers/001-result.json create mode 100644 src/tests/fixtures/allure/without-markers/002-result.json create mode 100644 src/tests/fixtures/allure/without-markers/003-result.json create mode 100644 src/utils/result-upload/allureParser.ts diff --git a/AGENTS.md b/AGENTS.md new file mode 120000 index 0000000..681311e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index 251a9d6..b587cb9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Project Overview -QAS CLI (`qas-cli`) is a Node.js CLI tool for uploading test automation results (JUnit XML / Playwright JSON) to [QA Sphere](https://qasphere.com/). It matches test case markers (e.g., `PRJ-123`) in report files to QA Sphere test cases, creates or reuses test runs, and uploads results with optional attachments. +QAS CLI (`qas-cli`) is a Node.js CLI tool for uploading test automation results (JUnit XML / Playwright JSON / Allure results directories) to [QA Sphere](https://qasphere.com/). It matches test case references (e.g., `PRJ-123` markers or Allure TMS links) to QA Sphere test cases, creates or reuses test runs, and uploads results with optional attachments. ## Commands @@ -29,7 +29,7 @@ Node.js compatibility tests: `cd mnode-test && ./docker-test.sh` (requires Docke ### Entry Point & CLI Framework - `src/bin/qasphere.ts` — Entry point (`#!/usr/bin/env node`). Validates Node version, delegates to `run()`. -- `src/commands/main.ts` — Yargs setup. Registers two commands (`junit-upload`, `playwright-json-upload`) as instances of the same `ResultUploadCommandModule` class. +- `src/commands/main.ts` — Yargs setup. Registers three commands (`junit-upload`, `playwright-json-upload`, `allure-upload`) as instances of the same `ResultUploadCommandModule` class. - `src/commands/resultUpload.ts` — `ResultUploadCommandModule` defines CLI options shared by both commands. Loads env vars, then delegates to `ResultUploadCommandHandler`. ### Core Upload Pipeline (src/utils/result-upload/) @@ -43,7 +43,7 @@ The upload flow has two stages handled by two classes, with a shared `MarkerPars - Also exports a standalone `formatMarker()` function used by parsers 2. **`ResultUploadCommandHandler`** — Orchestrates the overall flow: - - Parses report files using the appropriate parser (JUnit XML or Playwright JSON) + - Parses report inputs using the appropriate parser (JUnit XML file, Playwright JSON file, or Allure results directory) - Detects project code from test case names via `MarkerParser` (or from `--run-url`) - Creates a new test run (or reuses an existing one if title conflicts) - Delegates actual result uploading to `ResultUploader` @@ -57,6 +57,7 @@ The upload flow has two stages handled by two classes, with a shared `MarkerPars - `junitXmlParser.ts` — Parses JUnit XML via `xml2js` + Zod validation. Extracts attachments from `[[ATTACHMENT|path]]` markers in system-out/failure/error/skipped elements. - `playwrightJsonParser.ts` — Parses Playwright JSON report. Supports two test case linking methods: (1) test annotations with `type: "test case"` and URL description, (2) marker in test name. Handles nested suites recursively. +- `allureParser.ts` — Parses Allure 2 JSON results directories (`*-result.json` files only). Supports test case linking via TMS links (`type: "tms"`) or marker in test name, maps Allure statuses to QA Sphere result statuses, and resolves attachments via `attachments[].source`. - `types.ts` — Shared `TestCaseResult` and `Attachment` interfaces used by both parsers. ### API Layer (src/api/) @@ -78,10 +79,11 @@ Composable fetch wrappers using higher-order functions: Tests use **Vitest** with **MSW** (Mock Service Worker) for API mocking. Test files are in `src/tests/`: -- `result-upload.spec.ts` — Integration tests for the full upload flow (both JUnit and Playwright), with MSW intercepting all API calls. Includes hyphenless and CamelCase marker tests (JUnit only) +- `result-upload.spec.ts` — Integration tests for the full upload flow (JUnit, Playwright, and Allure), with MSW intercepting all API calls. Includes hyphenless and CamelCase marker tests (JUnit only) - `marker-parser.spec.ts` — Unit tests for `MarkerParser` (detection, extraction, matching across all marker formats and command types) - `junit-xml-parsing.spec.ts` — Unit tests for JUnit XML parser - `playwright-json-parsing.spec.ts` — Unit tests for Playwright JSON parser +- `allure-parsing.spec.ts` — Unit tests for Allure parser - `template-string-processing.spec.ts` — Unit tests for run name template processing Test fixtures live in `src/tests/fixtures/` (XML files, JSON files, and mock test case data). diff --git a/README.md b/README.md index acca4d4..2e5932d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ The QAS CLI is a command-line tool for submitting your test automation results to [QA Sphere](https://qasphere.com/). It provides the most efficient way to collect and report test results from your test automation workflow, CI/CD pipeline, and build servers. -The tool can upload test case results from JUnit XML and Playwright JSON files to QA Sphere test runs by matching test case names (mentions of special markers) to QA Sphere's test cases. +The tool can upload test case results from JUnit XML files, Playwright JSON files, and Allure result directories to QA Sphere test runs by matching test case references to QA Sphere test cases. ## Installation @@ -39,7 +39,7 @@ Verify installation: `qasphere --version` The CLI requires the following variables to be defined: - `QAS_TOKEN` - QA Sphere API token (see [docs](https://docs.qasphere.com/api/authentication) if you need help generating one) -- `QAS_URL` - Base URL of your QA Sphere instance (e.g., https://qas.eu2.qasphere.com) +- `QAS_URL` - Base URL of your QA Sphere instance (e.g., `https://qas.eu2.qasphere.com`) These variables could be defined: @@ -59,9 +59,9 @@ QAS_URL=https://qas.eu1.qasphere.com # QAS_URL=https://qas.eu1.qasphere.com ``` -## Commands: `junit-upload`, `playwright-json-upload` +## Commands: `junit-upload`, `playwright-json-upload`, `allure-upload` -The `junit-upload` and `playwright-json-upload` commands upload test results from JUnit XML and Playwright JSON reports to QA Sphere respectively. +The `junit-upload`, `playwright-json-upload`, and `allure-upload` commands upload test results to QA Sphere. There are two modes for uploading results using the commands: @@ -70,6 +70,7 @@ There are two modes for uploading results using the commands: ### Options +- `` / `` - Input paths. Use report files for `junit-upload` and `playwright-json-upload`, and Allure results directories for `allure-upload` - `-r`/`--run-url` - Upload results to an existing test run - `--project-code`, `--run-name`, `--create-tcases` - Create a new test run and upload results to it - `--project-code` - Project code for creating new test run. It can also be auto detected from test case markers in the results, but this is not fully reliable, so it is recommended to specify the project code explicitly @@ -170,14 +171,28 @@ Ensure the required environment variables are defined before running these comma This will exclude stdout from passed tests while still including it for failed, blocked, or skipped tests. 10. Skip both stdout and stderr for passed tests: + ```bash qasphere junit-upload --skip-report-stdout on-success --skip-report-stderr on-success ./test-results.xml ``` + This is useful when you have verbose logging in tests but only want to see output for failures. +11. Upload Allure results from a directory: + + ```bash + qasphere allure-upload -r https://qas.eu1.qasphere.com/project/P1/run/23 ./allure-results + ``` + +12. Continue Allure upload when some `*-result.json` files are malformed (skip invalid files): + + ```bash + qasphere allure-upload --force -r https://qas.eu1.qasphere.com/project/P1/run/23 ./allure-results + ``` + ## Test Report Requirements -The QAS CLI maps test results from your reports (JUnit XML or Playwright JSON) to corresponding test cases in QA Sphere using test case markers. If a test result lacks a valid marker, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to skip unmatched results. +The QAS CLI maps test results from your reports (JUnit XML, Playwright JSON, or Allure) to corresponding test cases in QA Sphere. If a test result lacks a valid marker/reference, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to skip unmatched results. ### JUnit XML @@ -237,6 +252,18 @@ Playwright JSON reports support two methods for referencing test cases (checked 2. **Hyphenated Marker in Name** - Include the `PROJECT-SEQUENCE` marker in the test name (same format as JUnit XML format 1). Hyphenless markers are **not** supported for Playwright JSON +### Allure + +Allure results use one `*-result.json` file per test in a results directory. `allure-upload` matches test cases using: + +1. **TMS links (Recommended)** - `links[]` entries with: + - `type`: `"tms"` + - `url`: QA Sphere test case URL, e.g. `https://qas.eu1.qasphere.com/project/PRJ/tcase/123` +2. **TMS link name fallback** - If `url` is not a QA Sphere URL, a marker in `links[].name` is used (for example `PRJ-123`) +3. **Test case marker in name** - Marker in `name` field (same `PROJECT-SEQUENCE` format as JUnit XML) + +Only Allure 2 JSON (`*-result.json`) is supported. Legacy Allure 1 XML files are ignored. + ## Development (for those who want to contribute to the tool) 1. Install and build: `npm install && npm run build && npm link` diff --git a/src/commands/main.ts b/src/commands/main.ts index 98ed8a3..05af261 100644 --- a/src/commands/main.ts +++ b/src/commands/main.ts @@ -13,6 +13,7 @@ Required variables: ${qasEnvs.join(', ')} ) .command(new ResultUploadCommandModule('junit-upload')) .command(new ResultUploadCommandModule('playwright-json-upload')) + .command(new ResultUploadCommandModule('allure-upload')) .demandCommand(1, '') .help('h') .alias('h', 'help') diff --git a/src/commands/resultUpload.ts b/src/commands/resultUpload.ts index 78b6687..0a530e8 100644 --- a/src/commands/resultUpload.ts +++ b/src/commands/resultUpload.ts @@ -10,11 +10,19 @@ import { const commandTypeDisplayStrings: Record = { 'junit-upload': 'JUnit XML', 'playwright-json-upload': 'Playwright JSON', + 'allure-upload': 'Allure', } -const commandTypeFileExtensions: Record = { - 'junit-upload': 'xml', - 'playwright-json-upload': 'json', +const commandTypeInputKinds: Record = { + 'junit-upload': 'files', + 'playwright-json-upload': 'files', + 'allure-upload': 'directories', +} + +const commandTypeExampleInputs: Record = { + 'junit-upload': './test-results.xml', + 'playwright-json-upload': './test-results.json', + 'allure-upload': './allure-results', } export class ResultUploadCommandModule implements CommandModule { @@ -25,10 +33,19 @@ export class ResultUploadCommandModule implements CommandModule { + argv.positional('files', { + describe: + this.type === 'allure-upload' + ? 'One or more Allure results directories' + : 'One or more test report files', + type: 'string', + array: true, + }) + argv.options({ 'run-url': { alias: 'r', @@ -83,20 +100,20 @@ export class ResultUploadCommandModule implements CommandModule = {}) => ({ + name: 'Sample test', + status: 'passed', + uuid: 'result-uuid', + start: 1000, + stop: 1200, + ...overrides, +}) + +const createTempAllureDir = async (files: Record) => { + const dir = await mkdtemp(join(tmpdir(), 'qas-allure-fixture-')) + tempDirsToCleanup.push(dir) + + await Promise.all( + Object.entries(files).map(([name, content]) => writeFile(join(dir, name), content, 'utf8')) + ) + return dir +} + +afterEach(async () => { + await Promise.all( + tempDirsToCleanup.splice(0).map((dir) => rm(dir, { recursive: true, force: true })) + ) +}) + +describe('Allure parsing', () => { + test('Should parse matching directory with marker extraction and status mapping', async () => { + const dir = `${allureBasePath}/matching-tcases` + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + allowPartialParse: false, + }) + + expect(testcases).toHaveLength(5) + expect(testcases[0].name).toBe('TEST-002: Test cart') + expect(testcases[1].name).toBe('TEST-003: Test checkout') + expect(testcases[2].name).toBe('TEST-004: About page content TEST-004') + expect(testcases[3].name).toBe('TEST-006: Navigation bar items') + expect(testcases[4].name).toBe('TEST-007: Welcome page content (updated)') + + expect(testcases[0].folder).toBe('ui.cart.spec.ts') + expect(testcases[1].folder).toBe('ui.cart.spec.ts') + expect(testcases[2].folder).toBe('ui.contents.spec.ts') + expect(testcases[3].folder).toBe('ui.contents.spec.ts') + expect(testcases[4].folder).toBe('') + + expect(testcases[0].timeTaken).toBe(500) + expect(testcases[3].status).toBe('failed') + expect(testcases[4].status).toBe('open') + expect(testcases[3].message).toContain('AssertionError: navbar items mismatch') + expect(testcases[3].message).toContain('Traceback line 2') + + testcases.forEach((tcase) => { + expect(tcase.attachments).toHaveLength(1) + expect(tcase.attachments[0].buffer).not.toBeNull() + expect(tcase.attachments[0].error).toBeNull() + }) + }) + + test('Should map broken and skipped statuses correctly', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ name: 'TEST-100 Broken test', status: 'broken' }) + ), + '002-result.json': JSON.stringify( + makeResult({ name: 'TEST-101 Skipped test', status: 'skipped' }) + ), + }) + + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(2) + expect(testcases[0].status).toBe('blocked') + expect(testcases[1].status).toBe('skipped') + }) + + test('Should honor skip-report options for message and trace blocks', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-200 Passed with status details', + statusDetails: { + message: 'stdout-like text', + trace: 'stderr-like text', + }, + }) + ), + }) + + const skippedOnSuccess = await parseAllureResults(dir, dir, { + skipStdout: 'on-success', + skipStderr: 'on-success', + }) + expect(skippedOnSuccess[0].message).toBe('') + + const neverSkip = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(neverSkip[0].message).toContain('stdout-like text') + expect(neverSkip[0].message).toContain('stderr-like text') + }) + + test('Should apply folder priority suite > parentSuite > feature > package', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-301 has suite and parentSuite', + labels: [ + { name: 'parentSuite', value: 'parent-folder' }, + { name: 'suite', value: 'suite-folder' }, + ], + }) + ), + '002-result.json': JSON.stringify( + makeResult({ + name: 'TEST-302 has parentSuite and feature', + labels: [ + { name: 'feature', value: 'feature-folder' }, + { name: 'parentSuite', value: 'parent-folder-2' }, + ], + }) + ), + '003-result.json': JSON.stringify( + makeResult({ + name: 'TEST-303 has feature and package', + labels: [ + { name: 'package', value: 'package-folder' }, + { name: 'feature', value: 'feature-folder-2' }, + ], + }) + ), + '004-result.json': JSON.stringify( + makeResult({ + name: 'TEST-304 has package', + labels: [{ name: 'package', value: 'package-only-folder' }], + }) + ), + '005-result.json': JSON.stringify(makeResult({ name: 'TEST-305 has no labels', labels: [] })), + }) + + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(5) + expect(testcases[0].folder).toBe('suite-folder') + expect(testcases[1].folder).toBe('parent-folder-2') + expect(testcases[2].folder).toBe('feature-folder-2') + expect(testcases[3].folder).toBe('package-only-folder') + expect(testcases[4].folder).toBe('') + }) + + test('Should keep attachment errors without crashing parse', async () => { + const dir = `${allureBasePath}/missing-attachments` + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(testcases).toHaveLength(5) + const erroredAttachments = testcases + .flatMap((t) => t.attachments) + .filter((a) => a.error !== null) + expect(erroredAttachments).toHaveLength(1) + expect(erroredAttachments[0].buffer).toBeNull() + expect(erroredAttachments[0].filename).toBe('missing-attachment.txt') + }) + + test('Should fail by default for malformed or schema-invalid files', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-002 Valid result', + }) + ), + '002-result.json': `{ + "name": "Malformed fixture", + "status": "passed", + "uuid": "malformed-uuid", + "start": 1, + "stop": 2,`, + }) + await expect( + parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + allowPartialParse: false, + }) + ).rejects.toThrowError() + }) + + test('Should skip malformed or schema-invalid files when partial parsing is allowed', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-002 Valid result', + }) + ), + '002-result.json': `{ + "name": "Malformed fixture", + "status": "passed", + "uuid": "malformed-uuid", + "start": 1, + "stop": 2,`, + '003-result.json': JSON.stringify({ + name: 'Schema invalid fixture', + uuid: 'schema-invalid-uuid', + start: 10, + stop: 20, + }), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + allowPartialParse: true, + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toContain('TEST-002') + }) + + test('Should prioritize marker extraction as TMS URL > TMS link name > test name', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-404 marker in test name', + links: [ + { + type: 'tms', + url: 'https://qas.eu1.qasphere.com/project/TEST/tcase/2', + name: 'TEST-003', + }, + ], + }) + ), + '002-result.json': JSON.stringify( + makeResult({ + name: 'TEST-405 marker in test name', + links: [ + { + type: 'tms', + url: 'https://external.example.com/tms/entry', + name: 'TEST-006', + }, + ], + }) + ), + }) + + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + + expect(testcases).toHaveLength(2) + expect(testcases[0].name).toBe('TEST-002: TEST-404 marker in test name') + expect(testcases[1].name).toBe('TEST-006: TEST-405 marker in test name') + }) +}) diff --git a/src/tests/fixtures/allure/empty-tsuite/001-result.json b/src/tests/fixtures/allure/empty-tsuite/001-result.json new file mode 100644 index 0000000..4ca5ec5 --- /dev/null +++ b/src/tests/fixtures/allure/empty-tsuite/001-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-002 Test cart", + "status": "passed", + "uuid": "empty-tsuite-001", + "start": 1700000040000, + "stop": 1700000040100, + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/empty-tsuite/002-container.json b/src/tests/fixtures/allure/empty-tsuite/002-container.json new file mode 100644 index 0000000..0aec70e --- /dev/null +++ b/src/tests/fixtures/allure/empty-tsuite/002-container.json @@ -0,0 +1,8 @@ +{ + "uuid": "empty-tsuite-container-002", + "children": [], + "befores": [], + "afters": [], + "start": 1700000041000, + "stop": 1700000041100 +} diff --git a/src/tests/fixtures/allure/invalid-results/001-result.json b/src/tests/fixtures/allure/invalid-results/001-result.json new file mode 100644 index 0000000..b9133d4 --- /dev/null +++ b/src/tests/fixtures/allure/invalid-results/001-result.json @@ -0,0 +1,7 @@ +{ + "name": "TEST-002 Test cart from partial directory", + "status": "passed", + "uuid": "invalid-results-001", + "start": 1700000050000, + "stop": 1700000050200 +} diff --git a/src/tests/fixtures/allure/invalid-results/003-result.json b/src/tests/fixtures/allure/invalid-results/003-result.json new file mode 100644 index 0000000..0bdb266 --- /dev/null +++ b/src/tests/fixtures/allure/invalid-results/003-result.json @@ -0,0 +1,6 @@ +{ + "name": "Schema invalid fixture", + "uuid": "invalid-results-003", + "start": 1700000052000, + "stop": 1700000052300 +} diff --git a/src/tests/fixtures/allure/matching-tcases/001-result.json b/src/tests/fixtures/allure/matching-tcases/001-result.json new file mode 100644 index 0000000..8513fa7 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/001-result.json @@ -0,0 +1,27 @@ +{ + "name": "Test cart", + "status": "passed", + "uuid": "matching-001", + "start": 1700000000000, + "stop": 1700000000500, + "statusDetails": {}, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ], + "links": [ + { + "type": "tms", + "url": "https://qas.eu1.qasphere.com/project/TEST/tcase/2" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/002-result.json b/src/tests/fixtures/allure/matching-tcases/002-result.json new file mode 100644 index 0000000..29628d1 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/002-result.json @@ -0,0 +1,28 @@ +{ + "name": "Test checkout", + "status": "passed", + "uuid": "matching-002", + "start": 1700000001000, + "stop": 1700000001600, + "statusDetails": null, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "parentSuite", + "value": "ui.cart.spec.ts" + } + ], + "links": [ + { + "type": "tms", + "url": "https://external.example.com/test-case/3", + "name": "TEST-003" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/003-result.json b/src/tests/fixtures/allure/matching-tcases/003-result.json new file mode 100644 index 0000000..467d963 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/003-result.json @@ -0,0 +1,21 @@ +{ + "name": "About page content TEST-004", + "status": "passed", + "uuid": "matching-003", + "start": 1700000002000, + "stop": 1700000002800, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "feature", + "value": "ui.contents.spec.ts" + } + ], + "links": [] +} diff --git a/src/tests/fixtures/allure/matching-tcases/004-result.json b/src/tests/fixtures/allure/matching-tcases/004-result.json new file mode 100644 index 0000000..e8dded2 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/004-result.json @@ -0,0 +1,30 @@ +{ + "name": "Navigation bar items", + "status": "failed", + "uuid": "matching-004", + "start": 1700000003000, + "stop": 1700000004200, + "statusDetails": { + "message": "AssertionError: navbar items mismatch", + "trace": "Traceback line 1\nTraceback line 2" + }, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [ + { + "name": "package", + "value": "ui.contents.spec.ts" + } + ], + "links": [ + { + "type": "tms", + "url": "https://qas.eu1.qasphere.com/project/TEST/tcase/6" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/005-result.json b/src/tests/fixtures/allure/matching-tcases/005-result.json new file mode 100644 index 0000000..16f04ab --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/005-result.json @@ -0,0 +1,24 @@ +{ + "name": "Welcome page content (updated)", + "status": "unknown", + "uuid": "matching-005", + "start": 1700000005000, + "stop": 1700000006200, + "statusDetails": { + "message": "No status was produced by adapter" + }, + "attachments": [ + { + "name": "attachment", + "source": "shared-attachment.txt", + "type": "text/plain" + } + ], + "labels": [], + "links": [ + { + "type": "tms", + "url": "https://qas.eu1.qasphere.com/project/TEST/tcase/7" + } + ] +} diff --git a/src/tests/fixtures/allure/matching-tcases/006-container.json b/src/tests/fixtures/allure/matching-tcases/006-container.json new file mode 100644 index 0000000..a2fd6be --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/006-container.json @@ -0,0 +1,8 @@ +{ + "uuid": "matching-container-006", + "children": ["matching-001"], + "befores": [], + "afters": [], + "start": 1700000000000, + "stop": 1700000000100 +} diff --git a/src/tests/fixtures/allure/matching-tcases/legacy-testsuite.xml b/src/tests/fixtures/allure/matching-tcases/legacy-testsuite.xml new file mode 100644 index 0000000..aa0e71f --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/legacy-testsuite.xml @@ -0,0 +1 @@ + diff --git a/src/tests/fixtures/allure/matching-tcases/shared-attachment.txt b/src/tests/fixtures/allure/matching-tcases/shared-attachment.txt new file mode 100644 index 0000000..9ac09f2 --- /dev/null +++ b/src/tests/fixtures/allure/matching-tcases/shared-attachment.txt @@ -0,0 +1 @@ +Allure fixture attachment content. diff --git a/src/tests/fixtures/allure/missing-attachments/001-result.json b/src/tests/fixtures/allure/missing-attachments/001-result.json new file mode 100644 index 0000000..044aae7 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/001-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-002 Test cart", + "status": "passed", + "uuid": "missing-attach-001", + "start": 1700000020000, + "stop": 1700000020200, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/002-result.json b/src/tests/fixtures/allure/missing-attachments/002-result.json new file mode 100644 index 0000000..679e995 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/002-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-003 Test checkout", + "status": "passed", + "uuid": "missing-attach-002", + "start": 1700000021000, + "stop": 1700000021350, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/003-result.json b/src/tests/fixtures/allure/missing-attachments/003-result.json new file mode 100644 index 0000000..dc2d74f --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/003-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-004 About page content", + "status": "passed", + "uuid": "missing-attach-003", + "start": 1700000022000, + "stop": 1700000022500, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/004-result.json b/src/tests/fixtures/allure/missing-attachments/004-result.json new file mode 100644 index 0000000..68b92c2 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/004-result.json @@ -0,0 +1,17 @@ +{ + "name": "TEST-006 Navigation bar items", + "status": "failed", + "uuid": "missing-attach-004", + "start": 1700000023000, + "stop": 1700000023300, + "statusDetails": { + "message": "AssertionError: navigation items mismatch" + }, + "attachments": [ + { + "name": "attachment", + "source": "existing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/005-result.json b/src/tests/fixtures/allure/missing-attachments/005-result.json new file mode 100644 index 0000000..09e5255 --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/005-result.json @@ -0,0 +1,14 @@ +{ + "name": "TEST-007 Welcome page content", + "status": "passed", + "uuid": "missing-attach-005", + "start": 1700000024000, + "stop": 1700000024700, + "attachments": [ + { + "name": "attachment", + "source": "missing-attachment.txt", + "type": "text/plain" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-attachments/existing-attachment.txt b/src/tests/fixtures/allure/missing-attachments/existing-attachment.txt new file mode 100644 index 0000000..102ee6a --- /dev/null +++ b/src/tests/fixtures/allure/missing-attachments/existing-attachment.txt @@ -0,0 +1 @@ +existing attachment content diff --git a/src/tests/fixtures/allure/missing-tcases/001-result.json b/src/tests/fixtures/allure/missing-tcases/001-result.json new file mode 100644 index 0000000..1d149fa --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/001-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-002 Test cart", + "status": "passed", + "uuid": "missing-001", + "start": 1700000010000, + "stop": 1700000010200, + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/002-result.json b/src/tests/fixtures/allure/missing-tcases/002-result.json new file mode 100644 index 0000000..61eb47d --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/002-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-003 Test checkout", + "status": "passed", + "uuid": "missing-002", + "start": 1700000011000, + "stop": 1700000011350, + "labels": [ + { + "name": "suite", + "value": "ui.cart.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/003-result.json b/src/tests/fixtures/allure/missing-tcases/003-result.json new file mode 100644 index 0000000..c18d456 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/003-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-004 About page content", + "status": "passed", + "uuid": "missing-003", + "start": 1700000012000, + "stop": 1700000012500, + "labels": [ + { + "name": "feature", + "value": "ui.contents.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/004-result.json b/src/tests/fixtures/allure/missing-tcases/004-result.json new file mode 100644 index 0000000..a0eca24 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/004-result.json @@ -0,0 +1,16 @@ +{ + "name": "TEST-006 Navigation bar items", + "status": "failed", + "uuid": "missing-004", + "start": 1700000013000, + "stop": 1700000013300, + "statusDetails": { + "message": "AssertionError: navigation items mismatch" + }, + "labels": [ + { + "name": "package", + "value": "ui.contents.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/missing-tcases/005-result.json b/src/tests/fixtures/allure/missing-tcases/005-result.json new file mode 100644 index 0000000..e7bf9c8 --- /dev/null +++ b/src/tests/fixtures/allure/missing-tcases/005-result.json @@ -0,0 +1,13 @@ +{ + "name": "TEST-999 This test does not exist in run", + "status": "passed", + "uuid": "missing-005", + "start": 1700000014000, + "stop": 1700000014700, + "labels": [ + { + "name": "suite", + "value": "ui.unknown.spec.ts" + } + ] +} diff --git a/src/tests/fixtures/allure/without-markers/001-result.json b/src/tests/fixtures/allure/without-markers/001-result.json new file mode 100644 index 0000000..bedabad --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/001-result.json @@ -0,0 +1,8 @@ +{ + "name": "The cart is still filled after refreshing the page", + "status": "passed", + "uuid": "without-markers-001", + "start": 1700000030000, + "stop": 1700000030200, + "labels": [] +} diff --git a/src/tests/fixtures/allure/without-markers/002-result.json b/src/tests/fixtures/allure/without-markers/002-result.json new file mode 100644 index 0000000..9054ee3 --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/002-result.json @@ -0,0 +1,11 @@ +{ + "name": "The checkout should finish successfully", + "status": "failed", + "uuid": "without-markers-002", + "start": 1700000031000, + "stop": 1700000031800, + "statusDetails": { + "message": "AssertionError: expected true to be false" + }, + "labels": [] +} diff --git a/src/tests/fixtures/allure/without-markers/003-result.json b/src/tests/fixtures/allure/without-markers/003-result.json new file mode 100644 index 0000000..fcb4294 --- /dev/null +++ b/src/tests/fixtures/allure/without-markers/003-result.json @@ -0,0 +1,8 @@ +{ + "name": "The cart is still filled after refreshing the page", + "status": "passed", + "uuid": "without-markers-003", + "start": 1700000032000, + "stop": 1700000032400, + "labels": [] +} diff --git a/src/tests/result-upload.spec.ts b/src/tests/result-upload.spec.ts index c9e4478..c5c9b20 100644 --- a/src/tests/result-upload.spec.ts +++ b/src/tests/result-upload.spec.ts @@ -159,18 +159,45 @@ const cleanupGeneratedMappingFiles = (existingMappingFiles?: Set) => { }) } +interface TestFileType { + name: string + command: 'junit-upload' | 'playwright-json-upload' | 'allure-upload' + dataBasePath: string + inputType: 'file' | 'directory' + fileExtension?: string +} + +const fixtureInputPath = (fileType: TestFileType, fixtureName: string) => { + if (fileType.inputType === 'directory') { + return `${fileType.dataBasePath}/${fixtureName}` + } + return `${fileType.dataBasePath}/${fixtureName}.${fileType.fileExtension}` +} + const fileTypes = [ { name: 'JUnit XML', command: 'junit-upload', dataBasePath: './src/tests/fixtures/junit-xml', fileExtension: 'xml', + inputType: 'file', }, { name: 'Playwright JSON', command: 'playwright-json-upload', dataBasePath: './src/tests/fixtures/playwright-json', fileExtension: 'json', + inputType: 'file', + }, +] as const satisfies readonly TestFileType[] + +const fileTypesWithAllure: TestFileType[] = [ + ...fileTypes, + { + name: 'Allure', + command: 'allure-upload', + dataBasePath: './src/tests/fixtures/allure', + inputType: 'directory', }, ] @@ -234,14 +261,14 @@ describe('CamelCase test case markers (Go/Java style)', () => { }) }) -fileTypes.forEach((fileType) => { +fileTypesWithAllure.forEach((fileType) => { describe(`Uploading ${fileType.name} files`, () => { describe('Argument parsing', () => { test('Passing correct Run URL pattern should result in success', async () => { const patterns = [ - `${fileType.command} --run-url ${runURL} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}/ ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}/tcase/1 ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, + `${fileType.command} --run-url ${runURL} ${fixtureInputPath(fileType, 'matching-tcases')}`, + `${fileType.command} -r ${runURL}/ ${fixtureInputPath(fileType, 'matching-tcases')}`, + `${fileType.command} -r ${runURL}/tcase/1 ${fixtureInputPath(fileType, 'matching-tcases')}`, ] for (const pattern of patterns) { @@ -258,7 +285,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(1) await run( - `${fileType.command} -r ${qasHost}/project/${projectCode}/run/${runId} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${qasHost}/project/${projectCode}/run/${runId} ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(5) // 5 results total @@ -266,8 +293,8 @@ fileTypes.forEach((fileType) => { test('Passing incorrect Run URL pattern should result in failure', async () => { const patterns = [ - `${fileType.command} -r ${qasHost}/projects/${projectCode}/runs/${runId} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, - `${fileType.command} -r ${runURL}abc/tcase/1 ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}`, + `${fileType.command} -r ${qasHost}/projects/${projectCode}/runs/${runId} ${fixtureInputPath(fileType, 'matching-tcases')}`, + `${fileType.command} -r ${runURL}abc/tcase/1 ${fixtureInputPath(fileType, 'matching-tcases')}`, ] for (const pattern of patterns) { @@ -293,7 +320,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(2) await run( - `${fileType.command} -r ${runURL} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(3) // 5 results total @@ -303,9 +330,7 @@ fileTypes.forEach((fileType) => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await expect( - run( - `${fileType.command} -r ${runURL} ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` - ) + run(`${fileType.command} -r ${runURL} ${fixtureInputPath(fileType, 'missing-tcases')}`) ).rejects.toThrowError() expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(0) @@ -316,7 +341,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(3) await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${fixtureInputPath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(2) // 4 results total @@ -326,7 +351,7 @@ fileTypes.forEach((fileType) => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await run( - `${fileType.command} -r ${runURL} --ignore-unmatched ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --ignore-unmatched ${fixtureInputPath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(1) // 4 results total @@ -337,7 +362,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(2) await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension} ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${fixtureInputPath(fileType, 'missing-tcases')} ${fixtureInputPath(fileType, 'missing-tcases')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(4) // 8 results total @@ -347,7 +372,7 @@ fileTypes.forEach((fileType) => { const numFileUploadCalls = countFileUploadApiCalls() const numResultUploadCalls = countResultUploadApiCalls() await run( - `${fileType.command} -r ${runURL} --force ${fileType.dataBasePath}/empty-tsuite.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --force ${fixtureInputPath(fileType, 'empty-tsuite')}` ) expect(numFileUploadCalls()).toBe(0) expect(numResultUploadCalls()).toBe(1) // 1 result total @@ -360,7 +385,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(3) await run( - `${fileType.command} -r ${runURL} --attachments ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numFileUploadCalls()).toBe(5) expect(numResultUploadCalls()).toBe(2) // 5 results total @@ -370,7 +395,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() await expect( run( - `${fileType.command} -r ${runURL} --attachments ${fileType.dataBasePath}/missing-attachments.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments ${fixtureInputPath(fileType, 'missing-attachments')}` ) ).rejects.toThrow() expect(numFileUploadCalls()).toBe(0) @@ -381,7 +406,7 @@ fileTypes.forEach((fileType) => { const numResultUploadCalls = countResultUploadApiCalls() setMaxResultsInRequest(1) await run( - `${fileType.command} -r ${runURL} --attachments --force ${fileType.dataBasePath}/missing-attachments.${fileType.fileExtension}` + `${fileType.command} -r ${runURL} --attachments --force ${fixtureInputPath(fileType, 'missing-attachments')}` ) expect(numFileUploadCalls()).toBe(4) expect(numResultUploadCalls()).toBe(5) // 5 results total @@ -402,7 +427,7 @@ fileTypes.forEach((fileType) => { try { // This should create a new run since no --run-url is specified await run( - `${fileType.command} --run-name "CI Build {env:TEST_BUILD_NUMBER}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "CI Build {env:TEST_BUILD_NUMBER}" ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe('CI Build 456') @@ -423,7 +448,7 @@ fileTypes.forEach((fileType) => { const expectedDay = String(now.getDate()).padStart(2, '0') await run( - `${fileType.command} --run-name "Test Run {YYYY}-{MM}-{DD}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "Test Run {YYYY}-{MM}-{DD}" ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe(`Test Run ${expectedYear}-${expectedMonth}-${expectedDay}`) @@ -435,7 +460,7 @@ fileTypes.forEach((fileType) => { try { await run( - `${fileType.command} --run-name "{env:TEST_PROJECT} - {YYYY}/{MM}" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "{env:TEST_PROJECT} - {YYYY}/{MM}" ${fixtureInputPath(fileType, 'matching-tcases')}` ) const now = new Date() @@ -458,7 +483,7 @@ fileTypes.forEach((fileType) => { createRunTitleConflict = true await run( - `${fileType.command} --run-name "duplicate run title" ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --run-name "duplicate run title" ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(lastCreatedRunTitle).toBe('duplicate run title') @@ -467,9 +492,7 @@ fileTypes.forEach((fileType) => { }) test('Should use default name template when --run-name is not specified', async () => { - await run( - `${fileType.command} ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` - ) + await run(`${fileType.command} ${fixtureInputPath(fileType, 'matching-tcases')}`) // Should use default format: "Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}" expect(lastCreatedRunTitle).toContain('Automated test run - ') @@ -507,7 +530,7 @@ fileTypes.forEach((fileType) => { } await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${fixtureInputPath(fileType, 'without-markers')}` ) expect(numCreateTCasesCalls()).toBe(1) expect(numResultUploadCalls()).toBe(3) // 3 results total @@ -544,7 +567,7 @@ fileTypes.forEach((fileType) => { } await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/without-markers.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${fixtureInputPath(fileType, 'without-markers')}` ) expect(numCreateTCasesCalls()).toBe(1) expect(numResultUploadCalls()).toBe(3) // 3 results total @@ -556,7 +579,7 @@ fileTypes.forEach((fileType) => { setMaxResultsInRequest(1) await run( - `${fileType.command} --project-code ${projectCode} --create-tcases ${fileType.dataBasePath}/matching-tcases.${fileType.fileExtension}` + `${fileType.command} --project-code ${projectCode} --create-tcases ${fixtureInputPath(fileType, 'matching-tcases')}` ) expect(numCreateTCasesCalls()).toBe(0) expect(numResultUploadCalls()).toBe(5) // 5 results total @@ -564,3 +587,19 @@ fileTypes.forEach((fileType) => { }) }) }) + +describe('Allure invalid result file handling', () => { + test('Should fail when directory contains malformed files without --force', async () => { + const numResultUploadCalls = countResultUploadApiCalls() + await expect( + run(`allure-upload -r ${runURL} ./src/tests/fixtures/allure/invalid-results`) + ).rejects.toThrowError() + expect(numResultUploadCalls()).toBe(0) + }) + + test('Should skip invalid files and continue when --force is set', async () => { + const numResultUploadCalls = countResultUploadApiCalls() + await run(`allure-upload -r ${runURL} --force ./src/tests/fixtures/allure/invalid-results`) + expect(numResultUploadCalls()).toBe(1) // 1 valid result total + }) +}) diff --git a/src/utils/result-upload/ResultUploadCommandHandler.ts b/src/utils/result-upload/ResultUploadCommandHandler.ts index cb3e4de..b32e6e5 100644 --- a/src/utils/result-upload/ResultUploadCommandHandler.ts +++ b/src/utils/result-upload/ResultUploadCommandHandler.ts @@ -10,17 +10,21 @@ import { TestCaseResult } from './types' import { ResultUploader } from './ResultUploader' import { parseJUnitXml } from './junitXmlParser' import { parsePlaywrightJson } from './playwrightJsonParser' +import { parseAllureResults } from './allureParser' -export type UploadCommandType = 'junit-upload' | 'playwright-json-upload' +export type UploadCommandType = 'junit-upload' | 'playwright-json-upload' | 'allure-upload' export type SkipOutputOption = 'on-success' | 'never' export interface ParserOptions { skipStdout: SkipOutputOption skipStderr: SkipOutputOption + allowPartialParse?: boolean } export type Parser = ( + // Primary parser input. File-based parsers receive file contents while + // directory-based parsers (like Allure) receive a directory path. data: string, attachmentBaseDirectory: string, options: ParserOptions @@ -63,6 +67,7 @@ const DEFAULT_MAPPING_FILENAME_TEMPLATE = 'qasphere-automapping-{YYYY}{MM}{DD}-{ const commandTypeParsers: Record = { 'junit-upload': parseJUnitXml, 'playwright-json-upload': parsePlaywrightJson, + 'allure-upload': parseAllureResults, } export class ResultUploadCommandHandler { @@ -132,13 +137,14 @@ export class ResultUploadCommandHandler { const parserOptions: ParserOptions = { skipStdout: this.args.skipReportStdout, skipStderr: this.args.skipReportStderr, + allowPartialParse: this.args.force, } for (const file of this.args.files) { - const fileData = readFileSync(file).toString() + const fileData = this.type === 'allure-upload' ? file : readFileSync(file).toString() const fileResults = await commandTypeParsers[this.type]( fileData, - dirname(file), + this.type === 'allure-upload' ? file : dirname(file), parserOptions ) results.push({ file, results: fileResults }) diff --git a/src/utils/result-upload/ResultUploader.ts b/src/utils/result-upload/ResultUploader.ts index eb6ecb8..6bfe9ce 100644 --- a/src/utils/result-upload/ResultUploader.ts +++ b/src/utils/result-upload/ResultUploader.ts @@ -83,6 +83,8 @@ export class ResultUploader { this.printJUnitGuidance() } else if (this.type === 'playwright-json-upload') { this.printPlaywrightGuidance(missing[0]?.name || 'your test name') + } else if (this.type === 'allure-upload') { + this.printAllureGuidance(missing[0]?.name || 'your test name') } console.error( chalk.yellow( @@ -130,6 +132,36 @@ ${chalk.yellow('To fix this issue, choose one of the following options:')} `) } + private printAllureGuidance(exampleTestName: string) { + console.error(` +${chalk.yellow('To fix this issue, choose one of the following options:')} + + ${chalk.bold('Option 1: Add a TMS Link (Recommended)')} + Add a TMS link in the Allure result with: + - ${chalk.green('type')}: ${chalk.green('"tms"')} + - ${chalk.green('url')}: ${chalk.green( + `"https://your-qas-instance.com/project/${this.project}/tcase/123"` + )} + + Example: + ${chalk.green(`{ + "links": [ + { + "type": "tms", + "url": "https://your-qas-instance.com/project/${this.project}/tcase/123" + } + ] +}`)} + + ${chalk.bold('Option 2: Include Test Case Marker in Name')} + Include marker ${chalk.green(`${this.project}-`)} in the test name: + + Format: ${chalk.green(`${this.project}-: Your test name`)} + Example: ${chalk.green(`${this.project}-1024: ${exampleTestName}`)} + ${chalk.dim('Where is the test case number (minimum 3 digits, zero-padded if needed)')} +`) + } + private validateAndPrintMissingAttachments = (results: TCaseWithResult[]) => { if (this.args.attachments) { let hasAttachmentErrors = false diff --git a/src/utils/result-upload/allureParser.ts b/src/utils/result-upload/allureParser.ts new file mode 100644 index 0000000..e891bc2 --- /dev/null +++ b/src/utils/result-upload/allureParser.ts @@ -0,0 +1,233 @@ +import escapeHtml from 'escape-html' +import { readdirSync, readFileSync } from 'node:fs' +import { join } from 'node:path' +import stripAnsi from 'strip-ansi' +import z from 'zod' +import { ResultStatus } from '../../api/schemas' +import { getTCaseMarker, parseTCaseUrl } from '../misc' +import { Parser, ParserOptions } from './ResultUploadCommandHandler' +import { Attachment, TestCaseResult } from './types' +import { getAttachments } from './utils' + +const allureStatusSchema = z.enum(['passed', 'failed', 'broken', 'skipped', 'unknown']) +type AllureStatus = z.infer + +const allureStatusDetailsSchema = z + .object({ + message: z.string().optional(), + trace: z.string().optional(), + known: z.boolean().optional(), + muted: z.boolean().optional(), + flaky: z.boolean().optional(), + }) + .nullish() + +const allureAttachmentSchema = z.object({ + name: z.string(), + source: z.string(), + type: z.string(), +}) + +const allureLabelSchema = z.object({ + name: z.string(), + value: z.string(), +}) + +const allureParameterSchema = z.object({ + name: z.string(), + value: z.string(), + excluded: z.boolean().optional(), + mode: z.enum(['default', 'masked', 'hidden']).optional(), +}) + +const allureLinkSchema = z.object({ + name: z.string().optional(), + url: z.string(), + type: z.string().optional(), +}) + +const allureResultSchema = z.object({ + name: z.string(), + status: allureStatusSchema, + uuid: z.string(), + start: z.number(), + stop: z.number(), + fullName: z.string().optional(), + historyId: z.string().optional(), + testCaseId: z.string().optional(), + description: z.string().optional(), + descriptionHtml: z.string().optional(), + stage: z.string().optional(), + statusDetails: allureStatusDetailsSchema, + attachments: z.array(allureAttachmentSchema).nullish(), + labels: z.array(allureLabelSchema).nullish(), + links: z.array(allureLinkSchema).nullish(), + parameters: z.array(allureParameterSchema).nullish(), + steps: z.array(z.unknown()).nullish(), +}) + +type AllureResult = z.infer + +const markerRegex = /\b([A-Za-z0-9]{1,5})-(\d+)\b/ + +export const parseAllureResults: Parser = async ( + resultsDirectory: string, + attachmentBaseDirectory: string, + options: ParserOptions +): Promise => { + const resultFiles = readdirSync(resultsDirectory) + .filter((f) => f.endsWith('-result.json')) + .sort() + + const testcases: TestCaseResult[] = [] + const attachmentsPromises: Array<{ + index: number + promise: Promise + }> = [] + const allowPartialParse = options.allowPartialParse ?? false + + for (const resultFile of resultFiles) { + const resultFilePath = join(resultsDirectory, resultFile) + + let parsedResult: AllureResult + try { + const fileContent = readFileSync(resultFilePath, 'utf8') + parsedResult = allureResultSchema.parse(JSON.parse(fileContent)) + } catch (error) { + if (allowPartialParse) { + console.warn( + `Warning: Skipping invalid Allure result file "${resultFilePath}": ${getErrorMessage(error)}` + ) + continue + } + + throw new Error( + `Failed to parse Allure result file "${resultFilePath}": ${getErrorMessage(error)}` + ) + } + + const status = mapAllureStatus(parsedResult.status) + const marker = extractMarker(parsedResult) + const index = + testcases.push({ + name: marker ? `${marker}: ${parsedResult.name}` : parsedResult.name, + folder: getFolder(parsedResult), + status, + message: buildMessage(parsedResult, status, options), + timeTaken: + parsedResult.stop >= parsedResult.start ? parsedResult.stop - parsedResult.start : null, + attachments: [], + }) - 1 + + const attachmentPaths = (parsedResult.attachments || []).map((attachment) => attachment.source) + attachmentsPromises.push({ + index, + promise: getAttachments(attachmentPaths, attachmentBaseDirectory || resultsDirectory), + }) + } + + const attachments = await Promise.all(attachmentsPromises.map((p) => p.promise)) + attachments.forEach((tcaseAttachment, i) => { + const tcaseIndex = attachmentsPromises[i].index + testcases[tcaseIndex].attachments = tcaseAttachment + }) + + return testcases +} + +const mapAllureStatus = (status: AllureStatus): ResultStatus => { + switch (status) { + case 'passed': + return 'passed' + case 'failed': + return 'failed' + case 'broken': + return 'blocked' + case 'skipped': + return 'skipped' + case 'unknown': + return 'open' + default: + return 'open' + } +} + +const getFolder = (result: AllureResult): string => { + const labels = result.labels || [] + const folderLabelPriority = ['suite', 'parentSuite', 'feature', 'package'] + + for (const labelName of folderLabelPriority) { + const label = labels.find((item) => item.name === labelName) + if (label?.value) { + return label.value + } + } + + return '' +} + +const buildMessage = ( + result: AllureResult, + status: ResultStatus, + options: ParserOptions +): string => { + const statusDetails = result.statusDetails + if (!statusDetails) { + return '' + } + + const includeStdout = !(status === 'passed' && options.skipStdout === 'on-success') + const includeStderr = !(status === 'passed' && options.skipStderr === 'on-success') + + let message = '' + + if (includeStdout && statusDetails.message) { + message += `
${escapeHtml(stripAnsi(statusDetails.message))}
` + } + if (includeStderr && statusDetails.trace) { + message += `
${escapeHtml(stripAnsi(statusDetails.trace))}
` + } + + return message +} + +const extractMarker = (result: AllureResult): string | undefined => { + return getMarkerFromTmsLinks(result.links) || getMarkerFromText(result.name) +} + +const getMarkerFromTmsLinks = (links: AllureResult['links']): string | undefined => { + const tmsLinks = (links || []).filter((link) => link.type?.toLowerCase() === 'tms') + + for (const link of tmsLinks) { + const parsed = parseTCaseUrl(link.url) + if (parsed) { + return getTCaseMarker(parsed.project, parsed.tcaseSeq) + } + } + + for (const link of tmsLinks) { + const markerFromName = getMarkerFromText(link.name) + if (markerFromName) { + return markerFromName + } + } + + return undefined +} + +const getMarkerFromText = (text: string | undefined): string | undefined => { + if (!text) { + return undefined + } + + const match = text.match(markerRegex) + if (!match) { + return undefined + } + + return getTCaseMarker(match[1], Number(match[2])) +} + +const getErrorMessage = (error: unknown) => { + return error instanceof Error ? error.message : String(error) +} From 71a1ce107f395aa6ea7182672cdd82285d632906 Mon Sep 17 00:00:00 2001 From: Satvik Choudhary Date: Fri, 6 Mar 2026 21:43:11 +0530 Subject: [PATCH 2/7] Improve Allure upload: MarkerParser integration, directoryInputTypes, and expanded tests - Switch allureParser to use formatMarker() from MarkerParser instead of the removed getTCaseMarker() from misc - Replace inline this.type === 'allure-upload' checks in parseFiles() with a module-level directoryInputTypes Set for cleaner extensibility - Add 7 new allure-parsing unit tests: ANSI stripping, null arrays, HTML escaping, container/non-result file filtering, empty directory, failed test skip behavior, and negative duration handling - Update CLAUDE.md to document directoryInputTypes, allowPartialParse, and allureParser behavior details Co-Authored-By: Claude Sonnet 4.6 --- CLAUDE.md | 5 +- src/tests/allure-parsing.spec.ts | 127 ++++++++++++++++++ .../ResultUploadCommandHandler.ts | 8 +- src/utils/result-upload/allureParser.ts | 7 +- 4 files changed, 140 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index b587cb9..ef031db 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,7 +43,8 @@ The upload flow has two stages handled by two classes, with a shared `MarkerPars - Also exports a standalone `formatMarker()` function used by parsers 2. **`ResultUploadCommandHandler`** — Orchestrates the overall flow: - - Parses report inputs using the appropriate parser (JUnit XML file, Playwright JSON file, or Allure results directory) + - Parses report inputs using the appropriate parser (JUnit XML file, Playwright JSON file, or Allure results directory). File-based parsers receive file contents; directory-based parsers (Allure) receive the path — controlled by the module-level `directoryInputTypes` Set + - `ParserOptions` includes `allowPartialParse` (set from `--force`) to skip invalid files instead of aborting - Detects project code from test case names via `MarkerParser` (or from `--run-url`) - Creates a new test run (or reuses an existing one if title conflicts) - Delegates actual result uploading to `ResultUploader` @@ -57,7 +58,7 @@ The upload flow has two stages handled by two classes, with a shared `MarkerPars - `junitXmlParser.ts` — Parses JUnit XML via `xml2js` + Zod validation. Extracts attachments from `[[ATTACHMENT|path]]` markers in system-out/failure/error/skipped elements. - `playwrightJsonParser.ts` — Parses Playwright JSON report. Supports two test case linking methods: (1) test annotations with `type: "test case"` and URL description, (2) marker in test name. Handles nested suites recursively. -- `allureParser.ts` — Parses Allure 2 JSON results directories (`*-result.json` files only). Supports test case linking via TMS links (`type: "tms"`) or marker in test name, maps Allure statuses to QA Sphere result statuses, and resolves attachments via `attachments[].source`. +- `allureParser.ts` — Parses Allure 2 JSON results directories (`*-result.json` files only; containers/XML/images ignored). Supports test case linking via TMS links (`type: "tms"`) or marker in test name, maps Allure statuses to QA Sphere result statuses (`unknown→open`, `broken→blocked`), strips ANSI codes and HTML-escapes messages, and resolves attachments via `attachments[].source`. Uses `formatMarker()` from `MarkerParser`. - `types.ts` — Shared `TestCaseResult` and `Attachment` interfaces used by both parsers. ### API Layer (src/api/) diff --git a/src/tests/allure-parsing.spec.ts b/src/tests/allure-parsing.spec.ts index 73c2228..c0b5ad0 100644 --- a/src/tests/allure-parsing.spec.ts +++ b/src/tests/allure-parsing.spec.ts @@ -268,4 +268,131 @@ describe('Allure parsing', () => { expect(testcases[0].name).toBe('TEST-002: TEST-404 marker in test name') expect(testcases[1].name).toBe('TEST-006: TEST-405 marker in test name') }) + + test('Should strip ANSI escape codes from message and trace', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-500 ANSI test', + status: 'failed', + statusDetails: { + message: '\x1b[31mred error\x1b[0m', + trace: '\x1b[1m\x1b[33mbold yellow trace\x1b[0m', + }, + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases[0].message).toContain('red error') + expect(testcases[0].message).not.toContain('\x1b') + expect(testcases[0].message).toContain('bold yellow trace') + }) + + test('Should handle null arrays (labels, attachments, links, parameters) gracefully', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-600 null arrays', + labels: null, + attachments: null, + links: null, + parameters: null, + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].folder).toBe('') + expect(testcases[0].attachments).toHaveLength(0) + }) + + test('Should HTML-escape special characters in message and trace', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-700 HTML test', + status: 'failed', + statusDetails: { + message: '
error
', + trace: '', + }, + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases[0].message).toContain('<div>') + expect(testcases[0].message).not.toContain('
') + expect(testcases[0].message).toContain('<script>') + }) + + test('Should ignore container files, non-result JSON, XML, and images', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify(makeResult({ name: 'valid result only' })), + '002-container.json': JSON.stringify({ uuid: 'c1', befores: [], afters: [] }), + 'report.xml': '', + 'screenshot.png': 'fake-png-data', + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toBe('valid result only') + }) + + test('Should return empty array for empty directory', async () => { + const dir = await createTempAllureDir({}) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(0) + }) + + test('Should include message and trace for failed tests even with on-success skip options', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-900 failed with details', + status: 'failed', + statusDetails: { + message: 'assertion failed', + trace: 'stack trace here', + }, + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'on-success', + skipStderr: 'on-success', + }) + expect(testcases[0].message).toContain('assertion failed') + expect(testcases[0].message).toContain('stack trace here') + }) + + test('Should return null timeTaken when stop is before start', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-1000 negative duration', + start: 2000, + stop: 1000, + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases[0].timeTaken).toBeNull() + }) }) diff --git a/src/utils/result-upload/ResultUploadCommandHandler.ts b/src/utils/result-upload/ResultUploadCommandHandler.ts index b32e6e5..67744de 100644 --- a/src/utils/result-upload/ResultUploadCommandHandler.ts +++ b/src/utils/result-upload/ResultUploadCommandHandler.ts @@ -70,6 +70,8 @@ const commandTypeParsers: Record = { 'allure-upload': parseAllureResults, } +const directoryInputTypes: Set = new Set(['allure-upload']) + export class ResultUploadCommandHandler { private api: Api private baseUrl: string @@ -141,10 +143,12 @@ export class ResultUploadCommandHandler { } for (const file of this.args.files) { - const fileData = this.type === 'allure-upload' ? file : readFileSync(file).toString() + const isDirectoryInput = directoryInputTypes.has(this.type) + const fileData = isDirectoryInput ? file : readFileSync(file).toString() + const attachmentBaseDir = isDirectoryInput ? file : dirname(file) const fileResults = await commandTypeParsers[this.type]( fileData, - this.type === 'allure-upload' ? file : dirname(file), + attachmentBaseDir, parserOptions ) results.push({ file, results: fileResults }) diff --git a/src/utils/result-upload/allureParser.ts b/src/utils/result-upload/allureParser.ts index e891bc2..858f754 100644 --- a/src/utils/result-upload/allureParser.ts +++ b/src/utils/result-upload/allureParser.ts @@ -4,7 +4,8 @@ import { join } from 'node:path' import stripAnsi from 'strip-ansi' import z from 'zod' import { ResultStatus } from '../../api/schemas' -import { getTCaseMarker, parseTCaseUrl } from '../misc' +import { parseTCaseUrl } from '../misc' +import { formatMarker } from './MarkerParser' import { Parser, ParserOptions } from './ResultUploadCommandHandler' import { Attachment, TestCaseResult } from './types' import { getAttachments } from './utils' @@ -201,7 +202,7 @@ const getMarkerFromTmsLinks = (links: AllureResult['links']): string | undefined for (const link of tmsLinks) { const parsed = parseTCaseUrl(link.url) if (parsed) { - return getTCaseMarker(parsed.project, parsed.tcaseSeq) + return formatMarker(parsed.project, parsed.tcaseSeq) } } @@ -225,7 +226,7 @@ const getMarkerFromText = (text: string | undefined): string | undefined => { return undefined } - return getTCaseMarker(match[1], Number(match[2])) + return formatMarker(match[1], Number(match[2])) } const getErrorMessage = (error: unknown) => { From 711303fc02a42b228df0396ce51c091699c84c4b Mon Sep 17 00:00:00 2001 From: Satvik Choudhary Date: Sat, 7 Mar 2026 13:25:09 +0530 Subject: [PATCH 3/7] fix allure parser review issues --- src/tests/allure-parsing.spec.ts | 43 +++++++++++++++++++++++++ src/utils/result-upload/MarkerParser.ts | 14 ++++++++ src/utils/result-upload/allureParser.ts | 41 +++++++++++------------ 3 files changed, 75 insertions(+), 23 deletions(-) diff --git a/src/tests/allure-parsing.spec.ts b/src/tests/allure-parsing.spec.ts index c0b5ad0..8b5d123 100644 --- a/src/tests/allure-parsing.spec.ts +++ b/src/tests/allure-parsing.spec.ts @@ -379,6 +379,38 @@ describe('Allure parsing', () => { expect(testcases[0].message).toContain('stack trace here') }) + test('Should parse results without start and stop timestamps', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'Result without timestamps', + start: undefined, + stop: undefined, + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases[0].timeTaken).toBeNull() + }) + + test('Should not extract markers with fewer than 3 digits', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'Short marker TEST-1 should stay raw', + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases[0].name).toBe('Short marker TEST-1 should stay raw') + }) + test('Should return null timeTaken when stop is before start', async () => { const dir = await createTempAllureDir({ '001-result.json': JSON.stringify( @@ -395,4 +427,15 @@ describe('Allure parsing', () => { }) expect(testcases[0].timeTaken).toBeNull() }) + + test('Should throw a friendly error when the results directory does not exist', async () => { + const missingDir = join(tmpdir(), `qas-missing-allure-results-${Date.now()}`) + + await expect( + parseAllureResults(missingDir, missingDir, { + skipStdout: 'never', + skipStderr: 'never', + }) + ).rejects.toThrow(`Failed to read Allure results directory "${missingDir}"`) + }) }) diff --git a/src/utils/result-upload/MarkerParser.ts b/src/utils/result-upload/MarkerParser.ts index ef332f7..c3e5702 100644 --- a/src/utils/result-upload/MarkerParser.ts +++ b/src/utils/result-upload/MarkerParser.ts @@ -33,6 +33,20 @@ const execRegexWithPriority = ( export const formatMarker = (projectCode: string, seq: number) => `${projectCode}-${seq.toString().padStart(3, '0')}` +/** Extract and normalize a hyphenated marker like "TEST-002" from free text. */ +export const getMarkerFromText = (text: string | undefined): string | undefined => { + if (!text) { + return undefined + } + + const match = text.match(/\b([A-Za-z0-9]{1,5})-(\d{3,})\b/) + if (!match) { + return undefined + } + + return formatMarker(match[1], Number(match[2])) +} + export class MarkerParser { constructor(private type: UploadCommandType) {} diff --git a/src/utils/result-upload/allureParser.ts b/src/utils/result-upload/allureParser.ts index 858f754..820b523 100644 --- a/src/utils/result-upload/allureParser.ts +++ b/src/utils/result-upload/allureParser.ts @@ -5,7 +5,7 @@ import stripAnsi from 'strip-ansi' import z from 'zod' import { ResultStatus } from '../../api/schemas' import { parseTCaseUrl } from '../misc' -import { formatMarker } from './MarkerParser' +import { formatMarker, getMarkerFromText } from './MarkerParser' import { Parser, ParserOptions } from './ResultUploadCommandHandler' import { Attachment, TestCaseResult } from './types' import { getAttachments } from './utils' @@ -51,8 +51,8 @@ const allureResultSchema = z.object({ name: z.string(), status: allureStatusSchema, uuid: z.string(), - start: z.number(), - stop: z.number(), + start: z.number().optional(), + stop: z.number().optional(), fullName: z.string().optional(), historyId: z.string().optional(), testCaseId: z.string().optional(), @@ -69,16 +69,21 @@ const allureResultSchema = z.object({ type AllureResult = z.infer -const markerRegex = /\b([A-Za-z0-9]{1,5})-(\d+)\b/ - export const parseAllureResults: Parser = async ( resultsDirectory: string, attachmentBaseDirectory: string, options: ParserOptions ): Promise => { - const resultFiles = readdirSync(resultsDirectory) - .filter((f) => f.endsWith('-result.json')) - .sort() + let resultFiles: string[] + try { + resultFiles = readdirSync(resultsDirectory) + .filter((f) => f.endsWith('-result.json')) + .sort() + } catch (error) { + throw new Error( + `Failed to read Allure results directory "${resultsDirectory}": ${getErrorMessage(error)}` + ) + } const testcases: TestCaseResult[] = [] const attachmentsPromises: Array<{ @@ -116,7 +121,11 @@ export const parseAllureResults: Parser = async ( status, message: buildMessage(parsedResult, status, options), timeTaken: - parsedResult.stop >= parsedResult.start ? parsedResult.stop - parsedResult.start : null, + parsedResult.start != null && + parsedResult.stop != null && + parsedResult.stop >= parsedResult.start + ? parsedResult.stop - parsedResult.start + : null, attachments: [], }) - 1 @@ -215,20 +224,6 @@ const getMarkerFromTmsLinks = (links: AllureResult['links']): string | undefined return undefined } - -const getMarkerFromText = (text: string | undefined): string | undefined => { - if (!text) { - return undefined - } - - const match = text.match(markerRegex) - if (!match) { - return undefined - } - - return formatMarker(match[1], Number(match[2])) -} - const getErrorMessage = (error: unknown) => { return error instanceof Error ? error.message : String(error) } From 62a75cc9ff37a1cc8be2de99293bbc9217e8faae Mon Sep 17 00:00:00 2001 From: Satvik Choudhary Date: Sat, 7 Mar 2026 14:49:38 +0530 Subject: [PATCH 4/7] Fix allure parser: optional TMS URL, nested step attachments, getMarkerFromText tests - Make links[].url optional in schema so @TmsLink without allure.link.tms.pattern works - Guard getMarkerFromTmsLinks against undefined URL - Add recursive AllureStep schema and collectStepAttachmentPaths for nested attachments - Add getMarkerFromText unit tests for edge cases - Add blank line before getErrorMessage --- src/tests/allure-parsing.spec.ts | 104 ++++++++++++++++++++++++ src/tests/marker-parser.spec.ts | 33 +++++++- src/utils/result-upload/allureParser.ts | 36 +++++++- 3 files changed, 169 insertions(+), 4 deletions(-) diff --git a/src/tests/allure-parsing.spec.ts b/src/tests/allure-parsing.spec.ts index 8b5d123..cfe71d5 100644 --- a/src/tests/allure-parsing.spec.ts +++ b/src/tests/allure-parsing.spec.ts @@ -428,6 +428,110 @@ describe('Allure parsing', () => { expect(testcases[0].timeTaken).toBeNull() }) + test('Should extract marker from TMS link name when URL is absent', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'Login page test', + links: [{ type: 'tms', name: 'TEST-042' }], + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toBe('TEST-042: Login page test') + }) + + test('Should fall through to test name when TMS link has neither URL nor name', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-099 Checkout test', + links: [{ type: 'tms' }], + }) + ), + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].name).toBe('TEST-099: TEST-099 Checkout test') + }) + + test('Should collect attachments from nested steps', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-800 step attachments', + steps: [ + { + name: 'Step 1', + status: 'passed', + attachments: [{ name: 'step-log.txt', source: 'step-log.txt', type: 'text/plain' }], + steps: [], + }, + ], + }) + ), + 'step-log.txt': 'step log content', + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].attachments).toHaveLength(1) + expect(testcases[0].attachments[0].filename).toBe('step-log.txt') + expect(testcases[0].attachments[0].buffer).not.toBeNull() + }) + + test('Should collect attachments from deeply nested sub-steps', async () => { + const dir = await createTempAllureDir({ + '001-result.json': JSON.stringify( + makeResult({ + name: 'TEST-801 deep step attachments', + attachments: [{ name: 'top-level.txt', source: 'top-level.txt', type: 'text/plain' }], + steps: [ + { + name: 'Step 1', + status: 'passed', + attachments: [{ name: 'level1.txt', source: 'level1.txt', type: 'text/plain' }], + steps: [ + { + name: 'Sub-step 1.1', + status: 'passed', + attachments: [ + { + name: 'level2.txt', + source: 'level2.txt', + type: 'text/plain', + }, + ], + steps: [], + }, + ], + }, + ], + }) + ), + 'top-level.txt': 'top', + 'level1.txt': 'level1', + 'level2.txt': 'level2', + }) + const testcases = await parseAllureResults(dir, dir, { + skipStdout: 'never', + skipStderr: 'never', + }) + expect(testcases).toHaveLength(1) + expect(testcases[0].attachments).toHaveLength(3) + const filenames = testcases[0].attachments.map((a) => a.filename).sort() + expect(filenames).toEqual(['level1.txt', 'level2.txt', 'top-level.txt']) + }) + test('Should throw a friendly error when the results directory does not exist', async () => { const missingDir = join(tmpdir(), `qas-missing-allure-results-${Date.now()}`) diff --git a/src/tests/marker-parser.spec.ts b/src/tests/marker-parser.spec.ts index 4f53ef1..0d43317 100644 --- a/src/tests/marker-parser.spec.ts +++ b/src/tests/marker-parser.spec.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest' -import { MarkerParser, formatMarker } from '../utils/result-upload/MarkerParser' +import { MarkerParser, formatMarker, getMarkerFromText } from '../utils/result-upload/MarkerParser' const junit = new MarkerParser('junit-upload') const playwright = new MarkerParser('playwright-json-upload') @@ -169,6 +169,37 @@ describe('extractSeq', () => { }) }) +describe('getMarkerFromText', () => { + test('returns marker from text containing a valid marker', () => { + expect(getMarkerFromText('Some TEST-002 test')).toBe('TEST-002') + }) + + test('returns undefined for undefined input', () => { + expect(getMarkerFromText(undefined)).toBeUndefined() + }) + + test('returns undefined for empty string', () => { + expect(getMarkerFromText('')).toBeUndefined() + }) + + test('returns undefined for markers with fewer than 3 digits', () => { + expect(getMarkerFromText('TEST-1')).toBeUndefined() + expect(getMarkerFromText('TEST-12')).toBeUndefined() + }) + + test('returns first match when multiple markers present', () => { + expect(getMarkerFromText('TEST-002 PROJ-003')).toBe('TEST-002') + }) + + test('handles max-length project code (5 chars)', () => { + expect(getMarkerFromText('ABCDE-123')).toBe('ABCDE-123') + }) + + test('rejects project code longer than 5 chars', () => { + expect(getMarkerFromText('ABCDEF-123')).toBeUndefined() + }) +}) + describe('nameMatchesTCase', () => { describe('hyphenated markers (all formats)', () => { test('case-insensitive match', () => { diff --git a/src/utils/result-upload/allureParser.ts b/src/utils/result-upload/allureParser.ts index 820b523..d4efd26 100644 --- a/src/utils/result-upload/allureParser.ts +++ b/src/utils/result-upload/allureParser.ts @@ -43,10 +43,28 @@ const allureParameterSchema = z.object({ const allureLinkSchema = z.object({ name: z.string().optional(), - url: z.string(), + url: z.string().optional(), type: z.string().optional(), }) +type AllureStep = { + name?: string + status?: string + attachments?: Array<{ name: string; source: string; type: string }> | null + steps?: AllureStep[] | null +} + +const allureStepSchema: z.ZodType = z.lazy(() => + z + .object({ + name: z.string().optional(), + status: z.string().optional(), + attachments: z.array(allureAttachmentSchema).nullish(), + steps: z.array(allureStepSchema).nullish(), + }) + .passthrough() +) + const allureResultSchema = z.object({ name: z.string(), status: allureStatusSchema, @@ -64,7 +82,7 @@ const allureResultSchema = z.object({ labels: z.array(allureLabelSchema).nullish(), links: z.array(allureLinkSchema).nullish(), parameters: z.array(allureParameterSchema).nullish(), - steps: z.array(z.unknown()).nullish(), + steps: z.array(allureStepSchema).nullish(), }) type AllureResult = z.infer @@ -129,7 +147,9 @@ export const parseAllureResults: Parser = async ( attachments: [], }) - 1 - const attachmentPaths = (parsedResult.attachments || []).map((attachment) => attachment.source) + const topLevelPaths = (parsedResult.attachments || []).map((a) => a.source) + const stepPaths = collectStepAttachmentPaths(parsedResult.steps) + const attachmentPaths = [...topLevelPaths, ...stepPaths] attachmentsPromises.push({ index, promise: getAttachments(attachmentPaths, attachmentBaseDirectory || resultsDirectory), @@ -145,6 +165,14 @@ export const parseAllureResults: Parser = async ( return testcases } +const collectStepAttachmentPaths = (steps: AllureStep[] | null | undefined): string[] => { + if (!steps) return [] + return steps.flatMap((step) => [ + ...(step.attachments || []).map((a) => a.source), + ...collectStepAttachmentPaths(step.steps), + ]) +} + const mapAllureStatus = (status: AllureStatus): ResultStatus => { switch (status) { case 'passed': @@ -209,6 +237,7 @@ const getMarkerFromTmsLinks = (links: AllureResult['links']): string | undefined const tmsLinks = (links || []).filter((link) => link.type?.toLowerCase() === 'tms') for (const link of tmsLinks) { + if (!link.url) continue const parsed = parseTCaseUrl(link.url) if (parsed) { return formatMarker(parsed.project, parsed.tcaseSeq) @@ -224,6 +253,7 @@ const getMarkerFromTmsLinks = (links: AllureResult['links']): string | undefined return undefined } + const getErrorMessage = (error: unknown) => { return error instanceof Error ? error.message : String(error) } From fd821bf56abbbb961cb1494da32d81657fcb1992 Mon Sep 17 00:00:00 2001 From: Satvik Choudhary Date: Mon, 9 Mar 2026 22:50:03 +0530 Subject: [PATCH 5/7] Bump version to 0.5.0 for Allure upload feature --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bf8e2c6..e3d1bb2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "qas-cli", - "version": "0.4.6", + "version": "0.5.0", "description": "QAS CLI is a command line tool for submitting your automation test results to QA Sphere at https://qasphere.com/", "type": "module", "main": "./build/bin/qasphere.js", From 09b1f1dd1fa336ddd98b65a96a8fccf950d6b56d Mon Sep 17 00:00:00 2001 From: Satvik Choudhary Date: Fri, 13 Mar 2026 14:46:13 +0530 Subject: [PATCH 6/7] Add schema reference comments to allure-parsing spec --- src/tests/allure-parsing.spec.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/tests/allure-parsing.spec.ts b/src/tests/allure-parsing.spec.ts index cfe71d5..67537ed 100644 --- a/src/tests/allure-parsing.spec.ts +++ b/src/tests/allure-parsing.spec.ts @@ -1,3 +1,11 @@ +// Allure 2 result file schema reference: +// https://allurereport.org/docs/how-it-works-test-result-file/ +// +// Real framework outputs used to verify field presence vs docs: +// https://github.com/allure-framework/allure-examples +// Frameworks tested: python-pytest, python-behave, python-robotframework, +// js-jest, ts-mocha (js-codeceptjs / ts-codeceptjs produce Allure 1 XML — not supported) + import { afterEach, describe, expect, test } from 'vitest' import { mkdtemp, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' From 39446266dc2eb1bc4df309708907962af952a704 Mon Sep 17 00:00:00 2001 From: Satvik Choudhary Date: Fri, 13 Mar 2026 14:46:20 +0530 Subject: [PATCH 7/7] Consolidate command display strings, input kinds, and example inputs into a single CommandDetails record --- src/commands/resultUpload.ts | 54 +++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/commands/resultUpload.ts b/src/commands/resultUpload.ts index 0a530e8..8b8fd53 100644 --- a/src/commands/resultUpload.ts +++ b/src/commands/resultUpload.ts @@ -7,33 +7,43 @@ import { UploadCommandType, } from '../utils/result-upload/ResultUploadCommandHandler' -const commandTypeDisplayStrings: Record = { - 'junit-upload': 'JUnit XML', - 'playwright-json-upload': 'Playwright JSON', - 'allure-upload': 'Allure', +interface CommandDetails { + displayString: string + inputKind: string + exampleInput: string } -const commandTypeInputKinds: Record = { - 'junit-upload': 'files', - 'playwright-json-upload': 'files', - 'allure-upload': 'directories', -} - -const commandTypeExampleInputs: Record = { - 'junit-upload': './test-results.xml', - 'playwright-json-upload': './test-results.json', - 'allure-upload': './allure-results', +const commandDetails: Record = { + 'junit-upload': { + displayString: 'JUnit XML', + inputKind: 'files', + exampleInput: './test-results.xml', + }, + 'playwright-json-upload': { + displayString: 'Playwright JSON', + inputKind: 'files', + exampleInput: './test-results.json', + }, + 'allure-upload': { + displayString: 'Allure', + inputKind: 'directories', + exampleInput: './allure-results', + }, } export class ResultUploadCommandModule implements CommandModule { - constructor(private readonly type: UploadCommandType) {} + private readonly details: CommandDetails + + constructor(private readonly type: UploadCommandType) { + this.details = commandDetails[type] + } get command() { return `${this.type} [args..] ` } get describe() { - return `Upload ${commandTypeDisplayStrings[this.type]} ${commandTypeInputKinds[this.type]} to a new or existing test run` + return `Upload ${this.details.displayString} ${this.details.inputKind} to a new or existing test run` } builder = (argv: Argv) => { @@ -101,20 +111,18 @@ export class ResultUploadCommandModule implements CommandModule