Skip to content

Commit e4de0f9

Browse files
satvik007satvik007
authored
[codex] Fix marker resolution and duplicate mapping validation (#63)
* Fix marker resolution and duplicate mapping * Bump package version to 0.6.1 * Prevent duplicate mapping failures from creating test cases * Remove issue report markdown files * Bump package version to 0.7.0
1 parent 503a755 commit e4de0f9

16 files changed

Lines changed: 729 additions & 98 deletions

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ There are two modes for uploading results using the commands:
182182
- `--run-name` - Optional name template for creating new test run. It supports `{env:VAR}`, `{YYYY}`, `{YY}`, `{MM}`, `{MMM}`, `{DD}`, `{HH}`, `{hh}`, `{mm}`, `{ss}`, `{AMPM}` placeholders (default: `Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}`)
183183
- `--create-tcases` - Automatically create test cases in QA Sphere for results that don't have valid test case markers. A mapping file (`qasphere-automapping-YYYYMMDD-HHmmss.txt`) is generated showing the sequence numbers assigned to each new test case (default: `false`)
184184
- `--attachments` - Try to detect and upload any attachments with the test result
185-
- `--force` - Ignore API request errors, invalid test cases, or attachments
185+
- `--force` - Ignore API request errors, invalid or duplicate test case mappings, or attachments
186186
- `--ignore-unmatched` - Suppress individual unmatched test messages, show summary only
187187
- `--skip-report-stdout` - Control when to skip stdout blocks from test report (choices: `on-success`, `never`; default: `never`)
188188
- `--skip-report-stderr` - Control when to skip stderr blocks from test report (choices: `on-success`, `never`; default: `never`)
@@ -297,7 +297,7 @@ Ensure the required environment variables are defined before running these comma
297297

298298
## Test Report Requirements
299299

300-
The QAS CLI maps test results from your reports (JUnit XML, Playwright JSON, or Allure) to corresponding test cases in QA Sphere. If a test result lacks a valid marker/reference, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to skip unmatched results.
300+
The QAS CLI maps test results from your reports (JUnit XML, Playwright JSON, or Allure) to corresponding test cases in QA Sphere. If a test result lacks a valid marker/reference, or multiple results resolve to the same run test case, the CLI will display an error unless you use `--create-tcases` to automatically create test cases, or `--ignore-unmatched`/`--force` to bypass the mapping issue.
301301

302302
### JUnit XML
303303

‎package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "qas-cli",
3-
"version": "0.6.0",
3+
"version": "0.7.0",
44
"description": "QAS CLI is a command line tool for submitting your automation test results to QA Sphere at https://qasphere.com/",
55
"type": "module",
66
"main": "./build/bin/qasphere.js",

‎src/commands/resultUpload.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ export class ResultUploadCommandModule implements CommandModule<unknown, ResultU
8484
type: 'boolean',
8585
},
8686
force: {
87-
describe: 'Ignore API request errors, invalid test cases or attachments',
87+
describe:
88+
'Ignore API request errors, invalid or duplicate test case mappings, or attachments',
8889
type: 'boolean',
8990
},
9091
'ignore-unmatched': {

‎src/tests/fixtures/playwright-json/describe-marker-collision.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"suites": [
3+
{
4+
"title": "describe-collision.spec.ts",
5+
"specs": [],
6+
"suites": [
7+
{
8+
"title": "Test Case Folders (TEST-002, TEST-003)",
9+
"specs": [
10+
{
11+
"title": "should create a new root folder (TEST-004)",
12+
"tags": [],
13+
"tests": [
14+
{
15+
"annotations": [],
16+
"expectedStatus": "passed",
17+
"projectName": "chromium",
18+
"results": [
19+
{
20+
"status": "passed",
21+
"errors": [],
22+
"stdout": [],
23+
"stderr": [],
24+
"retry": 0,
25+
"duration": 1000,
26+
"attachments": []
27+
}
28+
],
29+
"status": "expected"
30+
}
31+
]
32+
}
33+
],
34+
"suites": []
35+
}
36+
]
37+
}
38+
]
39+
}

‎src/tests/fixtures/playwright-json/describe-marker-unmatched.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"suites": [
3+
{
4+
"title": "describe-marker-unmatched.spec.ts",
5+
"specs": [],
6+
"suites": [
7+
{
8+
"title": "Dashboard Projects (TEST-034, TEST-035, TEST-365)",
9+
"specs": [
10+
{
11+
"title": "viewer should not see project management actions",
12+
"tags": [],
13+
"tests": [
14+
{
15+
"annotations": [],
16+
"expectedStatus": "passed",
17+
"projectName": "chromium",
18+
"results": [
19+
{
20+
"status": "passed",
21+
"errors": [],
22+
"stdout": [],
23+
"stderr": [],
24+
"retry": 0,
25+
"duration": 1000,
26+
"attachments": []
27+
}
28+
],
29+
"status": "expected"
30+
}
31+
]
32+
}
33+
],
34+
"suites": []
35+
}
36+
]
37+
}
38+
]
39+
}

‎src/tests/playwright-json-parsing.spec.ts‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -391,8 +391,14 @@ describe('Playwright JSON parsing', () => {
391391
// Fixture has 1 test with 3 annotations (2x 10427 deduped to 1, plus 10428) + 1 test with no annotations = 3 results
392392
expect(testcases).toHaveLength(3)
393393
expect(testcases[0].name).toBe('TEST-10427: Login flow covers multiple cases')
394+
expect(testcases[0].marker).toEqual({ projectCode: 'TEST', seq: 10427 })
395+
expect(testcases[0].markerResolution).toBe('resolved')
394396
expect(testcases[1].name).toBe('TEST-10428: Login flow covers multiple cases')
397+
expect(testcases[1].marker).toEqual({ projectCode: 'TEST', seq: 10428 })
398+
expect(testcases[1].markerResolution).toBe('resolved')
395399
expect(testcases[2].name).toBe('Navigation bar items TEST-006')
400+
expect(testcases[2].marker).toEqual({ projectCode: 'TEST', seq: 6 })
401+
expect(testcases[2].markerResolution).toBe('resolved')
396402

397403
// The two fan-out entries share the same status, duration, folder
398404
for (const tc of testcases.slice(0, 2)) {
@@ -454,6 +460,8 @@ describe('Playwright JSON parsing', () => {
454460

455461
expect(testcases).toHaveLength(1)
456462
expect(testcases[0].name).toBe('PRJ-100: Simple test')
463+
expect(testcases[0].marker).toEqual({ projectCode: 'PRJ', seq: 100 })
464+
expect(testcases[0].markerResolution).toBe('resolved')
457465
})
458466

459467
test('Should fan out by annotations even when name has a marker', async () => {
@@ -509,6 +517,118 @@ describe('Playwright JSON parsing', () => {
509517
expect(testcases).toHaveLength(2)
510518
expect(testcases[0].name).toBe('PRJ-100: PRJ-999: Test with marker in name')
511519
expect(testcases[1].name).toBe('PRJ-200: PRJ-999: Test with marker in name')
520+
expect(testcases[0].marker).toEqual({ projectCode: 'PRJ', seq: 100 })
521+
expect(testcases[1].marker).toEqual({ projectCode: 'PRJ', seq: 200 })
522+
expect(testcases[0].markerResolution).toBe('resolved')
523+
expect(testcases[1].markerResolution).toBe('resolved')
524+
})
525+
526+
test('Should extract fallback marker from spec title, not describe titles', async () => {
527+
const jsonContent = JSON.stringify({
528+
suites: [
529+
{
530+
title: 'describe-collision.spec.ts',
531+
specs: [],
532+
suites: [
533+
{
534+
title: 'Test Case Folders (TEST-002, TEST-003)',
535+
specs: [
536+
{
537+
title: 'should create a new root folder (TEST-004)',
538+
tags: [],
539+
tests: [
540+
{
541+
annotations: [],
542+
expectedStatus: 'passed',
543+
projectName: 'chromium',
544+
results: [
545+
{
546+
status: 'passed',
547+
errors: [],
548+
stdout: [],
549+
stderr: [],
550+
retry: 0,
551+
duration: 1000,
552+
attachments: [],
553+
},
554+
],
555+
status: 'expected',
556+
},
557+
],
558+
},
559+
],
560+
suites: [],
561+
},
562+
],
563+
},
564+
],
565+
})
566+
567+
const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', {
568+
skipStdout: 'never',
569+
skipStderr: 'never',
570+
})
571+
572+
expect(testcases).toHaveLength(1)
573+
expect(testcases[0].name).toBe(
574+
'Test Case Folders (TEST-002, TEST-003) › should create a new root folder (TEST-004)'
575+
)
576+
expect(testcases[0].marker).toEqual({ projectCode: 'TEST', seq: 4 })
577+
expect(testcases[0].markerResolution).toBe('resolved')
578+
})
579+
580+
test('Should keep marker unresolved-none when only describe title contains markers', async () => {
581+
const jsonContent = JSON.stringify({
582+
suites: [
583+
{
584+
title: 'describe-only-markers.spec.ts',
585+
specs: [],
586+
suites: [
587+
{
588+
title: 'Dashboard Projects (TEST-034, TEST-035, TEST-365)',
589+
specs: [
590+
{
591+
title: 'viewer should not see project management actions',
592+
tags: [],
593+
tests: [
594+
{
595+
annotations: [],
596+
expectedStatus: 'passed',
597+
projectName: 'chromium',
598+
results: [
599+
{
600+
status: 'passed',
601+
errors: [],
602+
stdout: [],
603+
stderr: [],
604+
retry: 0,
605+
duration: 1000,
606+
attachments: [],
607+
},
608+
],
609+
status: 'expected',
610+
},
611+
],
612+
},
613+
],
614+
suites: [],
615+
},
616+
],
617+
},
618+
],
619+
})
620+
621+
const { testCaseResults: testcases } = await parsePlaywrightJson(jsonContent, '', {
622+
skipStdout: 'never',
623+
skipStderr: 'never',
624+
})
625+
626+
expect(testcases).toHaveLength(1)
627+
expect(testcases[0].name).toBe(
628+
'Dashboard Projects (TEST-034, TEST-035, TEST-365) › viewer should not see project management actions'
629+
)
630+
expect(testcases[0].marker).toBeNull()
631+
expect(testcases[0].markerResolution).toBe('resolved-none')
512632
})
513633

514634
test('Should map test status correctly', async () => {

0 commit comments

Comments
 (0)