From 375aba65e5f2b97615952966626952e8871239ea Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Mon, 15 Dec 2025 15:33:32 +0100 Subject: [PATCH 1/5] fix: add missing attributes to parsed model --- test/converters/junitxml/models.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/converters/junitxml/models.go b/test/converters/junitxml/models.go index b9e4ec9f..f0df7480 100644 --- a/test/converters/junitxml/models.go +++ b/test/converters/junitxml/models.go @@ -79,18 +79,21 @@ type RerunError struct { type Failure struct { XMLName xml.Name `xml:"failure,omitempty"` Message string `xml:"message,attr,omitempty"` + Type string `xml:"type,attr,omitempty"` Value string `xml:",chardata"` } // Skipped ... type Skipped struct { XMLName xml.Name `xml:"skipped,omitempty"` + Message string `xml:"message,attr,omitempty"` } // Error ... type Error struct { XMLName xml.Name `xml:"error,omitempty"` Message string `xml:"message,attr,omitempty"` + Type string `xml:"type,attr,omitempty"` Value string `xml:",chardata"` } From b91a376b28a16d5e041dffa1ff23260b72bd7996 Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Mon, 15 Dec 2025 19:55:18 +0100 Subject: [PATCH 2/5] fix: extend test_report models --- test/testreport/test_report.go | 36 +++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/test/testreport/test_report.go b/test/testreport/test_report.go index 9059ddb0..82ea8c92 100644 --- a/test/testreport/test_report.go +++ b/test/testreport/test_report.go @@ -15,6 +15,7 @@ type TestSuite struct { Name string `xml:"name,attr"` Tests int `xml:"tests,attr"` Failures int `xml:"failures,attr"` + Errors int `xml:"errors,attr"` Skipped int `xml:"skipped,attr"` Time float64 `xml:"time,attr"` TestCases []TestCase `xml:"testcase"` @@ -25,15 +26,21 @@ type TestCase struct { XMLName xml.Name `xml:"testcase"` // ConfigurationHash is used to distinguish the same test case runs, // performed with different build configurations (e.g., Debug vs. Release) or different devices/simulators - ConfigurationHash string `xml:"configuration-hash,attr"` - Name string `xml:"name,attr"` - ClassName string `xml:"classname,attr"` - Time float64 `xml:"time,attr"` - // TODO: Currently a JUnit report's TestCase.Error and TestCase.SystemErr is merged into TestCase.Failure.Value field, - // this way test execution errors are not distinguished from test failures. - Failure *Failure `xml:"failure,omitempty"` - Skipped *Skipped `xml:"skipped,omitempty"` - Properties *Properties `xml:"properties,omitempty"` + ConfigurationHash string `xml:"configuration-hash,attr"` + Name string `xml:"name,attr"` + ClassName string `xml:"classname,attr"` + Time float64 `xml:"time,attr"` + Error *Error `xml:"error,omitempty"` + Failure *Failure `xml:"failure,omitempty"` + Skipped *Skipped `xml:"skipped,omitempty"` + Properties *Properties `xml:"properties,omitempty"` + SystemOut *SystemOut `xml:"system-out,omitempty"` + SystemErr *SystemErr `xml:"system-err,omitempty"` +} + +type Error struct { + XMLName xml.Name `xml:"error,omitempty"` + Value string `xml:",chardata"` } type Failure struct { @@ -43,6 +50,7 @@ type Failure struct { type Skipped struct { XMLName xml.Name `xml:"skipped,omitempty"` + Value string `xml:",chardata"` } type Property struct { @@ -55,3 +63,13 @@ type Properties struct { XMLName xml.Name `xml:"properties"` Property []Property `xml:"property"` } + +type SystemOut struct { + XMLName xml.Name `xml:"system-out,omitempty"` + Value string `xml:",chardata"` +} + +type SystemErr struct { + XMLName xml.Name `xml:"system-err,omitempty"` + Value string `xml:",chardata"` +} From 6e0ed4dcdd061bc6b6d8d494e82754782f8c36db Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Tue, 16 Dec 2025 16:07:37 +0100 Subject: [PATCH 3/5] feat: expose system-out and system-err and append them to message contents --- test/converters/junitxml/junitxml.go | 171 ++++++++-- test/converters/junitxml/junitxml_test.go | 386 +++++++++++++--------- 2 files changed, 369 insertions(+), 188 deletions(-) diff --git a/test/converters/junitxml/junitxml.go b/test/converters/junitxml/junitxml.go index 5ccd728f..36841e54 100644 --- a/test/converters/junitxml/junitxml.go +++ b/test/converters/junitxml/junitxml.go @@ -83,6 +83,7 @@ func convertTestSuite(testSuite TestSuite) testreport.TestSuite { tests := 0 failures := 0 + errors := 0 skipped := 0 flattenedTestCases := flattenGroupedTestCases(testSuite.TestCases) @@ -101,12 +102,12 @@ func convertTestSuite(testSuite TestSuite) testreport.TestSuite { for _, childSuite := range testSuite.TestSuites { convertedChildSuite := convertTestSuite(childSuite) - convertedTestSuite.TestSuites = append(convertedTestSuite.TestSuites, convertedChildSuite) } convertedTestSuite.Tests = tests convertedTestSuite.Failures = failures + convertedTestSuite.Errors = errors convertedTestSuite.Skipped = skipped return convertedTestSuite @@ -187,16 +188,20 @@ func convertTestCase(testCase TestCase) testreport.TestCase { Name: testCase.Name, ClassName: testCase.ClassName, Time: testCase.Time, + Failure: convertFailure(testCase.Failure), + Error: convertError(testCase.Error), + Skipped: convertSkipped(testCase.Skipped), + SystemOut: convertSystemOut(testCase.SystemOut), + SystemErr: convertSystemErr(testCase.SystemErr), Properties: convertProperties(testCase.Properties), } - if testCase.Skipped != nil { - convertedTestCase.Skipped = &testreport.Skipped{ - XMLName: testCase.Skipped.XMLName, - } + if convertedTestCase.Error != nil { + convertedTestCase.Failure = convertErrorToFailure(convertedTestCase.Error) + convertedTestCase.Error = nil } - convertedTestCase.Failure = convertErrorsToFailure(testCase.Failure, testCase.Error, testCase.SystemErr) + enrichWithSystemOutputs(&convertedTestCase, testCase.SystemOut, testCase.SystemErr) return convertedTestCase } @@ -218,38 +223,146 @@ func convertProperties(properties *Properties) *testreport.Properties { return convertedProperties } -func convertErrorsToFailure(failure *Failure, error *Error, systemErr string) *testreport.Failure { - var messages []string +func convertSystemOut(systemOut string) *testreport.SystemOut { + if systemOut == "" { + return nil + } - if failure != nil { - if len(strings.TrimSpace(failure.Message)) > 0 { - messages = append(messages, failure.Message) - } + return &testreport.SystemOut{ + XMLName: xml.Name{Local: "system-out"}, + Value: systemOut, + } +} - if len(strings.TrimSpace(failure.Value)) > 0 { - messages = append(messages, failure.Value) - } +func convertSystemErr(systemErr string) *testreport.SystemErr { + if systemErr == "" { + return nil } - if error != nil { - if len(strings.TrimSpace(error.Message)) > 0 { - messages = append(messages, "Error message:\n"+error.Message) - } + return &testreport.SystemErr{ + XMLName: xml.Name{Local: "system-err"}, + Value: systemErr, + } +} - if len(strings.TrimSpace(error.Value)) > 0 { - messages = append(messages, "Error value:\n"+error.Value) - } +func convertSkipped(skipped *Skipped) *testreport.Skipped { + if skipped == nil { + return nil + } + + var parts []string + + if len(strings.TrimSpace(skipped.Message)) > 0 { + parts = append(parts, strings.TrimSpace(skipped.Message)) + } + + return &testreport.Skipped{ + XMLName: xml.Name{Local: "skipped"}, + Value: strings.Join(parts, "\n\n"), + } +} + +func convertFailure(failure *Failure) *testreport.Failure { + if failure == nil { + return nil + } + + var parts []string + + var attributes []string + if len(strings.TrimSpace(failure.Type)) > 0 { + attributes = append(attributes, failure.Type) } + if len(strings.TrimSpace(failure.Message)) > 0 { + attributes = append(attributes, failure.Message) + } + if len(attributes) > 0 { + parts = append(parts, strings.Join(attributes, ": ")) + } + + if len(strings.TrimSpace(failure.Value)) > 0 { + parts = append(parts, failure.Value) + } + + return &testreport.Failure{ + XMLName: xml.Name{Local: "failure"}, + Value: strings.Join(parts, "\n\n"), + } +} - if len(systemErr) > 0 { - messages = append(messages, "System error:\n"+systemErr) +func convertError(error *Error) *testreport.Error { + if error == nil { + return nil } - if len(messages) > 0 { - return &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: strings.Join(messages, "\n\n"), + var parts []string + + var attributes []string + if len(strings.TrimSpace(error.Type)) > 0 { + attributes = append(attributes, error.Type) + } + if len(strings.TrimSpace(error.Message)) > 0 { + attributes = append(attributes, error.Message) + } + if len(attributes) > 0 { + parts = append(parts, strings.Join(attributes, ": ")) + } + + if len(strings.TrimSpace(error.Value)) > 0 { + parts = append(parts, error.Value) + } + + return &testreport.Error{ + XMLName: xml.Name{Local: "error"}, + Value: strings.Join(parts, "\n\n"), + } +} + +func convertErrorToFailure(error *testreport.Error) *testreport.Failure { + if error == nil { + return nil + } + + return &testreport.Failure{ + XMLName: xml.Name{Local: "failure"}, + Value: error.Value, + } +} + +func enrichWithSystemOutputs(testCase *testreport.TestCase, systemOut, systemErr string) { + var testOutputs []string = []string{} + + if len(strings.TrimSpace(systemErr)) > 0 { + testOutputs = append(testOutputs, "System error:\n"+systemErr) + } + + if len(strings.TrimSpace(systemOut)) > 0 { + testOutputs = append(testOutputs, "System output:\n"+systemOut) + } + + if (len(testOutputs)) == 0 { + return + } + + combinedOutput := strings.Join(testOutputs, "\n\n") + + if testCase.Error != nil { + if len(strings.TrimSpace(testCase.Error.Value)) > 0 { + testCase.Error.Value = testCase.Error.Value + "\n\n" + combinedOutput + } else { + testCase.Error.Value = combinedOutput + } + } else if testCase.Failure != nil { + if len(strings.TrimSpace(testCase.Failure.Value)) > 0 { + testCase.Failure.Value = testCase.Failure.Value + "\n\n" + combinedOutput + } else { + testCase.Failure.Value = combinedOutput + } + } else if testCase.Skipped != nil { + if len(strings.TrimSpace(testCase.Skipped.Value)) > 0 { + testCase.Skipped.Value = testCase.Skipped.Value + "\n\n" + combinedOutput + } else { + testCase.Skipped.Value = combinedOutput } } - return nil } diff --git a/test/converters/junitxml/junitxml_test.go b/test/converters/junitxml/junitxml_test.go index 265b8a10..d4a5d02c 100644 --- a/test/converters/junitxml/junitxml_test.go +++ b/test/converters/junitxml/junitxml_test.go @@ -35,279 +35,341 @@ func Test_convertTestReport(t *testing.T) { suites []TestSuite want []testreport.TestSuite }{ + // Passing test cases - no failure, error, or skipped { - name: "regroup error message", + name: "Passing test case - no details", suites: []TestSuite{{TestCases: []TestCase{ { - Error: &Error{ - Message: "error message", - }, + Name: "passingTest", + ClassName: "TestClass", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "Error message:\nerror message", - }, + Name: "passingTest", + ClassName: "TestClass", }, }}}, }, + { - name: "regroup error body", + name: "Passing test case - system error and output", suites: []TestSuite{{TestCases: []TestCase{ { - Error: &Error{ - Value: "error message", - }, + Name: "passingTestWithOutput", + ClassName: "TestClass", + SystemOut: "Test passed\nExecution time: 100ms", + SystemErr: "Info: deprecated method used", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "Error value:\nerror message", - }, + Name: "passingTestWithOutput", + ClassName: "TestClass", + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Test passed\nExecution time: 100ms"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Info: deprecated method used"}, }, }}}, }, + + // Failure test cases { - name: "regroup system err", + name: "Failure - no details", suites: []TestSuite{{TestCases: []TestCase{ { - SystemErr: "error message", + Name: "failureNoDetailsTest", + Failure: &Failure{}, }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "System error:\nerror message", - }, + Name: "failureNoDetailsTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: ""}, }, }}}, }, { - name: "regroup error message - multiple test cases", + name: "Failure - system error and output only", suites: []TestSuite{{TestCases: []TestCase{ { - Error: &Error{ - Message: "error message", - }, - }, - { - Error: &Error{ - Message: "error message2", - }, + Name: "failureSystemOutputTest", + Failure: &Failure{}, + SystemOut: "Test execution started", + SystemErr: "Error: NullPointerException", }, }}}, - want: []testreport.TestSuite{{Tests: 2, Failures: 2, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "Error message:\nerror message", - }, - }, - { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "Error message:\nerror message2", - }, + Name: "failureSystemOutputTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "System error:\nError: NullPointerException\n\nSystem output:\nTest execution started"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Test execution started"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Error: NullPointerException"}, }, }}}, }, + { - name: "regroup error body - multiple test cases", + name: "Failure - content only", suites: []TestSuite{{TestCases: []TestCase{ { - Error: &Error{ - Value: "error message", - }, + Name: "simpleFailureTest", + Failure: &Failure{Value: "assertion failed"}, }, + }}}, + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Error: &Error{ - Value: "error message2", - }, + Name: "simpleFailureTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "assertion failed"}, }, }}}, - want: []testreport.TestSuite{{Tests: 2, Failures: 2, Skipped: 0, TestCases: []testreport.TestCase{ + }, + + { + name: "Failure - all attributes and system output", + suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "Error value:\nerror message", - }, + Name: "complexFailureTest", + Failure: &Failure{Type: "AssertionError", Message: "expected true", Value: "Stack trace"}, + SystemOut: "Test output: started", + SystemErr: "Warning: deprecated API", }, + }}}, + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "Error value:\nerror message2", - }, + Name: "complexFailureTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "AssertionError: expected true\n\nStack trace\n\nSystem error:\nWarning: deprecated API\n\nSystem output:\nTest output: started"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Test output: started"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Warning: deprecated API"}, }, }}}, }, + + // Error test cases { - name: "regroup system err - multiple test cases", + name: "Error - no details", suites: []TestSuite{{TestCases: []TestCase{ { - SystemErr: "error message", + Name: "errorNoDetailsTest", + Error: &Error{}, }, + }}}, + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - SystemErr: "error message2", + Name: "errorNoDetailsTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: ""}, }, }}}, - want: []testreport.TestSuite{{Tests: 2, Failures: 2, Skipped: 0, TestCases: []testreport.TestCase{ + }, + + { + name: "Error - system error and output only", + suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "System error:\nerror message", - }, + Name: "errorSystemOutputTest", + Error: &Error{}, + SystemOut: "Initialization complete", + SystemErr: "Critical error occurred", }, + }}}, + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "System error:\nerror message2", - }, + Name: "errorSystemOutputTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "System error:\nCritical error occurred\n\nSystem output:\nInitialization complete"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Initialization complete"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Critical error occurred"}, }, }}}, }, + { - name: "should not touch failure", + name: "Error - content only", suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &Failure{ - Value: "error message", - }, + Name: "simpleErrorTest", + Error: &Error{Value: "null pointer"}, }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "error message", - }, + Name: "simpleErrorTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "null pointer"}, }, }}}, }, + { - name: "should append error body to failure", + name: "Error - all attributes and system output", suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &Failure{ - Value: "failure message", - }, - Error: &Error{ - Value: "error value", - }, + Name: "complexErrorTest", + Failure: &Failure{Type: "NullPointerException", Message: "object was null", Value: "Full stack trace"}, + SystemOut: "Test setup complete", + SystemErr: "System error: OutOfMemory", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "failure message\n\nError value:\nerror value", - }, + Name: "complexErrorTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "NullPointerException: object was null\n\nFull stack trace\n\nSystem error:\nSystem error: OutOfMemory\n\nSystem output:\nTest setup complete"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Test setup complete"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "System error: OutOfMemory"}, }, }}}, }, + + // Skipped test cases { - name: "should append error message to failure", + name: "Skipped - no details", suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &Failure{ - Value: "Failure message", - }, - Error: &Error{ - Message: "error value", - }, + Name: "skippedNoDetailsTest", + Skipped: &Skipped{}, }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 1, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "Failure message\n\nError message:\nerror value", - }, + Name: "skippedNoDetailsTest", + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: ""}, }, }}}, }, + { - name: "should append system error to failure", + name: "Skipped - system error and output only", suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &Failure{ - Value: "failure message", - }, - SystemErr: "error value", + Name: "skippedSystemOutputTest", + Skipped: &Skipped{}, + SystemOut: "Skipping test due to unmet prerequisites", + SystemErr: "Warning: missing configuration", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 1, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "failure message\n\nSystem error:\nerror value", - }, + Name: "skippedSystemOutputTest", + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: "System error:\nWarning: missing configuration\n\nSystem output:\nSkipping test due to unmet prerequisites"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Skipping test due to unmet prerequisites"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Warning: missing configuration"}, }, }}}, }, + { - name: "should append system error, error message, error body to failure", + name: "Skipped - content only", suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &Failure{ - Value: "failure message", - }, - SystemErr: "error value", + Name: "simpleSkippedTest", + Skipped: &Skipped{Message: "Test not implemented"}, }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 1, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "failure message\n\nSystem error:\nerror value", - }, + Name: "simpleSkippedTest", + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: "Test not implemented"}, }, }}}, }, + { - name: "should append system error, error message, error body to failure", + name: "Skipped - all attributes and system output", suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &Failure{ - Message: "failure message", - Value: "failure content", - }, - SystemErr: "error value", - Error: &Error{ - Message: "message", - Value: "value", - }, + Name: "complexSkippedTest", + Skipped: &Skipped{Message: "Environment not ready"}, + SystemOut: "Setup started but aborted", + SystemErr: "Resource pool unavailable", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 1, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "failure message\n\nfailure content\n\nError message:\nmessage\n\nError value:\nvalue\n\nSystem error:\nerror value", - }, + Name: "complexSkippedTest", + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: "Environment not ready\n\nSystem error:\nResource pool unavailable\n\nSystem output:\nSetup started but aborted"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Setup started but aborted"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Resource pool unavailable"}, }, }}}, }, + + // Multiple test cases in one suite { - name: "Should convert Message attribute of Failure element to the value of a Failure element", + name: "Multiple test cases with mixed results", suites: []TestSuite{{TestCases: []TestCase{ { - Failure: &Failure{ - Message: "ErrorMsg", - }, + Name: "passingTest", + ClassName: "TestClass", + }, + { + Name: "failureTest", + Failure: &Failure{Value: "expected true"}, + }, + { + Name: "errorTest", + Error: &Error{Value: "null pointer"}, + }, + { + Name: "skippedTest", + Skipped: &Skipped{}, + }, + { + Name: "anotherPassingTest", + ClassName: "TestClass", + SystemOut: "All good", + }, + { + Name: "anotherFailureTest", + Failure: &Failure{Type: "AssertionError", Message: "value mismatch", Value: "Stack trace"}, + SystemErr: "Error details", + }, + { + Name: "anotherErrorTest", + Error: &Error{Type: "IOException", Message: "file not found", Value: "Stack trace"}, + SystemOut: "Attempted to read file", + SystemErr: "File read error", + }, + { + Name: "anotherSkippedTest", + Skipped: &Skipped{Message: "Dependency missing"}, + SystemOut: "Skipping due to missing dependency", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 8, Failures: 4, Errors: 0, Skipped: 2, TestCases: []testreport.TestCase{ { - Failure: &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: "ErrorMsg", - }, + Name: "passingTest", + ClassName: "TestClass", + }, + { + Name: "failureTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "expected true"}, + }, + { + Name: "errorTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "null pointer"}, + }, + { + Name: "skippedTest", + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: ""}, + }, + { + Name: "anotherPassingTest", + ClassName: "TestClass", + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "All good"}, + }, + { + Name: "anotherFailureTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "AssertionError: value mismatch\n\nStack trace\n\nSystem error:\nError details"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Error details"}, + }, + { + Name: "anotherErrorTest", + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "IOException: file not found\n\nStack trace\n\nSystem error:\nFile read error\n\nSystem output:\nAttempted to read file"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Attempted to read file"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "File read error"}, + }, + { + Name: "anotherSkippedTest", + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: "Dependency missing\n\nSystem output:\nSkipping due to missing dependency"}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Skipping due to missing dependency"}, }, }}}, }, @@ -365,7 +427,7 @@ func TestConverter_Convert(t *testing.T) { Time: 0.17543494701385498, Failure: &testreport.Failure{ XMLName: xml.Name{Local: "failure"}, - Value: "XCTAssertTrue failed", + Value: "Failure message: XCTAssertTrue failed", }, }, }, @@ -507,7 +569,7 @@ Testmo test management software - https://www.testmo.com/ XMLName: xml.Name{ Local: "failure", }, - Value: "Assertion error message", + Value: "Failure type: AssertionError\n\nFailure message: Assertion error message", }, }, }, @@ -576,18 +638,24 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { { XMLName: xml.Name{Space: "", Local: "testsuite"}, Name: "My Test Suite", - Tests: 7, Failures: 6, Skipped: 1, + Tests: 7, Failures: 5, Errors: 1, Skipped: 1, Time: 28.844, TestCases: []testreport.TestCase{ {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 1", ClassName: "example.exampleTest", Time: 0.764, - Failure: nil, - Skipped: &testreport.Skipped{XMLName: xml.Name{Space: "", Local: "skipped"}}}, + Failure: nil, + Skipped: &testreport.Skipped{XMLName: xml.Name{Space: "", Local: "skipped"}}, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Space: "", Local: "system-out"}, Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 1\n[INFO] 13:12:43:\tLog line 2 1\n"}}, {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 2", ClassName: "example.exampleTest", Time: 0.164, - Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nSome error message 2"}, - Skipped: nil}, + Failure: nil, + Skipped: nil, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Space: "", Local: "system-out"}, Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 2\n[INFO] 13:12:43:\tLog line 2 2\n"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Space: "", Local: "system-err"}, Value: "Some error message 2"}}, {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0.445, - Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "Failure message\n\nError value:\nError\n\nSystem error:\nSome error message 3"}, - Skipped: nil}, + Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "Failure message"}, + Error: &testreport.Error{XMLName: xml.Name{Space: "", Local: "error"}, Value: "Error"}, + Skipped: nil, + SystemOut: &testreport.SystemOut{XMLName: xml.Name{Space: "", Local: "system-out"}, Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 3\n[INFO] 13:12:43:\tLog line 2 3\n"}, + SystemErr: &testreport.SystemErr{XMLName: xml.Name{Space: "", Local: "system-err"}, Value: "Some error message 3"}}, {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nFlaky failure system error"}, Skipped: nil}, From 327d70ecb6a526b387c68fd287941ccbd277ffc8 Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Tue, 16 Dec 2025 16:47:04 +0100 Subject: [PATCH 4/5] test: fix tests --- test/converters/converters_test.go | 2 + test/converters/junitxml/junitxml.go | 2 +- test/converters/junitxml/junitxml_test.go | 151 ++++++++++++++---- .../ios_device_config_xml_output.golden | 4 +- test/testdata/ios_xml_output.golden | 12 +- 5 files changed, 129 insertions(+), 42 deletions(-) diff --git a/test/converters/converters_test.go b/test/converters/converters_test.go index f1c4cef0..dddb4a48 100644 --- a/test/converters/converters_test.go +++ b/test/converters/converters_test.go @@ -26,6 +26,7 @@ func TestXCresult3Converters(t *testing.T) { Name: "rtgtrghtrgTests", Tests: 2, Failures: 0, + Errors: 0, Time: 0.26063, TestCases: []testreport.TestCase{ { // plain test case @@ -44,6 +45,7 @@ func TestXCresult3Converters(t *testing.T) { Name: "rtgtrghtrgUITests", Tests: 15, Failures: 3, + Errors: 0, Time: 0.759, TestCases: []testreport.TestCase{ // class rtgtrghtrg3UITests: XCTestCase inside rtgtrghtrgUITests class diff --git a/test/converters/junitxml/junitxml.go b/test/converters/junitxml/junitxml.go index 36841e54..119f8140 100644 --- a/test/converters/junitxml/junitxml.go +++ b/test/converters/junitxml/junitxml.go @@ -330,7 +330,7 @@ func convertErrorToFailure(error *testreport.Error) *testreport.Failure { } func enrichWithSystemOutputs(testCase *testreport.TestCase, systemOut, systemErr string) { - var testOutputs []string = []string{} + testOutputs := []string{} if len(strings.TrimSpace(systemErr)) > 0 { testOutputs = append(testOutputs, "System error:\n"+systemErr) diff --git a/test/converters/junitxml/junitxml_test.go b/test/converters/junitxml/junitxml_test.go index d4a5d02c..55bd32c9 100644 --- a/test/converters/junitxml/junitxml_test.go +++ b/test/converters/junitxml/junitxml_test.go @@ -427,7 +427,7 @@ func TestConverter_Convert(t *testing.T) { Time: 0.17543494701385498, Failure: &testreport.Failure{ XMLName: xml.Name{Local: "failure"}, - Value: "Failure message: XCTAssertTrue failed", + Value: "XCTAssertTrue failed", }, }, }, @@ -519,6 +519,7 @@ Testmo test management software - https://www.testmo.com/ Name: "Tests.Registration", Tests: 3, Failures: 0, + Errors: 0, Time: 6.605871, TestCases: []testreport.TestCase{ { @@ -569,7 +570,7 @@ Testmo test management software - https://www.testmo.com/ XMLName: xml.Name{ Local: "failure", }, - Value: "Failure type: AssertionError\n\nFailure message: Assertion error message", + Value: "AssertionError: Assertion error message", }, }, }, @@ -636,38 +637,122 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { XMLName: xml.Name{Space: "", Local: ""}, TestSuites: []testreport.TestSuite{ { - XMLName: xml.Name{Space: "", Local: "testsuite"}, - Name: "My Test Suite", - Tests: 7, Failures: 5, Errors: 1, Skipped: 1, - Time: 28.844, + XMLName: xml.Name{Space: "", Local: "testsuite"}, + Name: "My Test Suite", + Tests: 7, + Failures: 5, + Errors: 0, + Skipped: 1, + Time: 28.844, TestCases: []testreport.TestCase{ - {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 1", ClassName: "example.exampleTest", Time: 0.764, - Failure: nil, - Skipped: &testreport.Skipped{XMLName: xml.Name{Space: "", Local: "skipped"}}, - SystemOut: &testreport.SystemOut{XMLName: xml.Name{Space: "", Local: "system-out"}, Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 1\n[INFO] 13:12:43:\tLog line 2 1\n"}}, - {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 2", ClassName: "example.exampleTest", Time: 0.164, - Failure: nil, - Skipped: nil, - SystemOut: &testreport.SystemOut{XMLName: xml.Name{Space: "", Local: "system-out"}, Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 2\n[INFO] 13:12:43:\tLog line 2 2\n"}, - SystemErr: &testreport.SystemErr{XMLName: xml.Name{Space: "", Local: "system-err"}, Value: "Some error message 2"}}, - {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0.445, - Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "Failure message"}, - Error: &testreport.Error{XMLName: xml.Name{Space: "", Local: "error"}, Value: "Error"}, - Skipped: nil, - SystemOut: &testreport.SystemOut{XMLName: xml.Name{Space: "", Local: "system-out"}, Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 3\n[INFO] 13:12:43:\tLog line 2 3\n"}, - SystemErr: &testreport.SystemErr{XMLName: xml.Name{Space: "", Local: "system-err"}, Value: "Some error message 3"}}, - {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, - Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nFlaky failure system error"}, - Skipped: nil}, - {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, - Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nFlaky error system error"}, - Skipped: nil}, - {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, - Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nRerun failure system error"}, - Skipped: nil}, - {XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", Name: "Testcase number 3", ClassName: "example.exampleTest", Time: 0, - Failure: &testreport.Failure{XMLName: xml.Name{Space: "", Local: "failure"}, Value: "System error:\nRerun error system error"}, - Skipped: nil}, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 1", + ClassName: "example.exampleTest", + Time: 0.764, + Failure: nil, + Error: nil, + Skipped: &testreport.Skipped{ + XMLName: xml.Name{Space: "", Local: "skipped"}, + Value: "System output:\n[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 1\n[INFO] 13:12:43:\tLog line 2 1\n", + }, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 1\n[INFO] 13:12:43:\tLog line 2 1\n", + }, + }, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 2", + ClassName: "example.exampleTest", + Time: 0.164, + Failure: nil, + Skipped: nil, + Error: nil, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 2\n[INFO] 13:12:43:\tLog line 2 2\n", + }, + SystemErr: &testreport.SystemErr{ + XMLName: xml.Name{Space: "", Local: "system-err"}, + Value: "Some error message 2", + }, + }, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 3", + ClassName: "example.exampleTest", + Time: 0.445, + Error: nil, + Skipped: nil, + Failure: &testreport.Failure{ + XMLName: xml.Name{Space: "", Local: "failure"}, + Value: "Error\n\nSystem error:\nSome error message 3\n\nSystem output:\n[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 3\n[INFO] 13:12:43:\tLog line 2 3\n", + }, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 3\n[INFO] 13:12:43:\tLog line 2 3\n", + }, + SystemErr: &testreport.SystemErr{ + XMLName: xml.Name{Space: "", Local: "system-err"}, + Value: "Some error message 3", + }, + }, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 3", + ClassName: "example.exampleTest", + Time: 0, + Skipped: nil, + Error: nil, + Failure: &testreport.Failure{ + XMLName: xml.Name{Space: "", Local: "failure"}, + Value: "System error:\nFlaky failure system error", + }, + }, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 3", + ClassName: "example.exampleTest", + Time: 0, + Skipped: nil, + Error: nil, + Failure: &testreport.Failure{ + XMLName: xml.Name{Space: "", Local: "failure"}, + Value: "System error:\nFlaky error system error", + }, + }, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 3", + ClassName: "example.exampleTest", + Time: 0, + Skipped: nil, + Error: nil, + Failure: &testreport.Failure{ + XMLName: xml.Name{Space: "", Local: "failure"}, + Value: "System error:\nRerun failure system error", + }, + }, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 3", + ClassName: "example.exampleTest", + Time: 0, + Skipped: nil, + Error: nil, + Failure: &testreport.Failure{ + XMLName: xml.Name{Space: "", Local: "failure"}, + Value: "System error:\nRerun error system error", + }, + }, }, }, }, diff --git a/test/testdata/ios_device_config_xml_output.golden b/test/testdata/ios_device_config_xml_output.golden index 66849f25..3b5c1540 100644 --- a/test/testdata/ios_device_config_xml_output.golden +++ b/test/testdata/ios_device_config_xml_output.golden @@ -1,13 +1,13 @@ - + DarkAndLightModeTests.swift:25: failed - Reached 1 - + diff --git a/test/testdata/ios_xml_output.golden b/test/testdata/ios_xml_output.golden index 804515dd..7c7d82a4 100644 --- a/test/testdata/ios_xml_output.golden +++ b/test/testdata/ios_xml_output.golden @@ -1,32 +1,32 @@ - + - + - + - + /Users/vagrant/git/BitriseApp/BitriseAppUITests/BitriseOMUITests.swift:70 - XCTAssertTrue failed - No session found - + - + From 65d1b7f0bd622be254f2a2c8502320061a31c5ac Mon Sep 17 00:00:00 2001 From: Zoltan Szabo Date: Wed, 17 Dec 2025 16:29:37 +0100 Subject: [PATCH 5/5] feat: separate Failure, Error, system-out and system-err --- test/converters/junitxml/junitxml.go | 202 +++++------------- test/converters/junitxml/junitxml_test.go | 144 +++++++++---- test/converters/junitxml/models.go | 53 +++-- .../junitxml/testdata/flaky_test.xml | 10 +- .../ios_device_config_xml_output.golden | 12 +- test/testdata/ios_xml_output.golden | 42 ++-- test/testreport/test_report.go | 27 ++- 7 files changed, 244 insertions(+), 246 deletions(-) diff --git a/test/converters/junitxml/junitxml.go b/test/converters/junitxml/junitxml.go index 119f8140..90f62b96 100644 --- a/test/converters/junitxml/junitxml.go +++ b/test/converters/junitxml/junitxml.go @@ -5,7 +5,7 @@ import ( "strings" "github.com/bitrise-steplib/steps-deploy-to-bitrise-io/test/testreport" - "github.com/pkg/errors" + errorPkg "github.com/pkg/errors" ) func (c *Converter) Setup(_ bool) {} @@ -54,13 +54,9 @@ func parseTestReport(result resultReader) (TestReport, error) { return TestReport{TestSuites: []TestSuite{testSuite}}, nil } - return TestReport{}, errors.Wrap(errors.Wrap(testSuiteErr, string(data)), testReportErr.Error()) + return TestReport{}, errorPkg.Wrap(errorPkg.Wrap(testSuiteErr, string(data)), testReportErr.Error()) } -// merges Suites->Cases->Error and Suites->Cases->SystemErr field values into Suites->Cases->Failure field -// with 2 newlines and error category prefix -// the two newlines applied only if there is a failure message already -// this is required because our testing service currently handles failure field properly func convertTestReport(report TestReport) testreport.TestReport { convertedReport := testreport.TestReport{ XMLName: report.XMLName, @@ -76,9 +72,12 @@ func convertTestReport(report TestReport) testreport.TestReport { func convertTestSuite(testSuite TestSuite) testreport.TestSuite { convertedTestSuite := testreport.TestSuite{ - XMLName: testSuite.XMLName, - Name: testSuite.Name, - Time: testSuite.Time, + XMLName: testSuite.XMLName, + Name: testSuite.Name, + Time: testSuite.Time, + Assertions: testSuite.Assertions, + Timestamp: testSuite.Timestamp, + File: testSuite.File, } tests := 0 @@ -94,6 +93,9 @@ func convertTestSuite(testSuite TestSuite) testreport.TestSuite { if convertedTestCase.Failure != nil { failures++ } + if convertedTestCase.Error != nil { + errors++ + } if convertedTestCase.Skipped != nil { skipped++ } @@ -131,22 +133,49 @@ func flattenGroupedTestCases(testCases []TestCase) []TestCase { } for _, flakyFailure := range testCase.FlakyFailures { - flattenedTestCase.Failure = convertToFailure(flakyFailure.Type, flakyFailure.Message, flakyFailure.SystemErr) + flattenedTestCase.Failure = &Failure{ + Type: flakyFailure.Type, + Message: flakyFailure.Message, + Value: flakyFailure.Value, + } + flattenedTestCase.SystemErr = flakyFailure.SystemErr + flattenedTestCase.SystemOut = flakyFailure.SystemOut flattenedTestCases = append(flattenedTestCases, flattenedTestCase) } + flattenedTestCase.Failure = nil for _, flakyError := range testCase.FlakyErrors { - flattenedTestCase.Failure = convertToFailure(flakyError.Type, flakyError.Message, flakyError.SystemErr) + flattenedTestCase.Error = &Error{ + Type: flakyError.Type, + Message: flakyError.Message, + Value: flakyError.Value, + } + flattenedTestCase.SystemErr = flakyError.SystemErr + flattenedTestCase.SystemOut = flakyError.SystemOut flattenedTestCases = append(flattenedTestCases, flattenedTestCase) } + flattenedTestCase.Error = nil for _, rerunfailure := range testCase.RerunFailures { - flattenedTestCase.Failure = convertToFailure(rerunfailure.Type, rerunfailure.Message, rerunfailure.SystemErr) + flattenedTestCase.Failure = &Failure{ + Type: rerunfailure.Type, + Message: rerunfailure.Message, + Value: rerunfailure.Value, + } + flattenedTestCase.SystemErr = rerunfailure.SystemErr + flattenedTestCase.SystemOut = rerunfailure.SystemOut flattenedTestCases = append(flattenedTestCases, flattenedTestCase) } + flattenedTestCase.Failure = nil for _, rerunError := range testCase.RerunErrors { - flattenedTestCase.Failure = convertToFailure(rerunError.Type, rerunError.Message, rerunError.SystemErr) + flattenedTestCase.Error = &Error{ + Type: rerunError.Type, + Message: rerunError.Message, + Value: rerunError.Value, + } + flattenedTestCase.SystemErr = rerunError.SystemErr + flattenedTestCase.SystemOut = rerunError.SystemOut flattenedTestCases = append(flattenedTestCases, flattenedTestCase) } @@ -154,33 +183,6 @@ func flattenGroupedTestCases(testCases []TestCase) []TestCase { return flattenedTestCases } -func convertToFailure(itemType, failureMessage, systemErr string) *Failure { - var message string - if len(strings.TrimSpace(itemType)) > 0 { - message = itemType - } - if len(strings.TrimSpace(failureMessage)) > 0 { - if len(message) > 0 { - message += ": " - } - message += failureMessage - } - - if len(strings.TrimSpace(systemErr)) > 0 { - if len(message) > 0 { - message += "\n\n" - } - message += "System error:\n" + systemErr - } - - if len(message) > 0 { - return &Failure{ - Value: message, - } - } - return nil -} - func convertTestCase(testCase TestCase) testreport.TestCase { convertedTestCase := testreport.TestCase{ XMLName: testCase.XMLName, @@ -188,21 +190,17 @@ func convertTestCase(testCase TestCase) testreport.TestCase { Name: testCase.Name, ClassName: testCase.ClassName, Time: testCase.Time, + Assertions: testCase.Assertions, + File: testCase.File, + Line: testCase.Line, Failure: convertFailure(testCase.Failure), Error: convertError(testCase.Error), Skipped: convertSkipped(testCase.Skipped), + Properties: convertProperties(testCase.Properties), SystemOut: convertSystemOut(testCase.SystemOut), SystemErr: convertSystemErr(testCase.SystemErr), - Properties: convertProperties(testCase.Properties), } - if convertedTestCase.Error != nil { - convertedTestCase.Failure = convertErrorToFailure(convertedTestCase.Error) - convertedTestCase.Error = nil - } - - enrichWithSystemOutputs(&convertedTestCase, testCase.SystemOut, testCase.SystemErr) - return convertedTestCase } @@ -224,24 +222,24 @@ func convertProperties(properties *Properties) *testreport.Properties { } func convertSystemOut(systemOut string) *testreport.SystemOut { - if systemOut == "" { + if len(strings.TrimSpace(systemOut)) == 0 { return nil } return &testreport.SystemOut{ XMLName: xml.Name{Local: "system-out"}, - Value: systemOut, + Value: strings.TrimSpace(systemOut), } } func convertSystemErr(systemErr string) *testreport.SystemErr { - if systemErr == "" { + if len(strings.TrimSpace(systemErr)) == 0 { return nil } return &testreport.SystemErr{ XMLName: xml.Name{Local: "system-err"}, - Value: systemErr, + Value: strings.TrimSpace(systemErr), } } @@ -250,15 +248,10 @@ func convertSkipped(skipped *Skipped) *testreport.Skipped { return nil } - var parts []string - - if len(strings.TrimSpace(skipped.Message)) > 0 { - parts = append(parts, strings.TrimSpace(skipped.Message)) - } - return &testreport.Skipped{ XMLName: xml.Name{Local: "skipped"}, - Value: strings.Join(parts, "\n\n"), + Message: skipped.Message, + Value: strings.TrimSpace(skipped.Value), } } @@ -267,26 +260,11 @@ func convertFailure(failure *Failure) *testreport.Failure { return nil } - var parts []string - - var attributes []string - if len(strings.TrimSpace(failure.Type)) > 0 { - attributes = append(attributes, failure.Type) - } - if len(strings.TrimSpace(failure.Message)) > 0 { - attributes = append(attributes, failure.Message) - } - if len(attributes) > 0 { - parts = append(parts, strings.Join(attributes, ": ")) - } - - if len(strings.TrimSpace(failure.Value)) > 0 { - parts = append(parts, failure.Value) - } - return &testreport.Failure{ XMLName: xml.Name{Local: "failure"}, - Value: strings.Join(parts, "\n\n"), + Type: failure.Type, + Message: failure.Message, + Value: strings.TrimSpace(failure.Value), } } @@ -295,74 +273,10 @@ func convertError(error *Error) *testreport.Error { return nil } - var parts []string - - var attributes []string - if len(strings.TrimSpace(error.Type)) > 0 { - attributes = append(attributes, error.Type) - } - if len(strings.TrimSpace(error.Message)) > 0 { - attributes = append(attributes, error.Message) - } - if len(attributes) > 0 { - parts = append(parts, strings.Join(attributes, ": ")) - } - - if len(strings.TrimSpace(error.Value)) > 0 { - parts = append(parts, error.Value) - } - return &testreport.Error{ XMLName: xml.Name{Local: "error"}, - Value: strings.Join(parts, "\n\n"), - } -} - -func convertErrorToFailure(error *testreport.Error) *testreport.Failure { - if error == nil { - return nil - } - - return &testreport.Failure{ - XMLName: xml.Name{Local: "failure"}, - Value: error.Value, - } -} - -func enrichWithSystemOutputs(testCase *testreport.TestCase, systemOut, systemErr string) { - testOutputs := []string{} - - if len(strings.TrimSpace(systemErr)) > 0 { - testOutputs = append(testOutputs, "System error:\n"+systemErr) - } - - if len(strings.TrimSpace(systemOut)) > 0 { - testOutputs = append(testOutputs, "System output:\n"+systemOut) - } - - if (len(testOutputs)) == 0 { - return - } - - combinedOutput := strings.Join(testOutputs, "\n\n") - - if testCase.Error != nil { - if len(strings.TrimSpace(testCase.Error.Value)) > 0 { - testCase.Error.Value = testCase.Error.Value + "\n\n" + combinedOutput - } else { - testCase.Error.Value = combinedOutput - } - } else if testCase.Failure != nil { - if len(strings.TrimSpace(testCase.Failure.Value)) > 0 { - testCase.Failure.Value = testCase.Failure.Value + "\n\n" + combinedOutput - } else { - testCase.Failure.Value = combinedOutput - } - } else if testCase.Skipped != nil { - if len(strings.TrimSpace(testCase.Skipped.Value)) > 0 { - testCase.Skipped.Value = testCase.Skipped.Value + "\n\n" + combinedOutput - } else { - testCase.Skipped.Value = combinedOutput - } + Type: error.Type, + Message: error.Message, + Value: strings.TrimSpace(error.Value), } } diff --git a/test/converters/junitxml/junitxml_test.go b/test/converters/junitxml/junitxml_test.go index 55bd32c9..a63ba356 100644 --- a/test/converters/junitxml/junitxml_test.go +++ b/test/converters/junitxml/junitxml_test.go @@ -102,7 +102,7 @@ func Test_convertTestReport(t *testing.T) { want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { Name: "failureSystemOutputTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "System error:\nError: NullPointerException\n\nSystem output:\nTest execution started"}, + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: ""}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Test execution started"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Error: NullPointerException"}, }, @@ -138,7 +138,7 @@ func Test_convertTestReport(t *testing.T) { want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ { Name: "complexFailureTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "AssertionError: expected true\n\nStack trace\n\nSystem error:\nWarning: deprecated API\n\nSystem output:\nTest output: started"}, + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Type: "AssertionError", Message: "expected true", Value: "Stack trace"}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Test output: started"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Warning: deprecated API"}, }, @@ -154,10 +154,10 @@ func Test_convertTestReport(t *testing.T) { Error: &Error{}, }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 1, Skipped: 0, TestCases: []testreport.TestCase{ { - Name: "errorNoDetailsTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: ""}, + Name: "errorNoDetailsTest", + Error: &testreport.Error{XMLName: xml.Name{Local: "error"}, Value: ""}, }, }}}, }, @@ -172,10 +172,10 @@ func Test_convertTestReport(t *testing.T) { SystemErr: "Critical error occurred", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Name: "errorSystemOutputTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "System error:\nCritical error occurred\n\nSystem output:\nInitialization complete"}, + Error: &testreport.Error{XMLName: xml.Name{Local: "error"}, Value: ""}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Initialization complete"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Critical error occurred"}, }, @@ -190,10 +190,10 @@ func Test_convertTestReport(t *testing.T) { Error: &Error{Value: "null pointer"}, }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 1, Skipped: 0, TestCases: []testreport.TestCase{ { - Name: "simpleErrorTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "null pointer"}, + Name: "simpleErrorTest", + Error: &testreport.Error{XMLName: xml.Name{Local: "error"}, Value: "null pointer"}, }, }}}, }, @@ -203,15 +203,15 @@ func Test_convertTestReport(t *testing.T) { suites: []TestSuite{{TestCases: []TestCase{ { Name: "complexErrorTest", - Failure: &Failure{Type: "NullPointerException", Message: "object was null", Value: "Full stack trace"}, + Error: &Error{Type: "NullPointerException", Message: "object was null", Value: "Full stack trace"}, SystemOut: "Test setup complete", SystemErr: "System error: OutOfMemory", }, }}}, - want: []testreport.TestSuite{{Tests: 1, Failures: 1, Errors: 0, Skipped: 0, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 1, Skipped: 0, TestCases: []testreport.TestCase{ { Name: "complexErrorTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "NullPointerException: object was null\n\nFull stack trace\n\nSystem error:\nSystem error: OutOfMemory\n\nSystem output:\nTest setup complete"}, + Error: &testreport.Error{XMLName: xml.Name{Local: "error"}, Type: "NullPointerException", Message: "object was null", Value: "Full stack trace"}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Test setup complete"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "System error: OutOfMemory"}, }, @@ -248,7 +248,7 @@ func Test_convertTestReport(t *testing.T) { want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 1, TestCases: []testreport.TestCase{ { Name: "skippedSystemOutputTest", - Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: "System error:\nWarning: missing configuration\n\nSystem output:\nSkipping test due to unmet prerequisites"}, + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: ""}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Skipping test due to unmet prerequisites"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Warning: missing configuration"}, }, @@ -260,7 +260,7 @@ func Test_convertTestReport(t *testing.T) { suites: []TestSuite{{TestCases: []TestCase{ { Name: "simpleSkippedTest", - Skipped: &Skipped{Message: "Test not implemented"}, + Skipped: &Skipped{Value: "Test not implemented"}, }, }}}, want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 1, TestCases: []testreport.TestCase{ @@ -276,7 +276,7 @@ func Test_convertTestReport(t *testing.T) { suites: []TestSuite{{TestCases: []TestCase{ { Name: "complexSkippedTest", - Skipped: &Skipped{Message: "Environment not ready"}, + Skipped: &Skipped{Message: "Environment not ready", Value: "Resource pool unavailable"}, SystemOut: "Setup started but aborted", SystemErr: "Resource pool unavailable", }, @@ -284,7 +284,7 @@ func Test_convertTestReport(t *testing.T) { want: []testreport.TestSuite{{Tests: 1, Failures: 0, Errors: 0, Skipped: 1, TestCases: []testreport.TestCase{ { Name: "complexSkippedTest", - Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: "Environment not ready\n\nSystem error:\nResource pool unavailable\n\nSystem output:\nSetup started but aborted"}, + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Message: "Environment not ready", Value: "Resource pool unavailable"}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Setup started but aborted"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Resource pool unavailable"}, }, @@ -333,7 +333,7 @@ func Test_convertTestReport(t *testing.T) { SystemOut: "Skipping due to missing dependency", }, }}}, - want: []testreport.TestSuite{{Tests: 8, Failures: 4, Errors: 0, Skipped: 2, TestCases: []testreport.TestCase{ + want: []testreport.TestSuite{{Tests: 8, Failures: 2, Errors: 2, Skipped: 2, TestCases: []testreport.TestCase{ { Name: "passingTest", ClassName: "TestClass", @@ -343,8 +343,8 @@ func Test_convertTestReport(t *testing.T) { Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "expected true"}, }, { - Name: "errorTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "null pointer"}, + Name: "errorTest", + Error: &testreport.Error{XMLName: xml.Name{Local: "error"}, Value: "null pointer"}, }, { Name: "skippedTest", @@ -357,18 +357,18 @@ func Test_convertTestReport(t *testing.T) { }, { Name: "anotherFailureTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "AssertionError: value mismatch\n\nStack trace\n\nSystem error:\nError details"}, + Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Type: "AssertionError", Message: "value mismatch", Value: "Stack trace"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "Error details"}, }, { Name: "anotherErrorTest", - Failure: &testreport.Failure{XMLName: xml.Name{Local: "failure"}, Value: "IOException: file not found\n\nStack trace\n\nSystem error:\nFile read error\n\nSystem output:\nAttempted to read file"}, + Error: &testreport.Error{XMLName: xml.Name{Local: "error"}, Type: "IOException", Message: "file not found", Value: "Stack trace"}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Attempted to read file"}, SystemErr: &testreport.SystemErr{XMLName: xml.Name{Local: "system-err"}, Value: "File read error"}, }, { Name: "anotherSkippedTest", - Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Value: "Dependency missing\n\nSystem output:\nSkipping due to missing dependency"}, + Skipped: &testreport.Skipped{XMLName: xml.Name{Local: "skipped"}, Message: "Dependency missing"}, SystemOut: &testreport.SystemOut{XMLName: xml.Name{Local: "system-out"}, Value: "Skipping due to missing dependency"}, }, }}}, @@ -427,7 +427,7 @@ func TestConverter_Convert(t *testing.T) { Time: 0.17543494701385498, Failure: &testreport.Failure{ XMLName: xml.Name{Local: "failure"}, - Value: "XCTAssertTrue failed", + Message: "XCTAssertTrue failed", }, }, }, @@ -570,7 +570,8 @@ Testmo test management software - https://www.testmo.com/ XMLName: xml.Name{ Local: "failure", }, - Value: "AssertionError: Assertion error message", + Type: "AssertionError", + Message: "Assertion error message", }, }, }, @@ -639,9 +640,9 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { { XMLName: xml.Name{Space: "", Local: "testsuite"}, Name: "My Test Suite", - Tests: 7, - Failures: 5, - Errors: 0, + Tests: 8, + Failures: 3, + Errors: 3, Skipped: 1, Time: 28.844, TestCases: []testreport.TestCase{ @@ -655,11 +656,11 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { Error: nil, Skipped: &testreport.Skipped{ XMLName: xml.Name{Space: "", Local: "skipped"}, - Value: "System output:\n[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 1\n[INFO] 13:12:43:\tLog line 2 1\n", + Value: "Skipped", }, SystemOut: &testreport.SystemOut{ XMLName: xml.Name{Space: "", Local: "system-out"}, - Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 1\n[INFO] 13:12:43:\tLog line 2 1\n", + Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 1\n[INFO] 13:12:43:\tLog line 2 1", }, }, { @@ -673,7 +674,7 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { Error: nil, SystemOut: &testreport.SystemOut{ XMLName: xml.Name{Space: "", Local: "system-out"}, - Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 2\n[INFO] 13:12:43:\tLog line 2 2\n", + Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 2\n[INFO] 13:12:43:\tLog line 2 2", }, SystemErr: &testreport.SystemErr{ XMLName: xml.Name{Space: "", Local: "system-err"}, @@ -690,11 +691,11 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { Skipped: nil, Failure: &testreport.Failure{ XMLName: xml.Name{Space: "", Local: "failure"}, - Value: "Error\n\nSystem error:\nSome error message 3\n\nSystem output:\n[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 3\n[INFO] 13:12:43:\tLog line 2 3\n", + Value: "Failure message", }, SystemOut: &testreport.SystemOut{ XMLName: xml.Name{Space: "", Local: "system-out"}, - Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 3\n[INFO] 13:12:43:\tLog line 2 3\n", + Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 3\n[INFO] 13:12:43:\tLog line 2 3", }, SystemErr: &testreport.SystemErr{ XMLName: xml.Name{Space: "", Local: "system-err"}, @@ -711,7 +712,15 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { Error: nil, Failure: &testreport.Failure{ XMLName: xml.Name{Space: "", Local: "failure"}, - Value: "System error:\nFlaky failure system error", + Value: "", + }, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "Flaky failure system out", + }, + SystemErr: &testreport.SystemErr{ + XMLName: xml.Name{Space: "", Local: "system-err"}, + Value: "Flaky failure system error", }, }, { @@ -724,33 +733,78 @@ func TestConverter_Convert_Grouped_report(t *testing.T) { Error: nil, Failure: &testreport.Failure{ XMLName: xml.Name{Space: "", Local: "failure"}, - Value: "System error:\nFlaky error system error", + Value: "", + }, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "Rerun failure system out", + }, + SystemErr: &testreport.SystemErr{ + XMLName: xml.Name{Space: "", Local: "system-err"}, + Value: "Rerun failure system error", }, }, { XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", - Name: "Testcase number 3", + Name: "Testcase number 4", + ClassName: "example.exampleTest", + Time: 0.352, + Skipped: nil, + Failure: nil, + Error: &testreport.Error{ + XMLName: xml.Name{Space: "", Local: "error"}, + Value: "Error message", + }, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "[INFO] 13:12:43:\n[INFO] 13:12:43:\tLog line 1 4\n[INFO] 13:12:43:\tLog line 2 4", + }, + SystemErr: &testreport.SystemErr{ + XMLName: xml.Name{Space: "", Local: "system-err"}, + Value: "Some error message 4", + }, + }, + { + XMLName: xml.Name{Space: "", Local: "testcase"}, + ConfigurationHash: "", + Name: "Testcase number 4", ClassName: "example.exampleTest", Time: 0, Skipped: nil, - Error: nil, - Failure: &testreport.Failure{ - XMLName: xml.Name{Space: "", Local: "failure"}, - Value: "System error:\nRerun failure system error", + Failure: nil, + Error: &testreport.Error{ + XMLName: xml.Name{Space: "", Local: "error"}, + Value: "", + }, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "Flaky error system out", + }, + SystemErr: &testreport.SystemErr{ + XMLName: xml.Name{Space: "", Local: "system-err"}, + Value: "Flaky error system error", }, }, { XMLName: xml.Name{Space: "", Local: "testcase"}, ConfigurationHash: "", - Name: "Testcase number 3", + Name: "Testcase number 4", ClassName: "example.exampleTest", Time: 0, Skipped: nil, - Error: nil, - Failure: &testreport.Failure{ - XMLName: xml.Name{Space: "", Local: "failure"}, - Value: "System error:\nRerun error system error", + Failure: nil, + Error: &testreport.Error{ + XMLName: xml.Name{Space: "", Local: "error"}, + Value: "", + }, + SystemOut: &testreport.SystemOut{ + XMLName: xml.Name{Space: "", Local: "system-out"}, + Value: "Rerun error system out", + }, + SystemErr: &testreport.SystemErr{ + XMLName: xml.Name{Space: "", Local: "system-err"}, + Value: "Rerun error system error", }, }, }, diff --git a/test/converters/junitxml/models.go b/test/converters/junitxml/models.go index f0df7480..a7dbc4eb 100644 --- a/test/converters/junitxml/models.go +++ b/test/converters/junitxml/models.go @@ -16,24 +16,30 @@ type TestSuite struct { Name string `xml:"name,attr"` Tests int `xml:"tests,attr"` Failures int `xml:"failures,attr"` - Skipped int `xml:"skipped,attr"` Errors int `xml:"errors,attr"` + Skipped int `xml:"skipped,attr"` + Assertions int `xml:"assertions,attr,omitempty"` Time float64 `xml:"time,attr"` - TestCases []TestCase `xml:"testcase"` - TestSuites []TestSuite `xml:"testsuite"` + Timestamp string `xml:"timestamp,attr,omitempty"` + File string `xml:"file,attr,omitempty"` + TestCases []TestCase `xml:"testcase,omitempty"` + TestSuites []TestSuite `xml:"testsuite,omitempty"` } // TestCase ... type TestCase struct { XMLName xml.Name `xml:"testcase"` - ConfigurationHash string `xml:"configuration-hash,attr"` + ConfigurationHash string `xml:"configuration-hash,attr,omitempty"` Name string `xml:"name,attr"` ClassName string `xml:"classname,attr"` + Assertions int `xml:"assertions,attr,omitempty"` Time float64 `xml:"time,attr"` + File string `xml:"file,attr,omitempty"` + Line int `xml:"line,attr,omitempty"` Failure *Failure `xml:"failure,omitempty"` - Properties *Properties `xml:"properties,omitempty"` - Skipped *Skipped `xml:"skipped,omitempty"` Error *Error `xml:"error,omitempty"` + Skipped *Skipped `xml:"skipped,omitempty"` + Properties *Properties `xml:"properties,omitempty"` SystemErr string `xml:"system-err,omitempty"` SystemOut string `xml:"system-out,omitempty"` @@ -45,32 +51,36 @@ type TestCase struct { type FlakyFailure struct { XMLName xml.Name `xml:"flakyFailure"` - Message string `xml:"message,attr"` - Type string `xml:"type,attr"` + Type string `xml:"type,attr,omitempty"` + Message string `xml:"message,attr,omitempty"` + Value string `xml:",chardata"` SystemErr string `xml:"system-err,omitempty"` SystemOut string `xml:"system-out,omitempty"` } type FlakyError struct { XMLName xml.Name `xml:"flakyError"` - Message string `xml:"message,attr"` - Type string `xml:"type,attr"` + Type string `xml:"type,attr,omitempty"` + Message string `xml:"message,attr,omitempty"` + Value string `xml:",chardata"` SystemErr string `xml:"system-err,omitempty"` SystemOut string `xml:"system-out,omitempty"` } type RerunFailure struct { XMLName xml.Name `xml:"rerunFailure"` - Message string `xml:"message,attr"` - Type string `xml:"type,attr"` + Type string `xml:"type,attr,omitempty"` + Message string `xml:"message,attr,omitempty"` + Value string `xml:",chardata"` SystemErr string `xml:"system-err,omitempty"` SystemOut string `xml:"system-out,omitempty"` } type RerunError struct { XMLName xml.Name `xml:"rerunError"` - Message string `xml:"message,attr"` - Type string `xml:"type,attr"` + Type string `xml:"type,attr,omitempty"` + Message string `xml:"message,attr,omitempty"` + Value string `xml:",chardata"` SystemErr string `xml:"system-err,omitempty"` SystemOut string `xml:"system-out,omitempty"` } @@ -78,8 +88,16 @@ type RerunError struct { // Failure ... type Failure struct { XMLName xml.Name `xml:"failure,omitempty"` + Type string `xml:"type,attr,omitempty"` Message string `xml:"message,attr,omitempty"` + Value string `xml:",chardata"` +} + +// Error ... +type Error struct { + XMLName xml.Name `xml:"error,omitempty"` Type string `xml:"type,attr,omitempty"` + Message string `xml:"message,attr,omitempty"` Value string `xml:",chardata"` } @@ -87,13 +105,6 @@ type Failure struct { type Skipped struct { XMLName xml.Name `xml:"skipped,omitempty"` Message string `xml:"message,attr,omitempty"` -} - -// Error ... -type Error struct { - XMLName xml.Name `xml:"error,omitempty"` - Message string `xml:"message,attr,omitempty"` - Type string `xml:"type,attr,omitempty"` Value string `xml:",chardata"` } diff --git a/test/converters/junitxml/testdata/flaky_test.xml b/test/converters/junitxml/testdata/flaky_test.xml index 5e3a14a5..ecda1d37 100644 --- a/test/converters/junitxml/testdata/flaky_test.xml +++ b/test/converters/junitxml/testdata/flaky_test.xml @@ -40,6 +40,14 @@ Flaky failure system out Flaky failure system error + + + + + Error message Rerun error stack trace Rerun error system out @@ -50,6 +58,6 @@ Flaky error system out Flaky error system error - Error + \ No newline at end of file diff --git a/test/testdata/ios_device_config_xml_output.golden b/test/testdata/ios_device_config_xml_output.golden index 3b5c1540..3437627c 100644 --- a/test/testdata/ios_device_config_xml_output.golden +++ b/test/testdata/ios_device_config_xml_output.golden @@ -1,16 +1,16 @@ - + DarkAndLightModeTests.swift:25: failed - Reached 1 - - + + - - - + + + diff --git a/test/testdata/ios_xml_output.golden b/test/testdata/ios_xml_output.golden index 7c7d82a4..d53e11e9 100644 --- a/test/testdata/ios_xml_output.golden +++ b/test/testdata/ios_xml_output.golden @@ -1,40 +1,40 @@ - - - - + + + + - + - - + + - + /Users/vagrant/git/BitriseApp/BitriseAppUITests/BitriseOMUITests.swift:70 - XCTAssertTrue failed - No session found - - + + - - + + - - - - - - - - - + + + + + + + + + \ No newline at end of file diff --git a/test/testreport/test_report.go b/test/testreport/test_report.go index 82ea8c92..31faa430 100644 --- a/test/testreport/test_report.go +++ b/test/testreport/test_report.go @@ -17,39 +17,50 @@ type TestSuite struct { Failures int `xml:"failures,attr"` Errors int `xml:"errors,attr"` Skipped int `xml:"skipped,attr"` + Assertions int `xml:"assertions,attr,omitempty"` Time float64 `xml:"time,attr"` - TestCases []TestCase `xml:"testcase"` - TestSuites []TestSuite `xml:"testsuite"` + Timestamp string `xml:"timestamp,attr,omitempty"` + File string `xml:"file,attr,omitempty"` + TestCases []TestCase `xml:"testcase,omitempty"` + TestSuites []TestSuite `xml:"testsuite,omitempty"` } type TestCase struct { XMLName xml.Name `xml:"testcase"` // ConfigurationHash is used to distinguish the same test case runs, // performed with different build configurations (e.g., Debug vs. Release) or different devices/simulators - ConfigurationHash string `xml:"configuration-hash,attr"` + ConfigurationHash string `xml:"configuration-hash,attr,omitempty"` Name string `xml:"name,attr"` ClassName string `xml:"classname,attr"` + Assertions int `xml:"assertions,attr,omitempty"` Time float64 `xml:"time,attr"` - Error *Error `xml:"error,omitempty"` + File string `xml:"file,attr,omitempty"` + Line int `xml:"line,attr,omitempty"` Failure *Failure `xml:"failure,omitempty"` + Error *Error `xml:"error,omitempty"` Skipped *Skipped `xml:"skipped,omitempty"` Properties *Properties `xml:"properties,omitempty"` SystemOut *SystemOut `xml:"system-out,omitempty"` SystemErr *SystemErr `xml:"system-err,omitempty"` } -type Error struct { - XMLName xml.Name `xml:"error,omitempty"` +type Failure struct { + XMLName xml.Name `xml:"failure,omitempty"` + Type string `xml:"type,attr,omitempty"` + Message string `xml:"message,attr,omitempty"` Value string `xml:",chardata"` } -type Failure struct { - XMLName xml.Name `xml:"failure,omitempty"` +type Error struct { + XMLName xml.Name `xml:"error,omitempty"` + Type string `xml:"type,attr,omitempty"` + Message string `xml:"message,attr,omitempty"` Value string `xml:",chardata"` } type Skipped struct { XMLName xml.Name `xml:"skipped,omitempty"` + Message string `xml:"message,attr,omitempty"` Value string `xml:",chardata"` }