diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectApplication.kt new file mode 100644 index 0000000000..9d9e47f29a --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectApplication.kt @@ -0,0 +1,15 @@ +package com.foo.rest.examples.spring.openapi.v3.flakinessdetect + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class FlakinessDetectApplication { + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(FlakinessDetectApplication::class.java, *args) + } + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt new file mode 100644 index 0000000000..ed1474a2e0 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt @@ -0,0 +1,43 @@ +package com.foo.rest.examples.spring.openapi.v3.flakinessdetect + +import org.h2.util.MathUtils.randomInt +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import kotlin.math.max +import kotlin.math.min + +@RestController +@RequestMapping(path = ["/api/flakinessdetect"]) +class FlakinessDetectRest { + + companion object{ + val formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS yyyy-MM-dd EEEE 'Week' ww") + } + + @GetMapping(path = ["/stringfirst/{n}"]) + open fun getFirst( @PathVariable("n") n: Int) : ResponseEntity { + + return ResponseEntity.ok(getPartialDate(n)) + } + + @GetMapping(path = ["/next/{n}"]) + open fun getNext( @PathVariable("n") n: Int) : ResponseEntity { + + return ResponseEntity.ok(FlakinessDetectData(getPartialDate(n), randomInt(n))) + } + + + private fun getPartialDate(n: Int) : String { + val now = LocalDateTime.now().format(formatter) + val size = max(12, min(now.toString().length, n)) + + return now.substring(0, size) + } +} + +data class FlakinessDetectData(val first : String, val next : Int) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectController.kt new file mode 100644 index 0000000000..68f5854db6 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectController.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.spring.openapi.v3.flakinessdetect + +import com.foo.rest.examples.spring.openapi.v3.SpringController + +class FlakinessDetectController : SpringController(FlakinessDetectApplication::class.java) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt new file mode 100644 index 0000000000..8a53ce01c0 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt @@ -0,0 +1,45 @@ +package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect + +import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test +import java.nio.file.Files +import java.nio.file.Paths + +class FlakinessDetectEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FlakinessDetectController()) + } + } + + + @Test + fun testRunEM() { + runTestHandlingFlakyAndCompilation( + "FlakinessDetectEM", + "org.foo.FlakinessDetectEM", + 5 + ) { args: MutableList -> + + val executedMainActionToFile = "target/em-tests/FlakinessDetectEM/org/foo/FlakinessDetectEM.kt" + + args.add("--minimize") + args.add("true") + args.add("--detectFlakiness") + args.add("true") + + + val solution = initAndRun(args) + + val size = Files.readAllLines(Paths.get(executedMainActionToFile)).count { !it.contains("Flaky") && it.isNotBlank() } + assertTrue(size >= 3) + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index eeee3f5d54..3369c5d7b4 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2474,6 +2474,10 @@ class EMConfig { RANDOM } + @Experimental + @Cfg("Specify whether to detect flaky assertions during post handling of fuzzing.") + var detectFlakiness = false + @Experimental @Cfg("Specify a method to select the first external service spoof IP address.") var externalServiceIPSelectionStrategy = ExternalServiceIPSelectionStrategy.NONE diff --git a/core/src/main/kotlin/org/evomaster/core/output/Lines.kt b/core/src/main/kotlin/org/evomaster/core/output/Lines.kt index 81b387ddf0..e8611c37aa 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/Lines.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/Lines.kt @@ -75,6 +75,14 @@ class Lines(val format: OutputFormat) { buffer[buffer.lastIndex] = buffer.last().replace(regex, replacement) } + fun replaceFirstInCurrent(regex: Regex, replacement: String){ + if(buffer.isEmpty()){ + return + } + + buffer[buffer.lastIndex] = buffer.last().replaceFirst(regex, replacement) + } + /** * Is the current line just a comment // without any statement? */ diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt index b4764a062c..800a45620a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/ApiTestCaseWriter.kt @@ -77,27 +77,35 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { /** * handle assertion with text plain */ - fun handleTextPlainTextAssertion(bodyString: String?, lines: Lines, bodyVarName: String?) { + fun handleTextPlainTextAssertion(bodyString: String?, flakyBodyString: String?, lines: Lines, bodyVarName: String?) { - if (bodyString.isNullOrBlank()) { - lines.add(emptyBodyCheck(bodyVarName)) + val assertion = if (bodyString.isNullOrBlank()) { + emptyBodyCheck(bodyVarName) } else { //TODO in the call above BODY was used... what's difference from TEXT? - lines.add(bodyIsString(bodyString, GeneUtils.EscapeMode.TEXT, bodyVarName)) + bodyIsString(bodyString, GeneUtils.EscapeMode.TEXT, bodyVarName) + } + if (flakyBodyString == null || flakyBodyString == bodyString) { + lines.add(assertion) + }else{ + lines.addSingleCommentLine(flakyInfo("response in plain text", bodyString?:"null", flakyBodyString)) + lines.addSingleCommentLine(assertion) } } /** * handle assertion with json body string */ - fun handleJsonStringAssertion(bodyString: String?, lines: Lines, bodyVarName: String?, isTooLargeBody: Boolean) { + fun handleJsonStringAssertion(bodyString: String?, flakyBodyString : String?, lines: Lines, bodyVarName: String?, isTooLargeBody: Boolean) { when (bodyString?.trim()?.first()) { //TODO this should be handled recursively, and not ad-hoc here... '[' -> { try{ // This would be run if the JSON contains an array of objects. val list = Gson().fromJson(bodyString, List::class.java) - handleAssertionsOnList(list, lines, "", bodyVarName) + val flakyList = flakyBodyString?.let { Gson().fromJson(it, List::class.java) } + + handleAssertionsOnList(list, flakyList, lines, "", bodyVarName) } catch (e: JsonSyntaxException) { lines.addSingleCommentLine("Failed to parse JSON response") } @@ -106,13 +114,21 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { // JSON contains an object try { val resContents = Gson().fromJson(bodyString, Map::class.java) - handleAssertionsOnObject(resContents as Map, lines, "", bodyVarName) + val flakyMap = flakyBodyString?.let { Gson().fromJson(it, Map::class.java) as Map } + handleAssertionsOnObject(resContents as Map, flakyMap, lines, "", bodyVarName) } catch (e: JsonSyntaxException) { lines.addSingleCommentLine("Failed to parse JSON response") } } '"' -> { - lines.add(bodyIsString(bodyString, GeneUtils.EscapeMode.BODY, bodyVarName)) + val isString = bodyIsString(bodyString, GeneUtils.EscapeMode.BODY, bodyVarName) + if (flakyBodyString == null || flakyBodyString == bodyString) { + lines.add(isString) + }else{ + lines.addSingleCommentLine(flakyInfo("Body", bodyString, flakyBodyString)) + lines.addSingleCommentLine(isString) + } + } else -> { /* @@ -127,14 +143,14 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { bodyString.isNullOrBlank() -> lines.add(emptyBodyCheck(bodyVarName)) - else -> handlePrimitive(lines, bodyString, "", bodyVarName) + else -> handlePrimitive(lines, bodyString, flakyBodyString,"", bodyVarName) } } } } - private fun handlePrimitive(lines: Lines, bodyString: String, fieldPath: String, responseVariableName: String?) { + private fun handlePrimitive(lines: Lines, bodyString: String, flakyBodyString: String?, fieldPath: String, responseVariableName: String?) { /* If we arrive here, it means we have free text. @@ -150,24 +166,36 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { when { format.isJavaOrKotlin() -> { + /* Unfortunately, a limitation of RestAssured is that for JSON it only handles object and array. The rest is either ignored or leads to crash */ - lines.add(bodyIsString(s, GeneUtils.EscapeMode.BODY, responseVariableName)) + val value = bodyIsString(s,GeneUtils.EscapeMode.BODY, responseVariableName) + + val fs = flakyBodyString?.trim() + if (fs == null || fs == s) + lines.add(value) + else{ + lines.addSingleCommentLine(flakyInfo("Body", s, fs)) + lines.addSingleCommentLine(value) + } + } format.isJavaScript() || format.isCsharp() || format.isPython() -> { try { val number = s.toDouble() - handleAssertionsOnField(number, lines, fieldPath, responseVariableName) + // TODO only support flaky for JVM + handleAssertionsOnField(number, null, lines, fieldPath, responseVariableName) return } catch (e: NumberFormatException) { } if (s.equals("true", true) || s.equals("false", true)) { val tf = bodyString.toBoolean() - handleAssertionsOnField(tf, lines, fieldPath, responseVariableName) + // TODO flaky + handleAssertionsOnField(tf, null, lines, fieldPath, responseVariableName) return } @@ -175,13 +203,18 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { Note: for JS, this will not work, as call would crash due to invalid JSON payload (Java and Python don't seem to have such issue) */ + // TODO flaky lines.add(bodyIsString(s, GeneUtils.EscapeMode.BODY, responseVariableName)) } else -> throw IllegalStateException("Format not supported yet: $format") } } - protected fun handleAssertionsOnObject(resContents: Map, lines: Lines, fieldPath: String, responseVariableName: String?) { + protected fun handleAssertionsOnObject(resContents: Map, flakyMap: Map?, lines: Lines, fieldPath: String, responseVariableName: String?) { + if (flakyMap != null && flakyMap.size != resContents.size) { + lines.addSingleCommentLine(flakyInfo("mismatched size of fields for Object $fieldPath", resContents.size.toString(), flakyMap.size.toString())) + } + if (resContents.isEmpty()) { val k = when { @@ -206,7 +239,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { else -> throw IllegalStateException("Format not supported yet: $format") } - lines.add(instruction) + if (flakyMap.isNullOrEmpty()) + lines.add(instruction) + else{ + lines.addSingleCommentLine(instruction) + } } resContents.entries @@ -244,7 +281,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { "${fieldPath}${fieldName}" } - handleAssertionsOnField(it.value, lines, extendedPath, responseVariableName) + if (flakyMap == null || flakyMap.containsKey(it.key)) { + handleAssertionsOnField(it.value, flakyMap?.get(it.key), lines, extendedPath, responseVariableName) + }else{ + lines.addSingleCommentLine(flakyInfo("mismatched field name", fieldName, "NONE")) + } } } /* @@ -255,7 +296,7 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { return text.replace("\$", "\\\$") } - private fun handleAssertionsOnField(value: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { + private fun handleAssertionsOnField(value: Any?, flakyValue: Any?, lines: Lines, fieldPath: String, responseVariableName: String?) { if (value == null) { val instruction = when { @@ -271,11 +312,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { when (value) { is Map<*, *> -> { - handleAssertionsOnObject(value as Map, lines, fieldPath, responseVariableName) + handleAssertionsOnObject(value as Map, flakyValue as? Map,lines, fieldPath, responseVariableName) return } is List<*> -> { - handleAssertionsOnList(value, lines, fieldPath, responseVariableName) + handleAssertionsOnList(value, flakyValue as? List<*>, lines, fieldPath, responseVariableName) return } } @@ -290,7 +331,12 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { else -> throw IllegalStateException("Unsupported type: ${value::class}") } if (isSuitableToPrint(left)) { - lines.add(".body(\"$fieldPath\", $left)") + if (flakyValue == null || flakyValue == value) + lines.add(".body(\"$fieldPath\", $left)") + else{ + lines.addSingleCommentLine(flakyInfo("value of field \"$fieldPath\"", value.toString(), flakyValue.toString())) + lines.addSingleCommentLine(".body(\"$fieldPath\", $left)") + } } return } @@ -322,9 +368,16 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { } - protected fun handleAssertionsOnList(list: List<*>, lines: Lines, fieldPath: String, responseVariableName: String?) { + protected fun handleAssertionsOnList(list: List<*>, flakyList: List<*>?, lines: Lines, fieldPath: String, responseVariableName: String?) { + + val checkSize = collectionSizeCheck(responseVariableName, fieldPath, list.size) + if (flakyList == null || flakyList.size == list.size) { + lines.add(checkSize) + }else{ + lines.addSingleCommentLine(flakyInfo("size of $fieldPath", list.size.toString(), flakyList.size.toString())) + lines.addSingleCommentLine(checkSize) + } - lines.add(collectionSizeCheck(responseVariableName, fieldPath, list.size)) //assertions on contents if (list.isEmpty()) { @@ -335,11 +388,26 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { TODO could do the same for numbers */ if (format.isJavaOrKotlin() && list.all { it is String } && list.isNotEmpty()) { - lines.add(".body(\"$fieldPath\", hasItems(${ - (list as List).joinToString { + val items = (list as List).joinToString { + "\"${GeneUtils.applyEscapes(it, mode = GeneUtils.EscapeMode.ASSERTION, format = format)}\"" + } + + if (flakyList != null && (!flakyList.containsAll(list) || flakyList.size != list.size)) { + + val flakyItems = (flakyList as List).joinToString { "\"${GeneUtils.applyEscapes(it, mode = GeneUtils.EscapeMode.ASSERTION, format = format)}\"" } - }))") + + lines.addSingleCommentLine(flakyInfo("Body", items, flakyItems)) + lines.addSingleCommentLine(".body(\"$fieldPath\", hasItems(${ + items + }))") + + }else{ + lines.add(".body(\"$fieldPath\", hasItems(${ + items + }))") + } return } @@ -356,7 +424,11 @@ abstract class ApiTestCaseWriter : TestCaseWriter() { if (i == limit) { break } - handleAssertionsOnField(list[i], lines, "$fieldPath[$i]", responseVariableName) + if (flakyList != null && flakyList.size < i) + break + val flakyItem = if (flakyList != null) flakyList[i] else null + + handleAssertionsOnField(list[i], flakyItem, lines, "$fieldPath[$i]", responseVariableName) } if (skipped > 0) { lines.addSingleCommentLine("Skipping assertions on the remaining $skipped elements. This limit of $limit elements can be increased in the configurations") diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt index c47a22eaed..0f8b3d0407 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt @@ -697,7 +697,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { when { format.isJavaOrKotlin() -> { lines.add(".then()") - lines.add(".statusCode($code)") + if (res.getFlakyStatusCode() == null) { + lines.add(".statusCode($code)") + } else { + lines.addSingleCommentLine(flakyInfo("Status Code", code.toString(), res.getFlakyStatusCode().toString())) + lines.addSingleCommentLine(".statusCode($code)") + } } else -> throw IllegalStateException("No assertion in calls for format: $format") @@ -768,7 +773,12 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { format.isPython() -> "assert \"$bodyTypeSimplified\" in $responseVariableName.headers[\"content-type\"]" else -> throw IllegalStateException("Unsupported format $format") } - lines.add(instruction) + + // handle flaky body type + if (res.getFlakyBodyType() == null) + lines.add(instruction) + else + lines.addSingleCommentLine(instruction) } val type = res.getBodyType() @@ -789,7 +799,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append("JsonConvert.DeserializeObject(await $responseVariableName.Content.ReadAsStringAsync());") } - handleJsonStringAssertion(bodyString, lines, bodyVarName, res.getTooLargeBody()) + handleJsonStringAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName, res.getTooLargeBody()) } else if (type.isCompatible(MediaType.TEXT_PLAIN_TYPE)) { @@ -797,7 +807,7 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.append("await $responseVariableName.Content.ReadAsStringAsync();") } - handleTextPlainTextAssertion(bodyString, lines, bodyVarName) + handleTextPlainTextAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName) } else { if (format.isCsharp()) { lines.append("await $responseVariableName.Content.ReadAsStringAsync();") @@ -849,7 +859,9 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() { lines.replaceInCurrent(Regex("\\s*//"), "; //") } - } else { + } else if (config.detectFlakiness && lines.isCurrentACommentLine()){ + lines.replaceInCurrent(Regex("(?<=\\s)//"), "; //") + }else { lines.appendSemicolon() } } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt index a1712427b1..982baa325c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestCaseWriter.kt @@ -398,5 +398,6 @@ abstract class TestCaseWriter { // do nothing } + fun flakyInfo(category : String?, value : String, flaky : String) = "Flaky${if (category == null) "" else " $category"}: $value vs. $flaky" } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt index 7de51793b0..7e27274fff 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLBlackBoxModule.kt @@ -10,6 +10,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler @@ -53,6 +54,12 @@ class GraphQLBlackBoxModule( bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + if(usingRemoteController) { bind(RemoteController::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt index e9c773799d..c6cf794849 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/service/GraphQLModule.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.mutator.Mutator @@ -56,6 +57,11 @@ class GraphQLModule : EnterpriseModule() { bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() bind(RemoteController::class.java) .to(RemoteControllerImplementation::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt index 581704b72b..f8534147d6 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/HttpWsCallResult.kt @@ -29,6 +29,12 @@ abstract class HttpWsCallResult : EnterpriseActionResult { const val VULNERABLE_SSRF = "VULNERABLE_SSRF" const val VULNERABLE_SQLI = "VULNERABLE_SQLI" + + + const val FLAKY_STATUS_CODE = "FLAKY_STATUS_CODE" + const val FLAKY_BODY = "FLAKY_BODY" + const val FLAKY_BODY_TYPE = "FLAKY_BODY_TYPE" + const val FLAKY_ERROR_MESSAGE = "FLAKY_ERROR_MESSAGE" } /** @@ -134,4 +140,40 @@ abstract class HttpWsCallResult : EnterpriseActionResult { fun setResponseTimeMs(responseTime: Long) = addResultValue(RESPONSE_TIME_MS, responseTime.toString()) fun getResponseTimeMs(): Long? = getResultValue(RESPONSE_TIME_MS)?.toLong() + + + fun setFlakyErrorMessage(msg: String) = addResultValue(FLAKY_ERROR_MESSAGE, msg) + fun getFlakyErrorMessage() : String? = getResultValue(FLAKY_ERROR_MESSAGE) + + fun setFlakyStatusCode(code: Int) = addResultValue(FLAKY_STATUS_CODE, code.toString()) + fun getFlakyStatusCode() : Int? = getResultValue(FLAKY_STATUS_CODE)?.toInt() + + fun setFlakyBody(body: String) = addResultValue(FLAKY_BODY, body) + fun getFlakyBody() : String? = getResultValue(FLAKY_BODY) + + fun setFlakyBodyType(type: MediaType) = addResultValue(FLAKY_BODY_TYPE, type.toString()) + fun getFlakyBodyType() : MediaType? = getResultValue(FLAKY_BODY_TYPE)?.let { MediaType.valueOf(it) } + + + fun setFlakiness(previous: HttpWsCallResult){ + val pStatusCode = previous.getStatusCode() + if (pStatusCode != null && pStatusCode != getStatusCode()) { + setFlakyStatusCode(pStatusCode) + } + + val pBody = previous.getBody() + if (pBody != null && pBody != getBody()) { + setFlakyBody(pBody) + } + + val pBodyType = previous.getBodyType() + if (pBodyType != null && pBodyType != getBodyType()) { + setFlakyBodyType(pBodyType) + } + + val pMessage = previous.getErrorMessage() + if (pMessage != null && pMessage != getErrorMessage()) { + setFlakyErrorMessage(pMessage) + } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt index 1c4b47be3a..f09079e759 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/RestBaseModule.kt @@ -12,6 +12,7 @@ import org.evomaster.core.problem.rest.service.HttpSemanticsService import org.evomaster.core.problem.rest.service.RestIndividualBuilder import org.evomaster.core.problem.rest.service.SecurityRest import org.evomaster.core.search.service.Archive +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.seeding.service.rest.PirToRest @@ -42,6 +43,12 @@ open class RestBaseModule : EnterpriseModule() { bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + bind(object : TypeLiteral>() {}) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt index c40d5611eb..29837dbc17 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rpc/service/RPCModule.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.mutator.Mutator @@ -50,6 +51,12 @@ class RPCModule : EnterpriseModule(){ .to(RPCFitness::class.java) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + bind(object : TypeLiteral>() {}) .to(RPCFitness::class.java) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt index 42f5711906..f3fb1c9525 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/webfrontend/service/WebModule.kt @@ -11,6 +11,7 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.FitnessFunction +import org.evomaster.core.search.service.FlakinessDetector import org.evomaster.core.search.service.Minimizer import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.mutator.Mutator @@ -47,6 +48,12 @@ class WebModule: EnterpriseModule() { bind(object : TypeLiteral>(){}) .asEagerSingleton() + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + + bind(object : TypeLiteral>(){}) + .asEagerSingleton() + bind(object : TypeLiteral>() {}) .to(WebFitness::class.java) .asEagerSingleton() diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt new file mode 100644 index 0000000000..f191341a43 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/service/FlakinessDetector.kt @@ -0,0 +1,77 @@ +package org.evomaster.core.search.service + +import com.google.inject.Inject +import org.evomaster.core.EMConfig +import org.evomaster.core.logging.LoggingUtil +import org.evomaster.core.problem.httpws.HttpWsAction +import org.evomaster.core.problem.httpws.HttpWsCallResult +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Individual +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * it is used to detect flaky tests by checking responses or return value + * currently, such a detection is performed during post-handling of fuzzing + */ +class FlakinessDetector { + + companion object{ + private val log : Logger = LoggerFactory.getLogger(FlakinessDetector::class.java) + } + + @Inject + private lateinit var config: EMConfig + + @Inject + private lateinit var archive: Archive + + @Inject + private lateinit var fitness : FitnessFunction + + /** + * re-execute individuals in archive for identifying flakiness + */ + fun reexecuteToDetectFlakiness() { + + val currentIndividuals = archive.extractSolution().individuals + + LoggingUtil.getInfoLogger().info("Reexecuting all individual ${currentIndividuals.size} for identifying flakiness.") + + currentIndividuals.mapNotNull { + + val ei = fitness.computeWholeAchievedCoverageForPostProcessing(it.individual) + if(ei == null){ + log.warn("Failed to re-evaluate individual during flakiness analysis.") + }else + checkConsistency(ei, it) + + } + + } + + /** + * compare [inArchive] with [other] to check if the action results are same, the inconsistent info will be saved in [inArchive] evaluated individual + * @param inArchive the evaluated individual which saves info of flakiness + */ + fun checkConsistency(other: EvaluatedIndividual, inArchive: EvaluatedIndividual){ + val previousActions = other.evaluatedMainActions() + val currentActions = inArchive.evaluatedMainActions() + + if(previousActions.size != currentActions.size){ + log.warn("Mismatch between number of actions in re-executed individual." + + " Previous =${previousActions.size}, Current =${currentActions.size}") + return + } + + currentActions.forEachIndexed { index, it -> + val action = it.action + if(action is HttpWsAction){ + if (it.result is HttpWsCallResult){ + it.result.setFlakiness(previousActions[index].result as HttpWsCallResult) + } + + } + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt index 06670b62fe..43d19c5818 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt @@ -38,6 +38,9 @@ class Minimizer { @Inject private lateinit var idMapper: IdMapper + @Inject + private lateinit var flakinessDetector: FlakinessDetector + private var startTimer : Long = -1 @@ -235,6 +238,9 @@ class Minimizer { //don't check mismatch if possible issues, as then mismatches would be expected checkResultMismatches(it, ei) } + if (config.detectFlakiness && ei != null){ + flakinessDetector.checkConsistency(it, ei) + } ei } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt b/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt index 695dbd693a..fea0abe8d3 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt @@ -38,6 +38,9 @@ abstract class SearchAlgorithm where T : Individual { @Inject private lateinit var minimizer: Minimizer + @Inject + private lateinit var flakinessDetector: FlakinessDetector + @Inject private lateinit var ssu: SearchStatusUpdater @@ -111,6 +114,8 @@ abstract class SearchAlgorithm where T : Individual { minimizer.simplifyActions() val seconds = minimizer.passedTimeInSecond() LoggingUtil.getInfoLogger().info("Minimization phase took $seconds seconds") + } else if (config.detectFlakiness){ + flakinessDetector.reexecuteToDetectFlakiness() } if(config.addPreDefinedTests) { diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt index 6be68db0fc..ff257bb4fe 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestCaseWriterTest.kt @@ -1392,4 +1392,183 @@ public void test() throws Exception { assertEquals(expectedLines, lines.toString()) } + + @Test + fun testTestFlakyBodyForRestCallResponses(){ + val fooAction = RestCallAction("1", HttpVerb.GET, RestPath("/foo"), mutableListOf()) + + val (format, baseUrlOfSut, ei) = buildResourceEvaluatedIndividual( + dbInitialization = mutableListOf(), + groups = mutableListOf( + (mutableListOf() to mutableListOf(fooAction)) + ), + format = OutputFormat.JAVA_JUNIT_5 + ) + + val fooResult = ei.seeResult(fooAction.getLocalId()) as RestCallResult + fooResult.setTimedout(false) + fooResult.setStatusCode(200) + fooResult.setBody(""" + { + "p0":[1,2], + "p1":{}, + "p2":{ + "id":"foo", + "properties":[ + {}, + { + "name":"mapProperty1", + "type":"string", + "value":"one" + }, + { + "name":"mapProperty2", + "type":"string", + "value":"two" + }], + "empty":{} + } + } + """.trimIndent()) + fooResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + + val barResult = RestCallResult(fooAction.getLocalId()) + barResult.setTimedout(false) + barResult.setStatusCode(500) + barResult.setBody(""" + { + "p0":[1,2,3], + "p1":{}, + "p2":{ + "id":"foo", + "properties":[ + {}, + { + "name":"flaky1", + "type":"string", + "value":"flaky2" + }, + { + "name":"mapProperty2", + "type":"string", + "value":"two" + }, + { + "name":"flaky3", + "type":"string", + "value":"two" + }], + "empty":{ + "flakyField4": 42 + } + } + } + """.trimIndent()) + barResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + fooResult.setFlakiness(barResult) + + val config = getConfig(format) + config.detectFlakiness = true + + val test = TestCase(test = ei, name = "test") + + val writer = RestTestCaseWriter(config, PartialOracles()) + val lines = writer.convertToCompilableTestCode( test, baseUrlOfSut) + + val expectedLines = """ +@Test +public void test() throws Exception { + + given().accept("*/*") + .get(baseUrlOfSut + "/foo") + .then() + // Flaky Status Code: 200 vs. 500 + // .statusCode(200) + .assertThat() + .contentType("application/json") + // Flaky size of 'p0': 2 vs. 3 + // .body("'p0'.size()", equalTo(2)) + .body("'p0'[0]", numberMatches(1.0)) + .body("'p0'[1]", numberMatches(2.0)) + .body("'p1'.isEmpty()", is(true)) + // Flaky size of 'p2'.'properties': 3 vs. 4 + // .body("'p2'.'properties'.size()", equalTo(3)) + .body("'p2'.'properties'[0].isEmpty()", is(true)) + // Flaky value of field "'p2'.'properties'[1].'name'": mapProperty1 vs. flaky1 + // .body("'p2'.'properties'[1].'name'", containsString("mapProperty1")) + .body("'p2'.'properties'[1].'type'", containsString("string")) + // Flaky value of field "'p2'.'properties'[1].'value'": one vs. flaky2 + // .body("'p2'.'properties'[1].'value'", containsString("one")) + .body("'p2'.'properties'[2].'name'", containsString("mapProperty2")) + .body("'p2'.'properties'[2].'type'", containsString("string")) + .body("'p2'.'properties'[2].'value'", containsString("two")) + // Flaky mismatched size of fields for Object 'p2'.'empty': 0 vs. 1 + ; // .body("'p2'.'empty'.isEmpty()", is(true)) +} + +""".trimIndent() + assertEquals(expectedLines, lines.toString()) + } + + @Test + fun testTestFlakyListForRestCallResponses(){ + val fooAction = RestCallAction("1", HttpVerb.GET, RestPath("/foo"), mutableListOf()) + + val (format, baseUrlOfSut, ei) = buildResourceEvaluatedIndividual( + dbInitialization = mutableListOf(), + groups = mutableListOf( + (mutableListOf() to mutableListOf(fooAction)) + ), + format = OutputFormat.JAVA_JUNIT_5 + ) + + val fooResult = ei.seeResult(fooAction.getLocalId()) as RestCallResult + fooResult.setTimedout(false) + fooResult.setStatusCode(200) + fooResult.setBody(""" + ["foo", "bar"] + """.trimIndent()) + fooResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + + val barResult = RestCallResult(fooAction.getLocalId()) + barResult.setTimedout(false) + barResult.setStatusCode(500) + barResult.setBody(""" + ["foo", "abc", "bar"] + """.trimIndent()) + barResult.setBodyType(MediaType.APPLICATION_JSON_TYPE) + + fooResult.setFlakiness(barResult) + + val config = getConfig(format) + config.detectFlakiness = true + + val test = TestCase(test = ei, name = "test") + + val writer = RestTestCaseWriter(config, PartialOracles()) + val lines = writer.convertToCompilableTestCode( test, baseUrlOfSut) + + val expectedLines = """ +@Test +public void test() throws Exception { + + given().accept("*/*") + .get(baseUrlOfSut + "/foo") + .then() + // Flaky Status Code: 200 vs. 500 + // .statusCode(200) + .assertThat() + .contentType("application/json") + // Flaky size of : 2 vs. 3 + // .body("size()", equalTo(2)) + // Flaky Body: "foo", "bar" vs. "foo", "abc", "bar" + ; // .body("", hasItems("foo", "bar")) +} + +""".trimIndent() + assertEquals(expectedLines, lines.toString()) + } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt index fb182e77d6..f017f027a8 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/RestCallResultTest.kt @@ -2,12 +2,44 @@ package org.evomaster.core.problem.rest import org.evomaster.core.problem.rest.data.RestCallResult import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNotNull import org.junit.jupiter.api.Assertions.assertNull import org.junit.jupiter.api.Test +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import java.util.stream.Stream import javax.ws.rs.core.MediaType internal class RestCallResultTest { + companion object{ + @JvmStatic + fun getFlakyBodyDataProvider(): Stream { + + return Stream.of( + Arguments.of("42", "-1"), + Arguments.of("{\"id\":\"42\"}", "{\"id\":\"735\"}"), + Arguments.of(""" + { + "solid": "a", + "timid": "b", + "fooId": "c", + "void": "d" + } + """, """ + { + "solid": "a", + "timid": "b", + "fooId": "d", + "void": "d" + } + """), + ) + + } + } + @Test fun givenAStringIdWhenGetResourceIdThenItIsReturnedAsString() { val rc = RestCallResult("", false) @@ -97,4 +129,53 @@ internal class RestCallResultTest { assertEquals("42", res.value) assertEquals("/", res.pointer) } + + @ParameterizedTest + @MethodSource("getFlakyBodyDataProvider") + fun testSetFlakinessInBody(same: String, diff: String){ + val body = createCallResult(same) + val same = createCallResult(same) + val diff = createCallResult(diff) + + body.setFlakiness(same) + assertNotNull(body.getBodyType()) + + assertNull(body.getFlakyBody()) + assertNull(body.getFlakyBodyType()) + + body.setFlakiness(diff) + assertEquals(diff.getBody(), body.getFlakyBody()) + assertNull(body.getFlakyBodyType()) + + } + + + @Test + fun testSetFlakiness(){ + val code = 201 + val diffcode = 500 + val msg = "hello" + val diffmsg = "hello!" + + val body = RestCallResult("1") + val same = RestCallResult("2") + val diff = RestCallResult("3") + + body.setStatusCode(code) + same.setStatusCode(code) + diff.setStatusCode(diffcode) + + body.setErrorMessage(msg) + same.setErrorMessage(msg) + diff.setErrorMessage(diffmsg) + + body.setFlakiness(same) + assertNull(body.getFlakyStatusCode()) + assertNull(body.getFlakyErrorMessage()) + + body.setFlakiness(diff) + assertEquals(diffcode, body.getFlakyStatusCode()) + assertEquals(diffmsg, body.getFlakyErrorMessage()) + + } } diff --git a/docs/options.md b/docs/options.md index a3ead34bf7..e916a81072 100644 --- a/docs/options.md +++ b/docs/options.md @@ -258,6 +258,7 @@ There are 3 types of options: |`callbackURLHostname`| __String__. HTTP callback verifier hostname. Default is set to 'localhost'. If the SUT is running inside a container (i.e., Docker), 'localhost' will refer to the container. This can be used to change the hostname. *Default value*: `localhost`.| |`cgaNeighborhoodModel`| __Enum__. Cellular GA: neighborhood model (RING, L5, C9, C13). *Valid values*: `RING, L5, C9, C13`. *Default value*: `RING`.| |`classificationRepairThreshold`| __Double__. If using THRESHOLD for AI Classification Repair, specify its value. All classifications with probability equal or above such threshold value will be accepted. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.8`.| +|`detectFlakiness`| __Boolean__. Specify whether to detect flaky assertions during post handling of fuzzing. *Default value*: `false`.| |`discoveredInfoRewardedInFitness`| __Boolean__. If there is new discovered information from a test execution, reward it in the fitness function. *Default value*: `false`.| |`dockerLocalhost`| __Boolean__. Replace references to 'localhost' to point to the actual host machine. Only needed when running EvoMaster inside Docker. *Default value*: `false`.| |`dpcTargetTestSize`| __Int__. Specify a max size of a test to be targeted when either DPC_INCREASING or DPC_DECREASING is enabled. *Default value*: `1`.|