diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22fce91f23..cc260bfd8f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,7 +42,7 @@ env: build-jdk: 17 retention-days: 5 debug: false # put to true if need to debug a specific test - debugTestName: "org.evomaster.e2etests.spring.examples.sort.SortEMTest" # replace with test to debug + debugTestName: "org.evomaster.core.problem.rest.SamplerVerifierTest" # replace with test to debug # This build is quite expensive (some hours), so we run it whole only on some JVM versions and OSs. # For the moment, we need to support JVM 8 and all following LTS versions (e.g, 11 and 17). diff --git a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/AdditionalInfo.java b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/AdditionalInfo.java index 27663da299..b2bcbace86 100644 --- a/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/AdditionalInfo.java +++ b/client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/AdditionalInfo.java @@ -170,7 +170,10 @@ public void setRawAccessOfHttpBodyPayload(boolean rawAccessOfHttpBodyPayload) { public void addSpecialization(String taintInputName, StringSpecializationInfo info){ if(!ExecutionTracer.getTaintType(taintInputName).isTainted()){ - throw new IllegalArgumentException("No valid input name: " + taintInputName); + //this can happen in E2E where libraries used by "core" are instrumented (eg Kotlin) + SimpleLogger.error("No valid taint input name for specialization: " + taintInputName); + //throw new IllegalArgumentException(); + return; } Objects.requireNonNull(info); diff --git a/core-parent/pom.xml b/core-parent/pom.xml index 0b038eb76f..4e5144db38 100644 --- a/core-parent/pom.xml +++ b/core-parent/pom.xml @@ -15,9 +15,9 @@ - 1.0.61 + 1.0.64 - 2.1.8 + 2.1.14 2.2.7 diff --git a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/SamplerVerifierTest.kt b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/SamplerVerifierTest.kt index 430fe1c940..10e6d0233c 100644 --- a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/SamplerVerifierTest.kt +++ b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/SamplerVerifierTest.kt @@ -13,10 +13,11 @@ import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamCh import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsDto import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsResult import org.evomaster.core.BaseModule +import org.evomaster.core.Main import org.evomaster.core.problem.rest.schema.RestSchema import org.evomaster.core.problem.rest.service.module.BlackBoxRestModule import org.evomaster.core.problem.rest.service.module.ResourceRestModule -import org.evomaster.core.problem.rest.service.sampler.ResourceSampler +import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler import org.evomaster.core.remote.SutProblemException import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Individual @@ -28,10 +29,17 @@ import java.lang.reflect.InvocationTargetException import java.time.Duration import kotlin.io.path.Path import kotlin.sequences.filter +import kotlin.text.contains class SamplerVerifierTest { + companion object { + init { + Main.applyGlobalJVMSettings() + } + } + @Test fun testBase() { @@ -47,7 +55,7 @@ class SamplerVerifierTest { val injector = getInjector(sutInfo, controllerInfo, false) - val sampler = injector.getInstance(ResourceSampler::class.java) + val sampler = injector.getInstance(AbstractRestSampler::class.java) sampler.sample() //should not crash } @@ -64,11 +72,8 @@ class SamplerVerifierTest { ) assertTrue(tests.isNotEmpty()) return tests - //FIXME once handling performance issues - .filter { !it.displayName.contains("adyen") } } - @Disabled("Performance issue") @TestFactory fun testSamplingFromAllSchemasUnderCoreResourcesBlackBox(): Collection{ val tests = sampleFromSchemasAndCheckInvariants( @@ -81,8 +86,6 @@ class SamplerVerifierTest { } - //FIXME need to put back, and investigate performance bug - @Disabled("Major issues with timeouts. Even before, took more than 1 hour. Need refactoring. Maven was not showing the failures (likely bug in Surefire)") @TestFactory fun testSamplingFromAPIsGuru(): Collection{ val tests = sampleFromSchemasAndCheckInvariants( @@ -100,11 +103,22 @@ class SamplerVerifierTest { blackBox: Boolean ): Collection { + /* + In theory, we should not have such a high timeout for this kind of tests. + On local machine, all those analyses take at most 20s per test. + Maybe performance can be improved, but, considering some files are MBs, it is not + so unexpected. And it happens only for very large files. + Problem though is that CI is much SLOWER, and tests do timeout. + So that is why we have such high timeout, it is for CI. + Still, if it starts to fail there, then we really need to look into performance issues. + */ + val timeout = 60L + return scanForSchemas(relativePath, resourceFolder) .sorted().map { DynamicTest.dynamicTest(it) { System.gc() - assertTimeoutPreemptively(Duration.ofSeconds(30), it) { + assertTimeoutPreemptively(Duration.ofSeconds(timeout), it) { runInvariantCheck(it, 100, blackBox) } } @@ -121,16 +135,15 @@ class SamplerVerifierTest { .filter { it.isFile } .filter { !skipSchema(it.path) } .map { -// val s = it.path.replace("\\", "/") -// .replace(relativePath, resourceFolder) val s = Path(it.absolutePath).toAbsolutePath().normalize().toString() s }.toList() } private fun skipSchema(path: String) : Boolean { - return skipDueToOldChecks(path) || - skipDueToMissingPath(path) + return skipDueToOldChecks(path) + || skipDueToIssuesStillToInvestigate(path) + || skipDueToMissingPath(path) || skipDueToHashTag(path) || skipDueToQuestionMarkInPath(path) || skipDueToMissingReference(path) @@ -143,6 +156,29 @@ class SamplerVerifierTest { || skipDueToInvalidGenes(path) } + private fun skipDueToIssuesStillToInvestigate(path: String) : Boolean { + val toSkip = listOf( + "api.video/1/openapi.yaml", + "atlassian.com/jira/1001.0.0-SNAPSHOT/openapi.yaml", + "cloud-elements.com/ecwid/api-v2/swagger.yaml", + "github.com/api.github.com/1.1.4/openapi.yaml", + "googleapis.com/discovery/v1/openapi.yaml", + "here.com/positioning/2.1.1/openapi.yaml", + "maif.local/otoroshi/1.5.0-dev/openapi.yaml", + "mashape.com/geodb/1.0.0/swagger.yaml", + "microsoft.com/cognitiveservices-Training/1.2/openapi.yaml", + "microsoft.com/cognitiveservices-Training/2.0/openapi.yaml", + "microsoft.com/cognitiveservices-Training/2.1/openapi.yaml", + "microsoft.com/cognitiveservices-Training/2.2/openapi.yaml", + "microsoft.com/cognitiveservices-Training/3.0/openapi.yaml", + "microsoft.com/cognitiveservices-Training/3.1/openapi.yaml", + "neutrinoapi.net/3.5.0/openapi.yaml", + "openbankingproject.ch/1.3.8_2020-12-14 - Swiss edition 1.3.8.1-CH/openapi.yaml" + ) + + return toSkip.any { path.contains(it) } + } + private fun skipDueToOldChecks(path: String) : Boolean { return path.endsWith("features_service_null.json") //issue with parser || path.endsWith("trace_v2.json") // no actions are parsed @@ -244,7 +280,7 @@ class SamplerVerifierTest { || (contains("visualstudio.com") && contains("v1")) || (contains("youneedabudget.com") && contains("1.0.0")) || (contains("zenoti.com") && contains("1.0.0")) // No actions for schema - || (contains("zoom.us") && contains("2.0.0")) // The incoming YAML document exceeds the limit: 3145728 code points. + // || (contains("zoom.us") && contains("2.0.0")) // The incoming YAML document exceeds the limit: 3145728 code points. || (contains("zuora.com") && contains("2021-08-20")) //The incoming YAML document exceeds the limit: 3145728 code points. } } @@ -381,7 +417,7 @@ class SamplerVerifierTest { throw e } - val sampler = injector.getInstance(ResourceSampler::class.java) + val sampler = injector.getInstance(AbstractRestSampler::class.java) if(sampler.numberOfDistinctActions() == 0){ throw IllegalStateException("No actions for schema") diff --git a/core/src/main/kotlin/org/evomaster/core/BaseModule.kt b/core/src/main/kotlin/org/evomaster/core/BaseModule.kt index 3e503eedde..90041731c3 100644 --- a/core/src/main/kotlin/org/evomaster/core/BaseModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/BaseModule.kt @@ -3,10 +3,7 @@ package org.evomaster.core import com.google.inject.AbstractModule import com.google.inject.Provides import com.google.inject.Singleton -import org.evomaster.core.output.service.NoTestCaseWriter import org.evomaster.core.output.service.PartialOracles -import org.evomaster.core.output.service.TestCaseWriter -import org.evomaster.core.output.service.TestSuiteWriter import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector import org.evomaster.core.search.service.* import org.evomaster.core.search.service.monitor.SearchProcessMonitor diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index aead2fafa5..7b8c613582 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -54,6 +54,18 @@ import kotlin.system.exitProcess class Main { companion object { + /** + * Anything that impact whole JVM, needs to be done here, and called as first step in the main. + * Note: this is done as a function because tests will usually not call Main, and might rely + * on those settings for their behavior + */ + @JvmStatic + fun applyGlobalJVMSettings(){ + Locale.setDefault(Locale.ENGLISH) + //otherwise parser will crash on large OpenAPI schemas + System.setProperty("maxYamlCodePoints", "" + (50 * 1024 * 1024)) + } + /** * Main entry point of the EvoMaster application */ @@ -61,7 +73,7 @@ class Main { fun main(args: Array) { try { - Locale.setDefault(Locale.ENGLISH) + applyGlobalJVMSettings() printLogo() printVersion() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/OpenApiAccess.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/OpenApiAccess.kt index 32658e2910..8b93af0d7f 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/OpenApiAccess.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/OpenApiAccess.kt @@ -31,19 +31,29 @@ object OpenApiAccess { var parseResults: SwaggerParseResult? = null + val messages = mutableSetOf() + for (extension in OpenAPIV3Parser.getExtensions()) { parseResults = try { extension.readContents(schemaText, null, null) } catch (e: Exception) { throw SutProblemException("Failed to parse OpenApi schema: ${e.message}") } - if (parseResults != null && parseResults.openAPI != null) { - break + if (parseResults != null){ + if(parseResults.openAPI != null) { + //all good + break + } else { + //we might try another parser... but still shouldn't lose info of attempted parsers + messages.addAll(parseResults.messages.map { + extension.javaClass.simpleName + " -> " + it + "\n" + }) + } } } val schema = parseResults!!.openAPI - ?: throw SutProblemException("Failed to parse OpenApi schema: " + parseResults.messages.joinToString("\n")) + ?: throw SutProblemException("Failed to parse OpenApi schema:\n" + messages.joinToString("")) if(parseResults.messages.isNotEmpty()){ LoggingUtil.getInfoLogger().warn( diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/RestSchema.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/RestSchema.kt index d915727f1d..d261f12349 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/RestSchema.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/schema/RestSchema.kt @@ -3,7 +3,9 @@ package org.evomaster.core.problem.rest.schema import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.dataformat.yaml.YAMLFactory +import com.fasterxml.jackson.dataformat.yaml.YAMLFactoryBuilder import org.evomaster.core.remote.SutProblemException +import org.yaml.snakeyaml.LoaderOptions /** @@ -62,7 +64,16 @@ class RestSchema( //https://swagger.io/docs/specification/v3_0/using-ref/ - val mapper = ObjectMapper(YAMLFactory()) + // snakeyaml has default limits on size, which are very low + val yaml = YAMLFactoryBuilder(YAMLFactory()) + .loaderOptions(LoaderOptions().apply { + codePointLimit = 50 * 1024 * 1024 // 50MB + maxAliasesForCollections = 1000 + nestingDepthLimit = 100 + }) + .build() + + val mapper = ObjectMapper(yaml) val tree = mapper.readTree(schema.schemaRaw) val refs = findAllSRef(tree) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt index db60d8e048..83e872c618 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt @@ -7,7 +7,6 @@ import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler import org.evomaster.core.problem.rest.service.fitness.BlackBoxRestFitness import org.evomaster.core.problem.rest.service.sampler.RestSampler import org.evomaster.core.problem.enterprise.service.EnterpriseSampler -import org.evomaster.core.problem.rest.service.sampler.ResourceSampler import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.remote.service.RemoteControllerImplementation import org.evomaster.core.search.service.FitnessFunction @@ -22,7 +21,7 @@ class BlackBoxRestModule( super.configure() bind(object : TypeLiteral>() {}) - .to(ResourceSampler::class.java) + .to(RestSampler::class.java) .asEagerSingleton() bind(object : TypeLiteral>() {}) .to(RestSampler::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/RestSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/RestSampler.kt index 4f867dc920..52ca12c455 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/RestSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/RestSampler.kt @@ -274,7 +274,7 @@ class RestSampler : AbstractRestSampler(){ if (created && !get.path.isLastElementAParameter()) { val lastPost = test[test.size - 2] - Lazy.assert{lastPost.verb == HttpVerb.POST} + Lazy.assert{lastPost.verb == HttpVerb.POST || lastPost.verb == HttpVerb.PUT} val available = getMaxTestSizeDuringSampler() - test.size diff --git a/core/src/main/kotlin/org/evomaster/core/utils/CollectionUtils.kt b/core/src/main/kotlin/org/evomaster/core/utils/CollectionUtils.kt index 1977a20462..6058a11b00 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/CollectionUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/CollectionUtils.kt @@ -11,7 +11,9 @@ object CollectionUtils { */ fun duplicates(list: List) : Map { - return list.associateBy({ it },{ list.count { e -> it == e } }) + return list.groupingBy { it }.eachCount() + //previous implementation was too inefficient +// return list.associateBy({ it },{ list.count { e -> it == e } }) .filter { it.value > 1 } } } \ No newline at end of file