From f40ae2f3ba0b04b2983f94a3a78558aed85a599d Mon Sep 17 00:00:00 2001 From: Igor Egorov <118996755+iegorov777@users.noreply.github.com> Date: Thu, 27 Nov 2025 17:44:19 +0300 Subject: [PATCH 01/31] SED-4350 extend automation package test scope (#558) * SED-4305: unit tests for AutomationPackageManager * SED-4305: forbid libraries in local execution (CLI) * SED-4305: fix test * SED-4305: new unit test * SED-4305: remove redundant todo * SED-4350: new unit test * SED-4350: merge master into SED-4350 * SED-4350: fix test * SED-4350: unit test * SED-4350: unit test (cherry picked from commit 6be01732109d08ca1a018e8b179e14ac9d5b897c) --- .../AbstractAutomationPackageManagerTest.java | 12 +- .../AutomationPackageManagerEETest.java | 211 +++++++++++++++++- .../AutomationPackageManagerOSTest.java | 1 + 3 files changed, 216 insertions(+), 8 deletions(-) diff --git a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AbstractAutomationPackageManagerTest.java b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AbstractAutomationPackageManagerTest.java index 0da7edcca..8d260ffa0 100644 --- a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AbstractAutomationPackageManagerTest.java +++ b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AbstractAutomationPackageManagerTest.java @@ -38,7 +38,7 @@ import step.core.controller.ControllerSettingAccessorImpl; import step.core.dynamicbeans.DynamicBeanResolver; import step.core.dynamicbeans.DynamicValueResolver; -import step.core.objectenricher.ObjectHookRegistry; +import step.core.objectenricher.*; import step.core.plans.Plan; import step.core.plans.PlanAccessorImpl; import step.core.scheduler.ExecutionScheduler; @@ -132,7 +132,13 @@ public void before() { AutomationPackageReaderRegistry automationPackageReaderRegistry = new AutomationPackageReaderRegistry(YamlAutomationPackageVersions.ACTUAL_JSON_SCHEMA_PATH, automationPackageHookRegistry, serializationRegistry); automationPackageReaderRegistry.register(apReader); - this.manager = AutomationPackageManager.createMainAutomationPackageManager( + this.manager = createManager(automationPackageHookRegistry, automationPackageReaderRegistry); + + this.manager.setProvidersResolver(new MockedAutomationPackageProvidersResolver(new HashMap<>(), resourceManager, automationPackageReaderRegistry)); + } + + protected AutomationPackageManager createManager(AutomationPackageHookRegistry automationPackageHookRegistry, AutomationPackageReaderRegistry automationPackageReaderRegistry) { + return AutomationPackageManager.createMainAutomationPackageManager( automationPackageAccessor, functionManager, functionAccessor, @@ -144,8 +150,6 @@ public void before() { null, -1, new ObjectHookRegistry() ); - - this.manager.setProvidersResolver(new MockedAutomationPackageProvidersResolver(new HashMap<>(), resourceManager, automationPackageReaderRegistry)); } @After diff --git a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java index 421f80957..31ebdb2b0 100644 --- a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java +++ b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java @@ -25,6 +25,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import step.attachments.FileResolver; +import step.core.AbstractContext; import step.core.maven.MavenArtifactIdentifier; import step.core.objectenricher.*; import step.repositories.artifact.ResolvedMavenArtifact; @@ -34,6 +35,7 @@ import step.resources.ResourceMissingException; import step.resources.ResourceRevisionFileHandle; +import javax.xml.transform.Result; import java.io.*; import java.nio.file.Files; import java.time.Instant; @@ -193,11 +195,8 @@ public void testManagedLibrary(){ MockedAutomationPackageProvidersResolver providersResolver = (MockedAutomationPackageProvidersResolver) manager.getProvidersResolver(); providersResolver.getMavenArtifactMocks().put(libVersion1, new ResolvedMavenArtifact(libJar, new SnapshotMetadata("some timestamp", System.currentTimeMillis(), 1, false))); - try (InputStream is = new FileInputStream(automationPackageJar); - InputStream isAnother = new FileInputStream(anotherAutomationPackageJar); - ) { + try (InputStream is = new FileInputStream(automationPackageJar)) { AutomationPackageFileSource sample1ApSource = AutomationPackageFileSource.withInputStream(is, SAMPLE1_FILE_NAME); - AutomationPackageFileSource anotherApSource = AutomationPackageFileSource.withInputStream(isAnother, SAMPLE1_EXTENDED_FILE_NAME); AutomationPackageFileSource libSource = AutomationPackageFileSource.withMavenIdentifier(libVersion1); // 1. Create managed library by Global Admin @@ -279,6 +278,210 @@ public void testManagedLibrary(){ } } + @Test + public void testManagedLibraryInIsolatedProjects() throws IOException { + File automationPackageJar = new File("src/test/resources/samples/" + SAMPLE1_FILE_NAME); + File anotherAutomationPackageJar = new File("src/test/resources/samples/" + SAMPLE_ECHO_FILE_NAME); + + File libJar = new File("src/test/resources/samples/" + KW_LIB_FILE_NAME); + File libJarUpdated = new File("src/test/resources/samples/" + KW_LIB_FILE_UPDATED_NAME); + + MavenArtifactIdentifier libVersion1 = new MavenArtifactIdentifier("test-group", "test-lib", "1.0.0-SNAPSHOT", null, null); + MockedAutomationPackageProvidersResolver providersResolver = (MockedAutomationPackageProvidersResolver) manager.getProvidersResolver(); + providersResolver.getMavenArtifactMocks().put(libVersion1, new ResolvedMavenArtifact(libJar, new SnapshotMetadata("some timestamp", System.currentTimeMillis(), 1, false))); + + AutomationPackageFileSource libSource = AutomationPackageFileSource.withMavenIdentifier(libVersion1); + + // 1. Create managed library in project1 + AutomationPackageUpdateParameter user1Params = new AutomationPackageUpdateParameterBuilder() + .forJunit() + .withActorUser("user1") + .withEnricher(createTenantEnricher(PROJECT_1)) + .withObjectPredicate(createAccessPredicate(PROJECT_1)) + .withWriteAccessValidator(createWriteAccessValidator(PROJECT_1)) + .build(); + + Resource projectLibResource1 = manager.createAutomationPackageResource(ResourceManager.RESOURCE_TYPE_AP_MANAGED_LIBRARY, libSource, "testManagedLibrary", user1Params); + Assert.assertNotNull(projectLibResource1); + Assert.assertEquals(PROJECT_1, projectLibResource1.getAttribute(ATTRIBUTE_PROJECT_NAME)); + + // 2. Create managed library in project2 with the same name - it is allowed, because we use separate tenants + AutomationPackageUpdateParameter user2Params = new AutomationPackageUpdateParameterBuilder() + .forJunit() + .withActorUser("user2") + .withEnricher(createTenantEnricher(PROJECT_2)) + .withObjectPredicate(createAccessPredicate(PROJECT_2)) + .withWriteAccessValidator(createWriteAccessValidator(PROJECT_2)) + .build(); + + Resource projectLibResource2 = manager.createAutomationPackageResource(ResourceManager.RESOURCE_TYPE_AP_MANAGED_LIBRARY, libSource, "testManagedLibrary", user2Params); + Assert.assertNotNull(projectLibResource2); + Assert.assertEquals(PROJECT_2, projectLibResource2.getAttribute(ATTRIBUTE_PROJECT_NAME)); + + // 3. User1 cannot use the library from project2 + try (InputStream is = new FileInputStream(automationPackageJar)) { + AutomationPackageFileSource sample1ApSource = AutomationPackageFileSource.withInputStream(is, SAMPLE1_FILE_NAME); + try { + AutomationPackageUpdateParameter user1CreateApParams = new AutomationPackageUpdateParameterBuilder() + .forJunit() + .withApSource(sample1ApSource) + .withApLibrarySource(AutomationPackageFileSource.withResourceId(projectLibResource2.getId().toHexString())) + .withAllowUpdate(false) + .withAsync(false) + .withCheckForSameOrigin(true) + .withEnricher(createTenantEnricher(PROJECT_1)) + .withObjectPredicate(createAccessPredicate(PROJECT_1)) + .withWriteAccessValidator(createWriteAccessValidator(PROJECT_1)) + .build(); + + manager.createOrUpdateAutomationPackage(user1CreateApParams); + Assert.fail("Exception should be thrown"); + } catch (AutomationPackageManagerException ex) { + log.info("Exception: {}", ex.getMessage()); + } + } catch (IOException e) { + throw new RuntimeException("IO Exception", e); + } + + AutomationPackageUpdateResult resultAp1; + AutomationPackageUpdateResult resultAp2; + try (InputStream is = new FileInputStream(automationPackageJar); + InputStream isAnother = new FileInputStream(anotherAutomationPackageJar)) { + + // 4. User1 and User2 can create automation packages referencing managed library within their own projects + AutomationPackageFileSource sample1ApSource = AutomationPackageFileSource.withInputStream(is, SAMPLE1_FILE_NAME); + + AutomationPackageUpdateParameter user1CreateApParams = new AutomationPackageUpdateParameterBuilder() + .forJunit() + .withApSource(sample1ApSource) + .withApLibrarySource(AutomationPackageFileSource.withResourceId(projectLibResource1.getId().toHexString())) + .withAllowUpdate(false) + .withAsync(false) + .withCheckForSameOrigin(true) + .withEnricher(createTenantEnricher(PROJECT_1)) + .withObjectPredicate(createAccessPredicate(PROJECT_1)) + .withWriteAccessValidator(createWriteAccessValidator(PROJECT_1)) + .build(); + + resultAp1 = manager.createOrUpdateAutomationPackage(user1CreateApParams); + Assert.assertEquals(CREATED, resultAp1.getStatus()); + AutomationPackageFileSource sample2ApSource = AutomationPackageFileSource.withInputStream(isAnother, SAMPLE1_FILE_NAME); + + AutomationPackageUpdateParameter user2CreateApParams = new AutomationPackageUpdateParameterBuilder() + .forJunit() + .withApSource(sample2ApSource) + .withApLibrarySource(AutomationPackageFileSource.withResourceId(projectLibResource2.getId().toHexString())) + .withAllowUpdate(false) + .withAsync(false) + .withCheckForSameOrigin(true) + .withEnricher(createTenantEnricher(PROJECT_2)) + .withObjectPredicate(createAccessPredicate(PROJECT_2)) + .withWriteAccessValidator(createWriteAccessValidator(PROJECT_2)) + .build(); + + resultAp2 = manager.createOrUpdateAutomationPackage(user2CreateApParams); + Assert.assertEquals(CREATED, resultAp2.getStatus()); + } catch (IOException e) { + throw new RuntimeException("IO Exception", e); + } + + AutomationPackage ap1 = automationPackageAccessor.get(resultAp1.getId()); + AutomationPackage ap2 = automationPackageAccessor.get(resultAp2.getId()); + + // check tenants linked with both APs + Assert.assertEquals(PROJECT_1, ap1.getAttribute(ATTRIBUTE_PROJECT_NAME)); + Assert.assertEquals(PROJECT_2, ap2.getAttribute(ATTRIBUTE_PROJECT_NAME)); + + // check references to libs + Assert.assertEquals(FileResolver.resolveResourceId(ap1.getAutomationPackageLibraryResource()), projectLibResource1.getId().toHexString()); + Assert.assertEquals(FileResolver.resolveResourceId(ap2.getAutomationPackageLibraryResource()), projectLibResource2.getId().toHexString()); + + // 5. User1 updates the lib in project1 - AP from project2 should be untouched + providersResolver.getMavenArtifactMocks().put(libVersion1, new ResolvedMavenArtifact( + libJarUpdated, + new SnapshotMetadata("some timestamp", System.currentTimeMillis(), 1, true)) + ); + + Instant nowBeforeLib1Update = Instant.now(); + RefreshResourceResult refreshResourceResult = manager.getAutomationPackageResourceManager().refreshResourceAndLinkedPackages( + projectLibResource1.getId().toHexString(), user1Params, manager + ); + Assert.assertEquals(RefreshResourceResult.ResultStatus.REFRESHED, refreshResourceResult.getResultStatus()); + + // lib1 has been updated + Resource updatedLib1Resource = resourceManager.getResource(projectLibResource1.getId().toHexString()); + Assert.assertFalse(updatedLib1Resource.getLastModificationDate().toInstant().isBefore(nowBeforeLib1Update)); + Assert.assertArrayEquals(Files.readAllBytes(resourceManager.getResourceFile(projectLibResource1.getId().toHexString()).getResourceFile().toPath()), Files.readAllBytes(libJarUpdated.toPath())); + + // lib2 is not updated + Assert.assertFalse(projectLibResource2.getLastModificationDate().toInstant().isAfter(nowBeforeLib1Update)); + Assert.assertArrayEquals(Files.readAllBytes(resourceManager.getResourceFile(projectLibResource2.getId().toHexString()).getResourceFile().toPath()), Files.readAllBytes(libJar.toPath())); + + // take the actual state from db + ap1 = automationPackageAccessor.get(ap1.getId()); + ap2 = automationPackageAccessor.get(ap2.getId()); + + // ap1 has been reuploaded + Assert.assertFalse(ap1.getLastModificationDate().toInstant().isBefore(nowBeforeLib1Update)); + + // ap2 has not been reuploaded + Assert.assertFalse(ap2.getLastModificationDate().toInstant().isAfter(nowBeforeLib1Update)); + + // original tenants for automation packages should not be changed after reupload + Assert.assertEquals(PROJECT_1, ap1.getAttribute(ATTRIBUTE_PROJECT_NAME)); + Assert.assertEquals(PROJECT_2, ap2.getAttribute(ATTRIBUTE_PROJECT_NAME)); + + // 5. User1 still has the access to AP to read and delete it + AutomationPackage apTakenFromManager = manager.getAutomationPackageById(ap1.getId(), createAccessPredicate(PROJECT_1)); + Assert.assertNotNull(apTakenFromManager); + + manager.removeAutomationPackage(ap1.getId(), "user1", createAccessPredicate(PROJECT_1), createWriteAccessValidator(PROJECT_1)); + + // ap1 doesn't exist anymore + Assert.assertNull(automationPackageAccessor.get(ap1.getId())); + + try { + resourceManager.getResourceFile(updatedLib1Resource.getId().toString()); + Assert.fail("Exception should be thrown"); + } catch (ResourceMissingException ex){ + log.info("Resource deleted: {}", ex.getMessage()); + } + } + + protected AutomationPackageManager createManager(AutomationPackageHookRegistry automationPackageHookRegistry, AutomationPackageReaderRegistry automationPackageReaderRegistry) { + ObjectHookRegistry objectHookRegistry = new ObjectHookRegistry(); + objectHookRegistry.add(new ObjectHook() { + @Override + public ObjectFilter getObjectFilter(AbstractContext context) { + // TODO: maybe we need to mock the object filter also + return null; + } + + @Override + public ObjectEnricher getObjectEnricher(AbstractContext context) { + return createTenantEnricher(context.get("project") == null ? null : (String) context.get("project")); + } + + @Override + public void rebuildContext(AbstractContext context, EnricheableObject object) throws Exception { + context.put("project", object.getAttribute(ATTRIBUTE_PROJECT_NAME)); + } + }); + + return AutomationPackageManager.createMainAutomationPackageManager( + automationPackageAccessor, + functionManager, + functionAccessor, + planAccessor, + resourceManager, + automationPackageHookRegistry, + automationPackageReaderRegistry, + automationPackageLocks, + null, -1, + objectHookRegistry + ); + } + protected WriteAccessValidator createWriteAccessValidator(String ... projectNames){ return new WriteAccessValidator() { @Override diff --git a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java index 8b1545efd..aa18098e4 100644 --- a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java +++ b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java @@ -47,6 +47,7 @@ import java.nio.file.Files; import java.time.Duration; import java.util.*; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; From 1deafe96637e9e24f7cb79ca767bc7b53c2bbdb3 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Thu, 27 Nov 2025 16:44:34 +0100 Subject: [PATCH 02/31] SED-4350 adapting Junit after merge --- .../packages/AutomationPackageManagerEETest.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java index 31ebdb2b0..d7103b0cb 100644 --- a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java +++ b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java @@ -440,6 +440,15 @@ public void testManagedLibraryInIsolatedProjects() throws IOException { // ap1 doesn't exist anymore Assert.assertNull(automationPackageAccessor.get(ap1.getId())); + //Library resources were created manually, they shall not be deleted when deleting APs using them + resourceManager.getResourceFile(updatedLib1Resource.getId().toString()); + resourceManager.getResourceFile(projectLibResource2.getId().toString()); + try { + manager.getAutomationPackageResourceManager().deleteResource(updatedLib1Resource.getId().toString(), user1Params.writeAccessValidator); + } catch (AutomationPackageUnsupportedResourceTypeException e) { + log.error("Unable to delete library", e); + Assert.fail("Unable to delete library"); + } try { resourceManager.getResourceFile(updatedLib1Resource.getId().toString()); Assert.fail("Exception should be thrown"); From be23145b41ec5a6c7a36a563b29a8b51171678f1 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Tue, 2 Dec 2025 13:37:49 +0100 Subject: [PATCH 03/31] SED-4409 Keyword calls with no agent response have no measurements (#561) * SED-4409 Keyword calls with no agent response have no measurements * SED-4409 PR feedback * SED-4409 PR feedback + Junit --- .../measurements/MeasurementPlugin.java | 6 +- .../handlers/CallFunctionHandler.java | 391 ++++++++++-------- .../handlers/CallFunctionHandlerTest.java | 62 +++ 3 files changed, 279 insertions(+), 180 deletions(-) diff --git a/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java b/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java index 64e076c3f..11110a0b6 100644 --- a/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java +++ b/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java @@ -267,7 +267,10 @@ public void afterReportNodeExecution(ExecutionContext executionContext, ReportNo private Measurement createMeasurement(ExecutionContext executionContext, Measure measure, CallFunctionReportNode functionReport) { Map functionAttributes = functionReport.getFunctionAttributes(); Measurement measurement = initMeasurement(executionContext); - measurement.addCustomFields(functionAttributes); + if (functionAttributes != null) { + measurement.addCustomFields(functionAttributes); + measurement.addCustomField(ORIGIN, functionAttributes.get(AbstractOrganizableObject.NAME)); + } measurement.setName(measure.getName()); if (measure.getStatus() != null) { // Note: status should always be set for live measures, but is null unless explicitly set for "output measures". @@ -275,7 +278,6 @@ private Measurement createMeasurement(ExecutionContext executionContext, Measure measurement.setStatus(measure.getStatus().name()); } measurement.setType(getMeasureTypeOrDefault(measure)); - measurement.addCustomField(ORIGIN, functionAttributes.get(AbstractOrganizableObject.NAME)); measurement.setValue(measure.getDuration()); measurement.setBegin(measure.getBegin()); measurement.addCustomField(AGENT_URL, functionReport.getAgentUrl()); diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/CallFunctionHandler.java b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/CallFunctionHandler.java index 41625f61f..40bf7b723 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/CallFunctionHandler.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/CallFunctionHandler.java @@ -56,6 +56,7 @@ import step.core.plugins.ExecutionCallbacks; import step.core.reports.Error; import step.core.reports.ErrorType; +import step.core.reports.Measure; import step.core.variables.VariablesManager; import step.datapool.DataSetHandle; import step.expressions.ProtectedVariable; @@ -64,6 +65,7 @@ import step.functions.execution.FunctionExecutionService; import step.functions.execution.FunctionExecutionServiceException; import step.functions.handler.AbstractFunctionHandler; +import step.functions.handler.MeasureTypes; import step.functions.io.FunctionInput; import step.functions.io.Output; import step.functions.type.FunctionTypeRegistry; @@ -89,6 +91,7 @@ public class CallFunctionHandler extends ArtefactHandler resolveChildrenArtefactBySource_(CallFunction a @Override protected void execute_(CallFunctionReportNode node, CallFunction testArtefact) throws Exception { - String argumentStr = testArtefact.getArgument().get(); - node.setInput(argumentStr); + //This try / finally block is meant to ensure that a keyword measurement is always created even in case error like token selection or keyword call timeouts + long startTime = System.currentTimeMillis(); //initial value might be updated before the actual Keyword call + Long endTime = null; //can be updated right after the keyword call fails, or defined right before creating the error case measurement + String measureName = null; //Will be set to the function name (default for KW measure) if it can be resolved or the final node name otherwise + try { + String argumentStr = testArtefact.getArgument().get(); + node.setInput(argumentStr); - Function function = getFunction(testArtefact); + Function function = getFunction(testArtefact); + measureName = function.getAttributes().get(AbstractOrganizableObject.NAME); - ExecutionCallbacks executionCallbacks = context.getExecutionCallbacks(); - executionCallbacks.beforeFunctionExecution(context, node, function); + ExecutionCallbacks executionCallbacks = context.getExecutionCallbacks(); + executionCallbacks.beforeFunctionExecution(context, node, function); - node.setFunctionId(function.getId().toString()); - node.setFunctionAttributes(function.getAttributes()); + node.setFunctionId(function.getId().toString()); + node.setFunctionAttributes(function.getAttributes()); - String name = node.getName(); - // Name the report node after the keyword if it's not already the case - String functionName = function.getAttribute(AbstractOrganizableObject.NAME); - if(name.equals(CallFunction.ARTEFACT_NAME) && functionName != null) { - node.setName(functionName); - } - - BuildInputResults buildInputResults = buildInput(argumentStr); - node.setInput(buildInputResults.inputArgumentsObfuscated); - - validateInput(buildInputResults.input, function); - - Output output; - Map streamingAttachments = new ConcurrentHashMap<>(); - - if(!context.isSimulation()) { - FunctionGroupContext functionGroupContext = getFunctionGroupContext(); - boolean closeFunctionGroupSessionAfterExecution = (functionGroupContext == null); - FunctionGroupSession functionGroupSession = getOrCreateFunctionGroupSession(functionExecutionService, functionGroupContext); - - // Force local token selection for local plan executions - boolean forceLocalToken = context.getOperationMode() == OperationMode.LOCAL; - TokenWrapper token = selectToken(node, testArtefact, function, functionGroupContext, functionGroupSession, forceLocalToken); - - StreamingResourceUploadContext streamingUploadContext = null; - - try { - String agentUrl = token.getAgent().getAgentUrl(); - node.setAgentUrl(agentUrl); - node.setTokenId(token.getID()); - - Token gridToken = token.getToken(); - - /* Support for streaming uploads produced during this call. We create and register a new context, - provide the necessary information for the upload provider, and set up a listener for the context, - so we can populate the attachment metadata in realtime and attach it to the report node. - - Note / potential TODO: - This could also be "outsourced" to different classes, similarly to how the LiveReportingPlugin - handles the non-websocket context through before/afterReportNodeExecution hooks, then MeasurementPlugin - using that information. For the streaming attachments, all of the logic is concentrated in the following block. - */ - // FIXME: SED-4192 (Step 30+) This will currently only work in a full Step server, not for local AP executions, Unit Tests etc. - StreamingResourceUploadContexts streamingUploadContexts = context.get(StreamingResourceUploadContexts.class); - if (streamingUploadContexts != null) { - streamingUploadContext = new StreamingResourceUploadContext(); - streamingUploadContexts.registerContext(streamingUploadContext); - streamingUploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_EXECUTION_ID, context.getExecutionId()); - streamingUploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_VARIABLES_MANAGER, context.getVariablesManager()); - streamingUploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_REPORT_NODE, node); - ObjectEnricher enricher = context.getObjectEnricher(); - if (enricher != null) { - streamingUploadContext.getAttributes().put(LiveReportingConstants.ACCESSCONTROL_ENRICHER, enricher); - } + String name = node.getName(); + // Name the report node after the keyword if it's not already the case + String functionName = function.getAttribute(AbstractOrganizableObject.NAME); + if (name.equals(CallFunction.ARTEFACT_NAME) && functionName != null) { + node.setName(functionName); + } - buildInputResults.input.getProperties().put(LiveReportingConstants.LIVEREPORTING_CONTROLLER_URL, (String) context.get(LiveReportingConstants.LIVEREPORTING_CONTROLLER_URL)); - buildInputResults.input.getProperties().put(LiveReportingConstants.STREAMING_WEBSOCKET_UPLOAD_PATH, (String) context.get(LiveReportingConstants.STREAMING_WEBSOCKET_UPLOAD_PATH)); - buildInputResults.input.getProperties().put(StreamingResourceUploadContext.PARAMETER_NAME, streamingUploadContext.contextId); + BuildInputResults buildInputResults = buildInput(argumentStr); + node.setInput(buildInputResults.inputArgumentsObfuscated); - streamingUploadContexts.registerListener(streamingUploadContext.contextId, new StreamingResourceUploadContextListener() { + validateInput(buildInputResults.input, function); - @Override - public void onResourceCreationRefused(StreamingResourceMetadata metadata, String reasonPhrase) { - node.getAttachments().add(new SkippedAttachmentMeta(metadata.getFilename(), metadata.getMimeType(), reasonPhrase)); - reportNodeAccessor.save(node); - } + Output output; + Map streamingAttachments = new ConcurrentHashMap<>(); + + if (!context.isSimulation()) { + FunctionGroupContext functionGroupContext = getFunctionGroupContext(); + boolean closeFunctionGroupSessionAfterExecution = (functionGroupContext == null); + FunctionGroupSession functionGroupSession = getOrCreateFunctionGroupSession(functionExecutionService, functionGroupContext); + + // Force local token selection for local plan executions + boolean forceLocalToken = context.getOperationMode() == OperationMode.LOCAL; + TokenWrapper token = selectToken(node, testArtefact, function, functionGroupContext, functionGroupSession, forceLocalToken); - @Override - public void onResourceCreated(String resourceId, StreamingResourceMetadata metadata) { - // This will create an attachment with its immutable properties, but it will not yet "publish" it to the reportNode or set its status etc. - streamingAttachments.put(resourceId, new StreamingAttachmentMeta(new ObjectId(resourceId), metadata.getFilename(), metadata.getMimeType())); + StreamingResourceUploadContext streamingUploadContext = null; + + try { + String agentUrl = token.getAgent().getAgentUrl(); + node.setAgentUrl(agentUrl); + node.setTokenId(token.getID()); + + Token gridToken = token.getToken(); + + /* Support for streaming uploads produced during this call. We create and register a new context, + provide the necessary information for the upload provider, and set up a listener for the context, + so we can populate the attachment metadata in realtime and attach it to the report node. + + Note / potential TODO: + This could also be "outsourced" to different classes, similarly to how the LiveReportingPlugin + handles the non-websocket context through before/afterReportNodeExecution hooks, then MeasurementPlugin + using that information. For the streaming attachments, all of the logic is concentrated in the following block. + */ + // FIXME: SED-4192 (Step 30+) This will currently only work in a full Step server, not for local AP executions, Unit Tests etc. + StreamingResourceUploadContexts streamingUploadContexts = context.get(StreamingResourceUploadContexts.class); + if (streamingUploadContexts != null) { + streamingUploadContext = new StreamingResourceUploadContext(); + streamingUploadContexts.registerContext(streamingUploadContext); + streamingUploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_EXECUTION_ID, context.getExecutionId()); + streamingUploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_VARIABLES_MANAGER, context.getVariablesManager()); + streamingUploadContext.getAttributes().put(LiveReportingConstants.CONTEXT_REPORT_NODE, node); + ObjectEnricher enricher = context.getObjectEnricher(); + if (enricher != null) { + streamingUploadContext.getAttributes().put(LiveReportingConstants.ACCESSCONTROL_ENRICHER, enricher); } - @Override - public void onResourceStatusChanged(String resourceId, StreamingResourceStatus status) { - // Here's where we update the attachment status etc. - StreamingAttachmentMeta attachment = streamingAttachments.get(resourceId); - if (attachment != null) { - // initially, there is no status set (see above) - boolean isFirstUpdate = attachment.getStatus() == null; - attachment.setCurrentSize(status.getCurrentSize()); - attachment.setCurrentNumberOfLines(status.getNumberOfLines()); - attachment.setStatus(StreamingAttachmentMeta.Status.valueOf(status.getTransferStatus().name())); - if (isFirstUpdate) { - // this ensures that attachments are added to the node exactly once, and with meaningful initial data - node.getAttachments().add(attachment); - } + buildInputResults.input.getProperties().put(LiveReportingConstants.LIVEREPORTING_CONTROLLER_URL, (String) context.get(LiveReportingConstants.LIVEREPORTING_CONTROLLER_URL)); + buildInputResults.input.getProperties().put(LiveReportingConstants.STREAMING_WEBSOCKET_UPLOAD_PATH, (String) context.get(LiveReportingConstants.STREAMING_WEBSOCKET_UPLOAD_PATH)); + buildInputResults.input.getProperties().put(StreamingResourceUploadContext.PARAMETER_NAME, streamingUploadContext.contextId); + + streamingUploadContexts.registerListener(streamingUploadContext.contextId, new StreamingResourceUploadContextListener() { + + @Override + public void onResourceCreationRefused(StreamingResourceMetadata metadata, String reasonPhrase) { + node.getAttachments().add(new SkippedAttachmentMeta(metadata.getFilename(), metadata.getMimeType(), reasonPhrase)); reportNodeAccessor.save(node); - } else { - logger.warn("Unexpected: Unable to find attachment for resource '{}'", resourceId); } - } - }); - } - LiveReportingContext liveReportingContext = LiveReportingPlugin.getLiveReportingContext(context); - if (liveReportingContext != null) { - // set up the plumbing to let the handler know where to forward measures - buildInputResults.input.getProperties().put(LiveReportingConstants.LIVEREPORTING_CONTEXT_ID, liveReportingContext.id); - } + @Override + public void onResourceCreated(String resourceId, StreamingResourceMetadata metadata) { + // This will create an attachment with its immutable properties, but it will not yet "publish" it to the reportNode or set its status etc. + streamingAttachments.put(resourceId, new StreamingAttachmentMeta(new ObjectId(resourceId), metadata.getFilename(), metadata.getMimeType())); + } - if(gridToken.isLocal()) { - TokenReservationSession session = (TokenReservationSession) gridToken.getAttachedObject(TokenWrapper.TOKEN_RESERVATION_SESSION); - session.put(AbstractFunctionHandler.EXECUTION_CONTEXT_KEY, new ExecutionContextWrapper(context)); - session.put(AbstractFunctionHandler.ARTEFACT_PATH, currentArtefactPath()); - } else { - // only report non-local (i.e. actual agent) URLs - context.addAgentUrl(agentUrl); - } + @Override + public void onResourceStatusChanged(String resourceId, StreamingResourceStatus status) { + // Here's where we update the attachment status etc. + StreamingAttachmentMeta attachment = streamingAttachments.get(resourceId); + if (attachment != null) { + // initially, there is no status set (see above) + boolean isFirstUpdate = attachment.getStatus() == null; + attachment.setCurrentSize(status.getCurrentSize()); + attachment.setCurrentNumberOfLines(status.getNumberOfLines()); + attachment.setStatus(StreamingAttachmentMeta.Status.valueOf(status.getTransferStatus().name())); + if (isFirstUpdate) { + // this ensures that attachments are added to the node exactly once, and with meaningful initial data + node.getAttachments().add(attachment); + } + reportNodeAccessor.save(node); + } else { + logger.warn("Unexpected: Unable to find attachment for resource '{}'", resourceId); + } + } + }); + } - OperationManager.getInstance().enter(OPERATION_KEYWORD_CALL, new Object[]{function.getAttributes(), token.getToken(), token.getAgent()}, - node.getId().toString(), node.getArtefactHash()); + LiveReportingContext liveReportingContext = LiveReportingPlugin.getLiveReportingContext(context); + if (liveReportingContext != null) { + // set up the plumbing to let the handler know where to forward measures + buildInputResults.input.getProperties().put(LiveReportingConstants.LIVEREPORTING_CONTEXT_ID, liveReportingContext.id); + } - try { - output = functionExecutionService.callFunction(token.getID(), function, buildInputResults.input, JsonObject.class, context); - } finally { - OperationManager.getInstance().exit(); - } - executionCallbacks.afterFunctionExecution(context, node, function, output); + if (gridToken.isLocal()) { + TokenReservationSession session = (TokenReservationSession) gridToken.getAttachedObject(TokenWrapper.TOKEN_RESERVATION_SESSION); + session.put(AbstractFunctionHandler.EXECUTION_CONTEXT_KEY, new ExecutionContextWrapper(context)); + session.put(AbstractFunctionHandler.ARTEFACT_PATH, currentArtefactPath()); + } else { + // only report non-local (i.e. actual agent) URLs + context.addAgentUrl(agentUrl); + } - Error error = output.getError(); - if(error!=null) { - node.setError(error); - node.setStatus(error.getType()==ErrorType.TECHNICAL?ReportNodeStatus.TECHNICAL_ERROR:ReportNodeStatus.FAILED); - } else { - node.setStatus(ReportNodeStatus.PASSED); - } + OperationManager.getInstance().enter(OPERATION_KEYWORD_CALL, new Object[]{function.getAttributes(), token.getToken(), token.getAgent()}, + node.getId().toString(), node.getArtefactHash()); - if(output.getPayload() != null) { - Object outputPayload = (useLegacyOutput) ? output.getPayload() : new UserFriendlyJsonObject(output.getPayload()); - context.getVariablesManager().putVariable(node, "output", outputPayload); - node.setOutput(output.getPayload().toString()); - node.setOutputObject(output.getPayload()); - ReportNode parentNode = context.getReportNodeCache().get(node.getParentID()); - if(parentNode!=null) { - context.getVariablesManager().putVariable(parentNode, "previous", outputPayload); + startTime = System.currentTimeMillis(); + try { + output = functionExecutionService.callFunction(token.getID(), function, buildInputResults.input, JsonObject.class, context); + } finally { + endTime = System.currentTimeMillis(); + OperationManager.getInstance().exit(); + } + executionCallbacks.afterFunctionExecution(context, node, function, output); + + Error error = output.getError(); + if (error != null) { + node.setError(error); + node.setStatus(error.getType() == ErrorType.TECHNICAL ? ReportNodeStatus.TECHNICAL_ERROR : ReportNodeStatus.FAILED); + } else { + node.setStatus(ReportNodeStatus.PASSED); } - } - if(output.getAttachments()!=null) { - for(Attachment a:output.getAttachments()) { - AttachmentMeta attachmentMeta = reportNodeAttachmentManager.createAttachment(AttachmentHelper.hexStringToByteArray(a.getHexContent()), a.getName(), a.getMimeType()); - node.addAttachment(attachmentMeta); + if (output.getPayload() != null) { + Object outputPayload = (useLegacyOutput) ? output.getPayload() : new UserFriendlyJsonObject(output.getPayload()); + context.getVariablesManager().putVariable(node, "output", outputPayload); + node.setOutput(output.getPayload().toString()); + node.setOutputObject(output.getPayload()); + ReportNode parentNode = context.getReportNodeCache().get(node.getParentID()); + if (parentNode != null) { + context.getVariablesManager().putVariable(parentNode, "previous", outputPayload); + } } - } - if(output.getMeasures()!=null) { - node.setMeasures(output.getMeasures()); - } - String drainOutputValue = testArtefact.getResultMap().get(); - drainOutput(drainOutputValue, output); - } finally { - if(closeFunctionGroupSessionAfterExecution) { - functionGroupSession.releaseTokens(true); - } - if (streamingUploadContext != null) { - if (!streamingAttachments.isEmpty()) { - // Status updates come in an asynchronous fashion, so for uploads that were NOT properly finalized, - // the transition message from IN_PROGRESS to FAILED may be received slightly after the call is considered finished (SED-4277). - // If this is the case, try to wait a little bit for the message to arrive (will be handled in a different thread), - // before unregistering our context listener. - - // Testing shows that we usually need 0, 1, or (rarely) 2, (extremely rarely) 3 or 4 iterations, - // and that it's more likely to occur if the KW is run on the controller itself (makes sense, because - // when the call is local, the WS streaming overhead is - relatively seen - higher than if both are remote) - int max = 30; // x 50 ms = 1.5 seconds, this should be more than enough time - for (int i = 0; i <= max; ++i) { - boolean unfinished = streamingAttachments.values().stream() - .map(StreamingAttachmentMeta::getStatus) - .filter(Objects::nonNull) - .anyMatch(status -> !(status.equals(StreamingAttachmentMeta.Status.COMPLETED) || status.equals(StreamingAttachmentMeta.Status.FAILED))); - if (!unfinished) { - break; - } - if (i == max) { - logger.warn("Giving up waiting for streaming uploads to be finalized, reportNode {} may contain inconsistent attachment status metadata", node.getId()); - } else { - if (logger.isDebugEnabled()) { - logger.debug("Waiting for all streaming uploads to transition to a final state ({}/{}), rnId={}", (i + 1), max, node.getId()); + if (output.getAttachments() != null) { + for (Attachment a : output.getAttachments()) { + AttachmentMeta attachmentMeta = reportNodeAttachmentManager.createAttachment(AttachmentHelper.hexStringToByteArray(a.getHexContent()), a.getName(), a.getMimeType()); + node.addAttachment(attachmentMeta); + } + } + if (output.getMeasures() != null) { + node.setMeasures(output.getMeasures()); + } + + String drainOutputValue = testArtefact.getResultMap().get(); + drainOutput(drainOutputValue, output); + } finally { + if (closeFunctionGroupSessionAfterExecution) { + functionGroupSession.releaseTokens(true); + } + if (streamingUploadContext != null) { + if (!streamingAttachments.isEmpty()) { + // Status updates come in an asynchronous fashion, so for uploads that were NOT properly finalized, + // the transition message from IN_PROGRESS to FAILED may be received slightly after the call is considered finished (SED-4277). + // If this is the case, try to wait a little bit for the message to arrive (will be handled in a different thread), + // before unregistering our context listener. + + // Testing shows that we usually need 0, 1, or (rarely) 2, (extremely rarely) 3 or 4 iterations, + // and that it's more likely to occur if the KW is run on the controller itself (makes sense, because + // when the call is local, the WS streaming overhead is - relatively seen - higher than if both are remote) + int max = 30; // x 50 ms = 1.5 seconds, this should be more than enough time + for (int i = 0; i <= max; ++i) { + boolean unfinished = streamingAttachments.values().stream() + .map(StreamingAttachmentMeta::getStatus) + .filter(Objects::nonNull) + .anyMatch(status -> !(status.equals(StreamingAttachmentMeta.Status.COMPLETED) || status.equals(StreamingAttachmentMeta.Status.FAILED))); + if (!unfinished) { + break; + } + if (i == max) { + logger.warn("Giving up waiting for streaming uploads to be finalized, reportNode {} may contain inconsistent attachment status metadata", node.getId()); + } else { + if (logger.isDebugEnabled()) { + logger.debug("Waiting for all streaming uploads to transition to a final state ({}/{}), rnId={}", (i + 1), max, node.getId()); + } + Thread.sleep(50); } - Thread.sleep(50); } } - } - context.require(StreamingResourceUploadContexts.class).unregisterContext(streamingUploadContext); + context.require(StreamingResourceUploadContexts.class).unregisterContext(streamingUploadContext); + } + callChildrenArtefacts(node, testArtefact); } - callChildrenArtefacts(node, testArtefact); + } else { + output = new Output<>(); + output.setPayload(JsonProviderCache.createObjectBuilder().build()); + node.setOutputObject(output.getPayload()); + node.setOutput(output.getPayload().toString()); + node.setStatus(ReportNodeStatus.PASSED); } - } else { - output = new Output<>(); - output.setPayload(JsonProviderCache.createObjectBuilder().build()); - node.setOutputObject(output.getPayload()); - node.setOutput(output.getPayload().toString()); - node.setStatus(ReportNodeStatus.PASSED); + } finally { + createKeywordMeasureIfAbsent(node, measureName, endTime, startTime); + } + } + + /** + * A Call function shall always have a measure of type "keyword", if the output measure is empty (mostly happen in case the agent call was not completed due to interruption), we create one directly here + * Note don't use Map.of(), List.of() as these could be enriched with more content later + * The status is left null to keep the default logic: node status is used when converting Measure to Measurement by the MeasurementPlugin + * @param node the node for which we ensure a keyword measure exists + * @param measureName the name of the measure to be created, fallback to node name if null + * @param endTime the measure endtime, fallback to current time if null + * @param startTime the startTime of the measure + */ + private static void createKeywordMeasureIfAbsent(CallFunctionReportNode node, String measureName, Long endTime, long startTime) { + if (node.getMeasures() == null || node.getMeasures().isEmpty()) { + Map data = new HashMap<>(); + data.put(MeasureTypes.ATTRIBUTE_TYPE, MeasureTypes.TYPE_KEYWORD); + List measures = Objects.requireNonNullElse(node.getMeasures(), new ArrayList<>()); + measureName = Objects.requireNonNullElse(measureName, node.getName() + UNRESOLVED_KEYWORD_MEASUREMENT_SUFFIX); + long duration = Objects.requireNonNullElse(endTime, System.currentTimeMillis()) - startTime; + measures.add(new Measure(measureName, duration, startTime, data, null)); + node.setMeasures(measures); } } diff --git a/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/CallFunctionHandlerTest.java b/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/CallFunctionHandlerTest.java index ba4a91fc9..52b348c72 100644 --- a/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/CallFunctionHandlerTest.java +++ b/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/CallFunctionHandlerTest.java @@ -52,8 +52,10 @@ import step.datapool.DataSetHandle; import step.engine.plugins.FunctionPlugin; import step.expressions.ExpressionHandler; +import step.functions.handler.MeasureTypes; import step.functions.io.Output; import step.functions.io.OutputBuilder; +import step.grid.client.AbstractGridClientImpl; import step.grid.io.Attachment; import step.parameter.Parameter; import step.parameter.ParameterManager; @@ -196,6 +198,51 @@ public void testError() { CallFunctionReportNode node = getCallFunctionReportNode(result); assertEquals("My Error", node.getError().getMsg()); + assertEquals(1, node.getMeasures().size()); + assertEquals("MyFunction", node.getMeasures().get(0).getName()); + assertEquals(MeasureTypes.TYPE_KEYWORD, node.getMeasures().get(0).getData().get(MeasureTypes.ATTRIBUTE_TYPE)); + } + + @Test + public void testKeywordNotFoundError() { + MyFunction function = newFailingFunction(); + Plan plan = newCallFunctionPlan(function); + plan.setFunctions(List.of()); + + PlanRunnerResult result = executionEngine.execute(plan); + CallFunctionReportNode node = getCallFunctionReportNode(result); + + assertEquals("Unable to find keyword with attributes {\"name\":\"MyFunction\"}", node.getError().getMsg()); + assertEquals(1, node.getMeasures().size()); + assertEquals("MyFunction_UnresolvedKeyword", node.getMeasures().get(0).getName()); + assertEquals(MeasureTypes.TYPE_KEYWORD, node.getMeasures().get(0).getData().get(MeasureTypes.ATTRIBUTE_TYPE)); + } + + @Test + public void testTimeoutError() { + MyFunction function = newPassingFunction(); + function.setCallTimeout(new DynamicValue<>(1)); + Plan plan = newCallFunctionPlan(function); + + PlanRunnerResult result = executionEngine.execute(plan); + CallFunctionReportNode node = getCallFunctionReportNode(result); + + assertEquals("Unexpected error while calling keyword: java.lang.RuntimeException The defined call timeout of the function should be higher than 100ms", node.getError().getMsg()); + assertEquals(1, node.getMeasures().size()); + assertEquals("MyFunction", node.getMeasures().get(0).getName()); + assertEquals(MeasureTypes.TYPE_KEYWORD, node.getMeasures().get(0).getData().get(MeasureTypes.ATTRIBUTE_TYPE)); + + //Cannot directly test the call timeout here since we're using a local token which doesn't apply it + function = newFunctionThrowingException(); + function.setCallTimeout(new DynamicValue<>(100)); + plan = newCallFunctionPlan(function); + result = executionEngine.execute(plan); + node = getCallFunctionReportNode(result); + + assertEquals("Unexpected error while calling keyword: java.lang.RuntimeException Runtime Exception thrown", node.getError().getMsg()); + assertEquals(1, node.getMeasures().size()); + assertEquals("MyFunction", node.getMeasures().get(0).getName()); + assertEquals(MeasureTypes.TYPE_KEYWORD, node.getMeasures().get(0).getData().get(MeasureTypes.ATTRIBUTE_TYPE)); } @Test @@ -389,6 +436,21 @@ private static MyFunction newPassingFunction() { return function; } + private static MyFunction newFunctionThrowingException() { + MyFunction function = new MyFunction(input -> { + Output output = new Output<>(); + + + List measures = new ArrayList<>(); + measures.add(new Measure("Measure1", 1, 1, null)); + output.setMeasures(measures); + + throw new RuntimeException("Runtime Exception thrown"); + }); + function.addAttribute(AbstractOrganizableObject.NAME, "MyFunction"); + return function; + } + public static MyFunction newPassingFunctionWithInput() { MyFunction function = new MyFunction(input -> { Output output = new Output<>(); From 23a70a3763f4914c8ac7dc56de7729ddc981b8ff Mon Sep 17 00:00:00 2001 From: David Stephan Date: Tue, 2 Dec 2025 14:28:17 +0100 Subject: [PATCH 04/31] SED-000 AP Junit test flakiness --- .../AutomationPackageManagerEETest.java | 3 ++- .../AutomationPackageManagerOSTest.java | 24 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java index d7103b0cb..139604609 100644 --- a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java +++ b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerEETest.java @@ -279,7 +279,7 @@ public void testManagedLibrary(){ } @Test - public void testManagedLibraryInIsolatedProjects() throws IOException { + public void testManagedLibraryInIsolatedProjects() throws IOException, InterruptedException { File automationPackageJar = new File("src/test/resources/samples/" + SAMPLE1_FILE_NAME); File anotherAutomationPackageJar = new File("src/test/resources/samples/" + SAMPLE_ECHO_FILE_NAME); @@ -403,6 +403,7 @@ public void testManagedLibraryInIsolatedProjects() throws IOException { ); Instant nowBeforeLib1Update = Instant.now(); + Thread.sleep(1); RefreshResourceResult refreshResourceResult = manager.getAutomationPackageResourceManager().refreshResourceAndLinkedPackages( projectLibResource1.getId().toHexString(), user1Params, manager ); diff --git a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java index aa18098e4..4778e0817 100644 --- a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java +++ b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java @@ -321,8 +321,26 @@ public void testZipArchive() throws IOException { } } + private void retryFlakyTest(int retries, Runnable test, String testName) { + Error lastError = null; + for (int i = 1; i <= retries; i++) { + try { + test.run(); + return; // Test passed, return + } catch (Error e) { + log.warn("Flaky test '{}' failed on iteration {} of {}", testName, i, retries, e); + lastError = e; + } + } + throw lastError; + } + @Test - public void testUpdateAsync() throws IOException, InterruptedException { + public void testUpdateAsyncWithRetry() { + retryFlakyTest(3, this::testUpdateAsync, "testUpdateAsync"); + } + + public void testUpdateAsync() { File automationPackageJar = new File("src/test/resources/samples/" + SAMPLE1_FILE_NAME); try(InputStream is1 = new FileInputStream(automationPackageJar); @@ -358,6 +376,10 @@ public void testUpdateAsync() throws IOException, InterruptedException { , Map.of("OS", "WINDOWS", "TYPE", "PLAYWRIGHT")); } + } catch (FileNotFoundException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); } } From 7078864220092593a3d658ee97fb977eca8e6fbb Mon Sep 17 00:00:00 2001 From: Igor Egorov <118996755+iegorov777@users.noreply.github.com> Date: Wed, 3 Dec 2025 18:33:53 +0300 Subject: [PATCH 05/31] SED-4403 deploy library error messages are not clear enough (#562) SED-4404: improve error messages (cherry picked from commit 011fbcbd6202966711b5b79628b4ca405abe8cd7) --- ...omationPackageFromInputStreamProvider.java | 2 +- .../packages/AutomationPackageManager.java | 28 +++++++++---------- .../AutomationPackageManagerException.java | 6 ++++ .../AutomationPackageResourceManager.java | 10 +++---- .../execution/AutomationPackageExecutor.java | 2 +- .../IsolatedAutomationPackageRepository.java | 4 +-- ...epositoryWithAutomationPackageSupport.java | 6 ++-- .../core/maven/MavenArtifactIdentifier.java | 6 +++- 8 files changed, 37 insertions(+), 27 deletions(-) diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AbstractAutomationPackageFromInputStreamProvider.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AbstractAutomationPackageFromInputStreamProvider.java index efeb08513..a9c434cbc 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AbstractAutomationPackageFromInputStreamProvider.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AbstractAutomationPackageFromInputStreamProvider.java @@ -33,7 +33,7 @@ public AbstractAutomationPackageFromInputStreamProvider(InputStream packageStrea try { tempFile = InputStreamToTempFileDownloader.copyStreamToTempFile(packageStream, fileName); } catch (Exception ex) { - throw new AutomationPackageManagerException("Unable to store automation package file", ex); + throw new AutomationPackageManagerException("Unable to store automation package file", ex, true); } } diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java index 364ddbf21..1e46c407a 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java @@ -350,7 +350,7 @@ public AutomationPackageUpdateResult createOrUpdateAutomationPackage(AutomationP return createOrUpdateAutomationPackage(provider, apLibProvider, parameters); } } catch (IOException | AutomationPackageReadingException | ManagedLibraryMissingException ex) { - throw new AutomationPackageManagerException("Automation package cannot be created. Caused by: " + ex.getMessage(), ex); + throw new AutomationPackageManagerException("Automation package cannot be created.", ex, true); } } @@ -487,7 +487,7 @@ public AutomationPackageUpdateResult createOrUpdateAutomationPackage(AutomationP } catch (Throwable e) { log.error("Error while updating the automation package with name '{}':{}}.", packageName, packageId); handleAutomationPackageDeploymentErrors(parameters, packageName, packageId, packageResource.get(), libraryResource.get()); - throw (e instanceof AutomationPackageManagerException) ? (AutomationPackageManagerException) e: new AutomationPackageManagerException("Unable to update the automation package '" + packageName + "':" + packageId + ". Cause: " + e.getMessage(), e); + throw (e instanceof AutomationPackageManagerException) ? (AutomationPackageManagerException) e: new AutomationPackageManagerException("Unable to update the automation package '" + packageName + "':" + packageId + ".", e, true); } finally { if (!isRunningAsync.get() && staging.get() != null) { staging.get().getResourceManager().cleanup(); @@ -528,7 +528,7 @@ private ObjectId performUpdateTasks(AutomationPackageUpdateParameter parameters, packageName, packageId, e); handleAutomationPackageDeploymentErrors(parameters, packageName, packageId, packageResource, libraryResource); throw (e instanceof AutomationPackageManagerException) ? (AutomationPackageManagerException) e : - new AutomationPackageManagerException("Unable to update the automation package '" + packageName + "':" + packageId + ". Cause: " + e.getMessage(), e); + new AutomationPackageManagerException("Unable to update the automation package '" + packageName + "':" + packageId + ".", e, true); } else { throw e; } @@ -687,7 +687,7 @@ public Resource createAutomationPackageResource(String resourceType, AutomationP true); } } catch (IOException | ManagedLibraryMissingException e) { - throw new AutomationPackageManagerException("Automation package library provider exception", e); + throw new AutomationPackageManagerException("Automation package library provider exception.", e, true); } case ResourceManager.RESOURCE_TYPE_AP_LIBRARY: // We upload the new resource for package library. Existing resource cannot be reused - to update existing AP resources there is a separate 'refresh' action @@ -698,7 +698,7 @@ public Resource createAutomationPackageResource(String resourceType, AutomationP null, parameters, false, true); } catch (IOException | ManagedLibraryMissingException e) { - throw new AutomationPackageManagerException("Automation package library provider exception", e); + throw new AutomationPackageManagerException("Automation package library provider exception.", e, true); } case ResourceManager.RESOURCE_TYPE_AP: // We upload the new main resource for AP. Existing resource cannot be reused - to update existing AP resources there is a separate 'refresh' action @@ -710,13 +710,13 @@ public Resource createAutomationPackageResource(String resourceType, AutomationP false, true); } catch (IOException e) { - throw new AutomationPackageManagerException("Automation package library provider exception", e); + throw new AutomationPackageManagerException("Automation package library provider exception.", e, true); } default: throw new AutomationPackageManagerException("Unsupported resource type: " + resourceType); } } catch (AutomationPackageReadingException ex) { - throw new AutomationPackageManagerException("Cannot create new resource: " + resourceType, ex); + throw new AutomationPackageManagerException("Cannot create new resource (" + resourceType + ").", ex, true); } } @@ -751,10 +751,10 @@ public Resource updateAutomationPackageManagedLibrary(String id, AutomationPacka true); } } catch (IOException | ManagedLibraryMissingException e) { - throw new AutomationPackageManagerException("Automation package library provider exception: " + e.getMessage(), e); + throw new AutomationPackageManagerException("Automation package library provider exception.", e, true); } } catch (AutomationPackageReadingException ex) { - throw new AutomationPackageManagerException("Cannot update the managed library with id " + id + "; reason: " + ex.getMessage(), ex); + throw new AutomationPackageManagerException("Cannot update the managed library with id " + id + ".", ex, true); } //Now that we updated the managed library, we trigger the reload of APs using it Set automationPackagesIdsByResourceId = linkedAutomationPackagesFinder.findAutomationPackagesIdsByResourceId(resource.getId().toHexString(), List.of()); @@ -902,7 +902,7 @@ protected void fillStaging(AutomationPackage newPackage, AutomationPackageStagin } catch (Exception e){ String fieldNameStr = hookEntry.fieldName == null ? "" : " for '" + hookEntry.fieldName + "'"; // throw AutomationPackageManagerException to be handled as ControllerException in services - throw new AutomationPackageManagerException("onPrepareStaging hook invocation failed" + fieldNameStr + " in the automation package '" + packageContent.getName() + "'. " + e.getMessage(), e); + throw new AutomationPackageManagerException("onPrepareStaging hook invocation failed" + fieldNameStr + " in the automation package '" + packageContent.getName() + "'.", e, true); } } } @@ -917,7 +917,7 @@ protected void persistStagedEntities(AutomationPackage newPackage, AutomationPac resourceManager.copyResource(resource, staging.getResourceManager(), actorUser); } } catch (IOException | InvalidResourceFormatException e) { - throw new AutomationPackageManagerException("Unable to persist a resource in automation package", e); + throw new AutomationPackageManagerException("Unable to persist a resource in automation package.", e, true); } for (Function completeFunction : staging.getFunctions()) { @@ -945,7 +945,7 @@ protected void persistStagedEntities(AutomationPackage newPackage, AutomationPac } catch (Exception e){ String fieldNameStr = hookEntry.fieldName == null ? "" : " for '" + hookEntry.fieldName + "'"; // throw AutomationPackageManagerException to be handled as ControllerException in services - throw new AutomationPackageManagerException("onCreate hook invocation failed" + fieldNameStr + " in the automation package '" + packageContent.getName() + "'. " + e.getMessage(), e); + throw new AutomationPackageManagerException("onCreate hook invocation failed" + fieldNameStr + " in the automation package '" + packageContent.getName() + "'.", e, true); } } @@ -961,7 +961,7 @@ public void runExtensionsBeforeIsolatedExecution(AutomationPackage automationPac automationPackageHookRegistry.beforeIsolatedExecution(automationPackage, executionContext, apManagerExtensions, importResult); } catch (Exception e){ // throw AutomationPackageManagerException to be handled as ControllerException in services - throw new AutomationPackageManagerException("beforeIsolatedExecution hook invocation failed in the automation package '" + automationPackage.getAttribute(AbstractOrganizableObject.NAME) + "'. " + e.getMessage(), e); + throw new AutomationPackageManagerException("beforeIsolatedExecution hook invocation failed in the automation package '" + automationPackage.getAttribute(AbstractOrganizableObject.NAME) + "'.", e, true); } } @@ -1189,7 +1189,7 @@ protected void deleteAdditionalData(AutomationPackage automationPackage, Automat automationPackageHookRegistry.onAutomationPackageDelete(automationPackage, context, null); } catch (Exception e){ // throw AutomationPackageManagerException to be handled as ControllerException in services - throw new AutomationPackageManagerException("onAutomationPackageDelete hook invocation failed in the automation package '" + automationPackage.getAttribute(AbstractOrganizableObject.NAME) + "'. " + e.getMessage(), e); + throw new AutomationPackageManagerException("onAutomationPackageDelete hook invocation failed in the automation package '" + automationPackage.getAttribute(AbstractOrganizableObject.NAME) + "'.", e, true); } } diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManagerException.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManagerException.java index cad5cf43f..835775ab5 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManagerException.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManagerException.java @@ -27,4 +27,10 @@ public AutomationPackageManagerException(String message) { public AutomationPackageManagerException(String message, Throwable cause) { super(message, cause); } + + public AutomationPackageManagerException(String message, Throwable cause, boolean addCauseToMessage) { + // the cause is added to the message to be displayed on UI (propagated to step.core.deployment.ControllerServiceException) + super(message + (addCauseToMessage ? " Caused by: " + cause.getMessage() : ""), cause); + } + } diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java index ebf3e6574..47b6ccd7b 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java @@ -106,7 +106,7 @@ public void validateUploadOrReuseAutomationPackageLibrary(AutomationPackageLibra // all these exceptions are technical, so we log the whole stack trace here, but throw the AutomationPackageManagerException // to provide the short error message without technical details to the client log.error("Unable to upload the automation package library", e); - throw new AutomationPackageManagerException("Unable to upload the automation package library: " + apLibProvider, e); + throw new AutomationPackageManagerException("Unable to upload the automation package library.", e, true); } } @@ -176,7 +176,7 @@ public Resource uploadOrReuseAutomationPackageLibrary(AutomationPackageLibraryPr // all these exceptions are technical, so we log the whole stack trace here, but throw the AutomationPackageManagerException // to provide the short error message without technical details to the client log.error("Unable to upload the automation package library", e); - throw new AutomationPackageManagerException("Unable to upload the automation package library: " + apLibProvider, e); + throw new AutomationPackageManagerException("Unable to upload the automation package library.", e, true); } return uploadedResource; } @@ -223,7 +223,7 @@ public Resource uploadOrReuseApResource(AutomationPackageArchiveProvider apProvi ); } catch (IOException | InvalidResourceFormatException | AutomationPackageUnsupportedResourceTypeException e) { - throw new AutomationPackageManagerException("Unable to create the resource for automation package", e); + throw new AutomationPackageManagerException("Unable to create the resource for automation package.", e, true); } } } @@ -424,7 +424,7 @@ private RefreshResourceResult refreshResourceAndLinkedPackages(Resource resource } } } catch (AutomationPackageReadingException e) { - throw new AutomationPackageManagerException("Cannot restore the file from maven artifactory", e); + throw new AutomationPackageManagerException("Cannot restore the file from maven artifactory.", e, true); } if (refreshResourceResult.getResultStatus() == RefreshResourceResult.ResultStatus.REFRESHED) { @@ -452,7 +452,7 @@ private void updateMavenFileContentInResourceManager(Resource resource, MavenArt resourceManager.saveResource(updated); } } catch (InvalidResourceFormatException | IOException | AutomationPackageReadingException ex) { - throw new AutomationPackageManagerException("Cannot restore the file from maven artifactory", ex); + throw new AutomationPackageManagerException("Cannot restore the file from maven artifactory.", ex, true); } } diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/AutomationPackageExecutor.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/AutomationPackageExecutor.java index 40df96f29..ecab7ed5d 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/AutomationPackageExecutor.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/AutomationPackageExecutor.java @@ -139,7 +139,7 @@ public List runInIsolation(AutomationPackageFileSource automationPackage } } } catch (IOException | AutomationPackageReadingException | ManagedLibraryMissingException e) { - throw new AutomationPackageManagerException("Unable to read the provided package library. Reason: " + e.getMessage(), e); + throw new AutomationPackageManagerException("Unable to read the provided package library.", e, true); } // and then we read the ap from just stored file diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java index 6e8c26f6d..bc9411ac7 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java @@ -269,7 +269,7 @@ private Resource saveApResource(String contextId, InputStream apStream, String f return resource; } catch (IOException | InvalidResourceFormatException ex) { - throw new AutomationPackageManagerException("Cannot save automation package as resource: " + fileName, ex); + throw new AutomationPackageManagerException("Cannot save automation package as resource: " + fileName + ".", ex, true); } } @@ -283,7 +283,7 @@ public void setApNameForResource(Resource resource, String apName){ resource.addCustomField(AP_NAME_CUSTOM_FIELD, apName); resourceManager.saveResource(resource); } catch (IOException ex) { - throw new AutomationPackageManagerException("Cannot update the automation package name in resource: " + resource.getId(), ex); + throw new AutomationPackageManagerException("Cannot update the automation package name in resource: " + resource.getId() + ".", ex, true); } } diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/RepositoryWithAutomationPackageSupport.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/RepositoryWithAutomationPackageSupport.java index 885d2082b..e1f0a671b 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/RepositoryWithAutomationPackageSupport.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/RepositoryWithAutomationPackageSupport.java @@ -350,9 +350,9 @@ protected AutomationPackageFile restoreLibraryFile(String contextId, Map foundResources = resourceManager.findManyByCriteria( @@ -384,7 +384,7 @@ protected boolean tryToReloadResourceFromMaven(Resource resource) { return true; } } catch (InvalidResourceFormatException | IOException | AutomationPackageReadingException ex) { - throw new AutomationPackageManagerException("Cannot restore the file for from maven artifactory", ex); + throw new AutomationPackageManagerException("Cannot restore the file for from maven artifactory.", ex, true); } } } diff --git a/step-core/src/main/java/step/core/maven/MavenArtifactIdentifier.java b/step-core/src/main/java/step/core/maven/MavenArtifactIdentifier.java index 4c53e8e61..28ec0f287 100644 --- a/step-core/src/main/java/step/core/maven/MavenArtifactIdentifier.java +++ b/step-core/src/main/java/step/core/maven/MavenArtifactIdentifier.java @@ -22,6 +22,7 @@ import step.resources.ResourceOriginType; import java.util.Objects; +import java.util.Optional; public class MavenArtifactIdentifier implements ResourceOrigin { @@ -145,7 +146,10 @@ public static boolean isMvnIdentifierShortString(String shortString){ } public String toShortString() { - String res = String.format(MVN_PREFIX + ":%s:%s:%s", getGroupId(), this.getArtifactId(), getVersion()); + String res = String.format(MVN_PREFIX + ":%s:%s:%s", + Optional.ofNullable(getGroupId()).orElse(""), + Optional.ofNullable(getArtifactId()).orElse(""), + Optional.ofNullable(getVersion()).orElse("")); if (this.getClassifier() != null || this.getType() != null) { res += ":"; if (this.getClassifier() != null) { From 291833fe2408fc6f2b159448a8b41cdc7c7eeea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 8 Dec 2025 13:08:51 +0100 Subject: [PATCH 06/31] SED-4428 Uploading an AP defining K6 keywords without scriptDirectory fails (#565) --- .../step/automation/packages/AutomationPackageManager.java | 2 +- .../automation/packages/AutomationPackageResourceManager.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java index 1e46c407a..3226528be 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageManager.java @@ -485,7 +485,7 @@ public AutomationPackageUpdateResult createOrUpdateAutomationPackage(AutomationP ); } } catch (Throwable e) { - log.error("Error while updating the automation package with name '{}':{}}.", packageName, packageId); + log.error("Error while updating the automation package with name '{}':{}}.", packageName, packageId, e); handleAutomationPackageDeploymentErrors(parameters, packageName, packageId, packageResource.get(), libraryResource.get()); throw (e instanceof AutomationPackageManagerException) ? (AutomationPackageManagerException) e: new AutomationPackageManagerException("Unable to update the automation package '" + packageName + "':" + packageId + ".", e, true); } finally { diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java index 47b6ccd7b..c10351684 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/AutomationPackageResourceManager.java @@ -189,6 +189,10 @@ public Resource uploadOrReuseApResource(AutomationPackageArchiveProvider apProvi String origin = Optional.ofNullable(apProvider.getOrigin()).map(ResourceOrigin::toStringRepresentation).orElse(null); File originalFile = automationPackageArchive.getOriginalFile(); if (originalFile == null) { + // When running an AP locally using the JUnit runner, we do not set the automation package resource + // This causes the methods that require it (like AbstractKeyword#retrieveAndExtractAutomationPackage) to fail + // In the future we should support this properly. An idea would e to use the ClassLoaderArchiver to create an + // archive out of the classloader. return null; } From 5eff1e90b72cd8977f839b5b6ad22430d81e21bc Mon Sep 17 00:00:00 2001 From: Jonathan Rubiero <30461894+rubij@users.noreply.github.com> Date: Tue, 9 Dec 2025 09:09:16 +0100 Subject: [PATCH 07/31] SED-4413 added connect and read timeouts to AbstractRemoteClient (#563) * SED-4413 added connect and read timeouts to AbstractRemoteClient * SED-4413 PR feedback (cherry picked from commit 984f98efb3324ee2c339eb48027e21eb25f28f0b) --- .../src/main/java/step/client/AbstractRemoteClient.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java b/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java index 345eadc28..82dba2b75 100644 --- a/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java +++ b/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; public class AbstractRemoteClient implements Closeable { @@ -72,7 +73,10 @@ public AbstractRemoteClient(){ } private void createClient() { - client = ClientBuilder.newClient(); + client = ClientBuilder.newBuilder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .build(); client.register(JacksonMapperProvider.class); client.register(MultiPartFeature.class); client.register(JacksonFeature.class); From 1ff5a3b0acd3de91b093de33042efc08d6db8c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Wed, 10 Dec 2025 14:01:27 +0100 Subject: [PATCH 08/31] SED-4401 Application context of step-functions-handler-initializer.jar not released (#567) * SED-4401 Application context of step-functions-handler-initializer.jar not released * SED-4413 added connect and read timeouts to AbstractRemoteClient (#563) * SED-4413 added connect and read timeouts to AbstractRemoteClient * SED-4413 PR feedback (cherry picked from commit 984f98efb3324ee2c339eb48027e21eb25f28f0b) * SED-4401 Application context of step-functions-handler-initializer.jar not released * Revert "SED-4413 added connect and read timeouts to AbstractRemoteClient (#563)" This reverts commit 5eff1e90b72cd8977f839b5b6ad22430d81e21bc. * SED-4401 Fixing NoClassDefFound * SED-4401 Fixing NPE --------- Co-authored-by: Jonathan Rubiero <30461894+rubij@users.noreply.github.com> --- .../step/client/AbstractRemoteClient.java | 6 +---- .../handler/FunctionMessageHandler.java | 27 ++++++++++++++----- 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java b/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java index 82dba2b75..345eadc28 100644 --- a/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java +++ b/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.function.Supplier; public class AbstractRemoteClient implements Closeable { @@ -73,10 +72,7 @@ public AbstractRemoteClient(){ } private void createClient() { - client = ClientBuilder.newBuilder() - .connectTimeout(10, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .build(); + client = ClientBuilder.newClient(); client.register(JacksonMapperProvider.class); client.register(MultiPartFeature.class); client.register(JacksonFeature.class); diff --git a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java index 12939e1eb..1f8ae0f3b 100644 --- a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java +++ b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java @@ -29,7 +29,9 @@ import step.grid.agent.handler.AbstractMessageHandler; import step.grid.agent.handler.context.OutputMessageBuilder; import step.grid.agent.tokenpool.AgentTokenWrapper; +import step.grid.agent.tokenpool.TokenReservationSession; import step.grid.contextbuilder.ApplicationContextBuilder; +import step.grid.contextbuilder.ApplicationContextControl; import step.grid.contextbuilder.LocalResourceApplicationContextFactory; import step.grid.contextbuilder.RemoteApplicationContextFactory; import step.grid.filemanager.FileVersionId; @@ -145,7 +147,7 @@ public OutputMessage handle(AgentTokenWrapper token, InputMessage inputMessage) JavaType javaType = mapper.getTypeFactory().constructParametricType(Input.class, functionHandler.getInputPayloadClass()); Input input = mapper.readValue(mapper.treeAsTokens(inputMessage.getPayload()), javaType); - LiveReporting liveReporting = initializeLiveReporting(input.getProperties()); + LiveReporting liveReporting = initializeLiveReporting(input.getProperties(), token.getTokenReservationSession()); functionHandler.setLiveReporting(liveReporting); // Handle the input @@ -176,8 +178,10 @@ public OutputMessage handle(AgentTokenWrapper token, InputMessage inputMessage) }); } - private LiveReporting initializeLiveReporting(Map properties) throws Exception { - applicationContextBuilder.pushContext(BRANCH_HANDLER_INITIALIZER, new LocalResourceApplicationContextFactory(this.getClass().getClassLoader(), "step-functions-handler-initializer.jar"), true); + private LiveReporting initializeLiveReporting(Map properties, TokenReservationSession tokenReservationSession) throws Exception { + ApplicationContextControl applicationContextControl = applicationContextBuilder.pushContext(BRANCH_HANDLER_INITIALIZER, new LocalResourceApplicationContextFactory(this.getClass().getClassLoader(), "step-functions-handler-initializer.jar"), true); + // The usage of this application context will be released when the session is closed, underlying registered file won't be cleanable before this release happens + tokenReservationSession.registerObjectToBeClosedWithSession(applicationContextControl); return applicationContextBuilder.runInContext(BRANCH_HANDLER_INITIALIZER, () -> { // There's no easy way to do this in the AbstractFunctionHandler itself, because // the only place where the Input properties are guaranteed to be available is in the (abstract) @@ -250,15 +254,24 @@ private Map getMergedAgentProperties(AgentTokenWrapper token) { @Override public void close() throws Exception { + Object webSocketContainer = webSocketContainerRef.getAndSet(null); + if(webSocketContainer != null) { + // The stop method of the websocket container has to be closed within its own context class loader + ClassLoader previousCl = Thread.currentThread().getContextClassLoader(); + Class webSocketContainerClass = webSocketContainer.getClass(); + Thread.currentThread().setContextClassLoader(webSocketContainerClass.getClassLoader()); + try { + webSocketContainerClass.getMethod("stop").invoke(webSocketContainer); + } finally { + Thread.currentThread().setContextClassLoader(previousCl); + } + } + if (applicationContextBuilder != null) { applicationContextBuilder.close(); } if (liveReportingExecutor != null) { liveReportingExecutor.shutdownNow(); } - Object webSocketContainer = webSocketContainerRef.getAndSet(null); - if (webSocketContainer != null) { - webSocketContainer.getClass().getMethod("stop").invoke(webSocketContainer); - } } } From 2f8e20ea953be52401218b16b7d1c4140e5d28aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Wed, 10 Dec 2025 14:11:25 +0100 Subject: [PATCH 09/31] SED-4413 Re-added connect and read timeouts to AbstractRemoteClient after merge of SED-4401 (#563) --- .../src/main/java/step/client/AbstractRemoteClient.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java b/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java index 345eadc28..82dba2b75 100644 --- a/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java +++ b/step-controller/step-controller-remote-client/src/main/java/step/client/AbstractRemoteClient.java @@ -40,6 +40,7 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.function.Supplier; public class AbstractRemoteClient implements Closeable { @@ -72,7 +73,10 @@ public AbstractRemoteClient(){ } private void createClient() { - client = ClientBuilder.newClient(); + client = ClientBuilder.newBuilder() + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .build(); client.register(JacksonMapperProvider.class); client.register(MultiPartFeature.class); client.register(JacksonFeature.class); From 732a312ee3cb704fd9f1c218a86b56840c3b00de Mon Sep 17 00:00:00 2001 From: David Stephan Date: Wed, 10 Dec 2025 20:57:00 +0100 Subject: [PATCH 10/31] =?UTF-8?q?SED-4439=20Optimize=20reporting=20request?= =?UTF-8?q?=20and=20query=20performance=20for=20Step=2029=E2=80=A6=20(#569?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SED-4439 Optimize reporting request and query performance for Step 29 branch --- .../src/main/java/step/core/Constants.java | 2 +- .../AggregatedReportViewBuilder.java | 29 +++++++++++++++---- .../aggregated/ReportNodeTimeSeries.java | 2 +- .../TimeSeriesCollectionsBuilder.java | 8 +++-- .../TimeSeriesCollectionsSettings.java | 2 +- 5 files changed, 32 insertions(+), 11 deletions(-) diff --git a/step-core/src/main/java/step/core/Constants.java b/step-core/src/main/java/step/core/Constants.java index 1da256f2a..3762b32ae 100644 --- a/step-core/src/main/java/step/core/Constants.java +++ b/step-core/src/main/java/step/core/Constants.java @@ -19,7 +19,7 @@ package step.core; public interface Constants { - String STEP_API_VERSION_STRING = "3.29.0"; + String STEP_API_VERSION_STRING = "3.29.1"; Version STEP_API_VERSION = new Version(STEP_API_VERSION_STRING); String STEP_YAML_SCHEMA_VERSION_STRING = "1.2.0"; diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java index 41db30a91..6b9a5f521 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/AggregatedReportViewBuilder.java @@ -25,6 +25,7 @@ import step.core.timeseries.bucket.BucketBuilder; import java.util.*; +import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -112,9 +113,17 @@ public AggregatedReport buildAggregatedReport(AggregatedReportViewRequest reques } } // Generate complete aggregated report tree - //First aggregate time series data for the given execution context grouped by artefactHash - Map> countByHashAndStatus = mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS); - Map> countByHashAndErrorMessage = mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE); + //First aggregate time series data for the given execution context grouped by artefactHash and status as wells as artefactHash and error message + CompletableFuture>> countByHashAndStatusFuture = + CompletableFuture.supplyAsync(() -> + mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS) + ); + CompletableFuture>> countByHashAndErrorMessageFuture = + CompletableFuture.supplyAsync(() -> + mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE) + ); + Map> countByHashAndStatus = countByHashAndStatusFuture.join(); + Map> countByHashAndErrorMessage = countByHashAndErrorMessageFuture.join(); //Because the time series time range extends to the time series resolution we need to use the same range when querying the report nodes Range resolvedRange = getResolvedRange(request, countByHashAndStatus); return new AggregatedReport(recursivelyBuildAggregatedReportTree(rootResolvedPlanNode, request, countByHashAndStatus, countByHashAndErrorMessage, mainReportNodeAccessor, null, runningCountByArtefactHash, operationsByArtefactHash, resolvedRange)); @@ -130,9 +139,17 @@ public AggregatedReport buildAggregatedReport(AggregatedReportViewRequest reques Set reportArtefactHashSet = buildPartialReportNodeTimeSeries(aggregatedReport, request.selectedReportNodeId, partialReportNodesTimeSeries, inMemoryReportNodeAccessor, runningCountByArtefactHash, operationsByArtefactHash, request.fetchCurrentOperations); // Only pass the reportArtefactHashSet if aggregate view filtering is enabled reportArtefactHashSet = (request.filterResolvedPlanNodes) ? reportArtefactHashSet : null; - //Aggregate time series data for the given execution reporting context grouped by artefactHash - Map> countByHashAndStatus = partialReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS); - Map> countByHashAndErrorMessage = mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE); + //Aggregate time series data for the given execution reporting context grouped by artefactHash and status ans artefactHash and error messages + CompletableFuture>> countByHashAndStatusFuture = + CompletableFuture.supplyAsync(() -> + partialReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, STATUS) + ); + CompletableFuture>> countByHashAndErrorMessageFuture = + CompletableFuture.supplyAsync(() -> + mainReportNodesTimeSeries.queryByExecutionIdAndGroupBy(executionId, request.range, ARTEFACT_HASH, ERROR_MESSAGE) + ); + Map> countByHashAndStatus = countByHashAndStatusFuture.join(); + Map> countByHashAndErrorMessage = countByHashAndErrorMessageFuture.join(); //Because the time series time range extends to the time series resolution we need to use the same range when querying the report nodes Range resolvedRange = getResolvedRange(request, countByHashAndStatus); aggregatedReport.aggregatedReportView = recursivelyBuildAggregatedReportTree(rootResolvedPlanNode, request, countByHashAndStatus, countByHashAndErrorMessage, inMemoryReportNodeAccessor, reportArtefactHashSet, runningCountByArtefactHash, operationsByArtefactHash, resolvedRange); diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java index 095f33229..11358d3ee 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java @@ -99,7 +99,7 @@ public Map> queryByExecutionIdAndGroupBy(String exec Filter filter = Filters.equals("attributes." + EXECUTION_ID, executionId); Set groupBy = Set.of(groupLevel1, groupLevel2); TimeSeriesAggregationQueryBuilder queryBuilder = new TimeSeriesAggregationQueryBuilder() - .withOptimizationType(TimeSeriesOptimizationType.MOST_ACCURATE) + .withOptimizationType(TimeSeriesOptimizationType.MOST_EFFICIENT) .withFilter(filter) .withGroupDimensions(groupBy) .split(1); diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index 9db714c8a..5eb8215e0 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -19,6 +19,8 @@ package step.core.timeseries; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import step.core.collections.CollectionFactory; import step.core.timeseries.bucket.Bucket; @@ -29,6 +31,8 @@ public class TimeSeriesCollectionsBuilder { + private static final Logger logger = LoggerFactory.getLogger(TimeSeriesCollectionsBuilder.class); + public static final String TIME_SERIES_SUFFIX_PER_MINUTE = "_minute"; public static final String TIME_SERIES_SUFFIX_HOURLY = "_hour"; public static final String TIME_SERIES_SUFFIX_DAILY = "_day"; @@ -63,8 +67,8 @@ private void addIfEnabled(List enabledCollections, String if (enabled) { enabledCollections.add(collection); } else { - // disabled resolutions will be completely dropped from db - collection.drop(); + // disabled resolutions are not dropped automatically + logger.warn("The time-series resolution with name '{}' is disabled. To reclaim space you can delete the corresponding DB table.", collectionName); } } } diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java index cab6c2c05..de8d2cf70 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java @@ -167,7 +167,7 @@ public TimeSeriesCollectionsSettings setWeeklyFlushInterval(long weeklyFlushInte } public static TimeSeriesCollectionsSettings readSettings(Configuration configuration, String collectionName) { - long mainResolution = getPropertyAsLong(configuration, TIME_SERIES_MAIN_RESOLUTION, collectionName, 1000L); + long mainResolution = getPropertyAsLong(configuration, TIME_SERIES_MAIN_RESOLUTION, collectionName, 5000L); validateMainResolutionParam(mainResolution); return new TimeSeriesCollectionsSettings() .setMainResolution(mainResolution) From 480f1f9e564774b0b66a47959a348bdb072a0d5c Mon Sep 17 00:00:00 2001 From: David Stephan Date: Thu, 11 Dec 2025 08:35:55 +0100 Subject: [PATCH 11/31] SED-4387 from a project it is possible to use bulk deletion to delete resources of a global project (#568) * SED-4387 delete resource services do not check write access for tenant * SED-4387 improve error message displyed in the UI --- .../main/java/step/resources/ResourceServices.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java b/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java index e4f35b386..dac90d182 100644 --- a/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java +++ b/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java @@ -152,6 +152,8 @@ public Response getResourceContent(@PathParam("id") String resourceId, @QueryPar @Path("/{id}") @Secured(right = "resource-delete") public void deleteResource(@PathParam("id") String resourceId) { + Resource resource = resourceManager.getResource(resourceId); + assertEntityIsEditableInContext(resource); resourceManager.deleteResource(resourceId); } @@ -160,6 +162,8 @@ public void deleteResource(@PathParam("id") String resourceId) { @Path("/{id}/revisions") @Secured(right = "resource-delete") public void deleteResourceRevisions(@PathParam("id") String resourceId) { + Resource resource = resourceManager.getResource(resourceId); + assertEntityIsEditableInContext(resource); resourceManager.deleteResourceRevisionContent(resourceId); } @@ -171,8 +175,12 @@ public AsyncTaskStatus bulkDelete(TableBulkOperationRe Consumer consumer = t -> { try { deleteResource(t); - } catch (Exception e) { - throw new RuntimeException(e); + } catch (Throwable e) { + if (e instanceof RuntimeException) { + throw e; + } else { + throw new RuntimeException(e); + } } }; return scheduleAsyncTaskWithinSessionContext(h -> From 1a1506a651b7fe9af3fcccb852b7bc0000945bdd Mon Sep 17 00:00:00 2001 From: David Stephan Date: Thu, 11 Dec 2025 14:41:49 +0100 Subject: [PATCH 12/31] SED-4444 timeseries-re-ingestion-doesnt-work-for-psql (#570) * SED-4444 timeseries-re-ingestion-doesnt-work-for-psql * SED-4444 timeseries-re-ingestion-doesnt-work-for-psql --- pom.xml | 2 +- .../step/core/controller/StepControllerPlugin.java | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0269fb995..b39e27a50 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2.5.0 + 2025.12.11-693aaad2a52e533162cf65cb 3.0.23 diff --git a/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java b/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java index ed5708faf..8cab9a8a8 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java +++ b/step-controller/step-controller-server/src/main/java/step/core/controller/StepControllerPlugin.java @@ -3,8 +3,10 @@ import ch.exense.commons.app.Configuration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import step.controller.services.async.AsyncTaskManager; import step.core.Controller; import step.core.GlobalContext; +import step.core.artefacts.reports.aggregated.ReportNodeTimeSeries; import step.core.controller.errorhandling.ErrorFilter; import step.core.deployment.ControllerServices; import step.core.execution.model.Execution; @@ -124,6 +126,14 @@ public void recover(GlobalContext context) throws Exception { @Override public void finalizeStart(GlobalContext context) throws Exception { context.require(ExecutionScheduler.class).start(); + //Initialize new empty resolutions after start (require the async task manager) + //Because the ReportNodeTimeSeries is created in ControllerServer.init directly and not in a plugin, this the right place to do it + ReportNodeTimeSeries reportNodeTimeSeries = context.require(ReportNodeTimeSeries.class); + AsyncTaskManager asyncTaskManager = context.require(AsyncTaskManager.class); + asyncTaskManager.scheduleAsyncTask((empty) -> { + reportNodeTimeSeries.getTimeSeries().ingestDataForEmptyCollections(); + return null; + }); } @Override From 0790bc5dfee96fad05b58be465a3fd17b6bd8c49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Thu, 11 Dec 2025 15:56:33 +0100 Subject: [PATCH 13/31] SED-4443 ClassCastException in FunctionMessageHandler (#572) * SED-4443 Fixing ClassCastException in FunctionMessageHandler * SED-4443 Fixing ClassCastException in FunctionMessageHandler --- .../FileApplicationContextFactory.java | 61 +++++++++++++++++++ .../handler/FunctionMessageHandler.java | 13 +++- 2 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java diff --git a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java new file mode 100644 index 000000000..6cb7cc1fc --- /dev/null +++ b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) 2025, exense GmbH + * + * This file is part of Step + * + * Step is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Step is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with Step. If not, see . + */ + +package step.functions.handler; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import step.grid.contextbuilder.ApplicationContextFactory; +import step.grid.contextbuilder.ClassPathHelper; + +import java.io.File; +import java.net.URL; +import java.net.URLClassLoader; +import java.util.List; + +public class FileApplicationContextFactory extends ApplicationContextFactory { + + private static final Logger logger = LoggerFactory.getLogger(FileApplicationContextFactory.class); + + private final File jar; + + public FileApplicationContextFactory(File jar) { + this.jar = jar; + } + + public String getId() { + return this.jar.getAbsolutePath(); + } + + public boolean requiresReload() { + return false; + } + + public ClassLoader buildClassLoader(ClassLoader parentClassLoader) { + if (logger.isDebugEnabled()) { + logger.debug("Creating URLClassLoader from extracted local file {}", this.jar.getAbsolutePath()); + } + List urls = ClassPathHelper.forSingleFile(this.jar); + URL[] urlArray = urls.toArray(new URL[urls.size()]); + return new URLClassLoader(urlArray, parentClassLoader); + } + + public void onClassLoaderClosed() { + } +} diff --git a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java index 1f8ae0f3b..c4b4c6e36 100644 --- a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java +++ b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java @@ -30,9 +30,9 @@ import step.grid.agent.handler.context.OutputMessageBuilder; import step.grid.agent.tokenpool.AgentTokenWrapper; import step.grid.agent.tokenpool.TokenReservationSession; +import step.grid.bootstrap.ResourceExtractor; import step.grid.contextbuilder.ApplicationContextBuilder; import step.grid.contextbuilder.ApplicationContextControl; -import step.grid.contextbuilder.LocalResourceApplicationContextFactory; import step.grid.contextbuilder.RemoteApplicationContextFactory; import step.grid.filemanager.FileVersionId; import step.grid.io.InputMessage; @@ -41,8 +41,10 @@ import step.livereporting.client.LiveReportingClient; import step.reporting.LiveReporting; +import java.io.File; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; +import java.nio.file.Files; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -71,6 +73,7 @@ public class FunctionMessageHandler extends AbstractMessageHandler { private ApplicationContextBuilder applicationContextBuilder; public FunctionHandlerFactory functionHandlerFactory; + private File functionHandlerInitializerJar; public FunctionMessageHandler() { super(); @@ -115,6 +118,7 @@ public void init(AgentTokenServices agentTokenServices) { applicationContextBuilder.forkCurrentContext(BRANCH_HANDLER_INITIALIZER); functionHandlerFactory = new FunctionHandlerFactory(applicationContextBuilder, agentTokenServices.getFileManagerClient()); + functionHandlerInitializerJar = ResourceExtractor.extractResource(this.getClass().getClassLoader(), "step-functions-handler-initializer.jar"); } @Override @@ -179,8 +183,8 @@ public OutputMessage handle(AgentTokenWrapper token, InputMessage inputMessage) } private LiveReporting initializeLiveReporting(Map properties, TokenReservationSession tokenReservationSession) throws Exception { - ApplicationContextControl applicationContextControl = applicationContextBuilder.pushContext(BRANCH_HANDLER_INITIALIZER, new LocalResourceApplicationContextFactory(this.getClass().getClassLoader(), "step-functions-handler-initializer.jar"), true); - // The usage of this application context will be released when the session is closed, underlying registered file won't be cleanable before this release happens + ApplicationContextControl applicationContextControl = applicationContextBuilder.pushContext(BRANCH_HANDLER_INITIALIZER, new FileApplicationContextFactory(functionHandlerInitializerJar), false); + // The usage of this application context will be released when the session is closed tokenReservationSession.registerObjectToBeClosedWithSession(applicationContextControl); return applicationContextBuilder.runInContext(BRANCH_HANDLER_INITIALIZER, () -> { // There's no easy way to do this in the AbstractFunctionHandler itself, because @@ -273,5 +277,8 @@ public void close() throws Exception { if (liveReportingExecutor != null) { liveReportingExecutor.shutdownNow(); } + if (functionHandlerInitializerJar != null) { + Files.deleteIfExists(functionHandlerInitializerJar.toPath()); + } } } From 6098a471d84dcd166ea61083c2ff68b8dcf05210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 12 Dec 2025 09:05:03 +0100 Subject: [PATCH 14/31] SED-4443 Switching to explicit creation of initializer CL (#574) --- .../FileApplicationContextFactory.java | 2 +- .../handler/FunctionMessageHandler.java | 35 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java index 6cb7cc1fc..979e009d8 100644 --- a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java +++ b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FileApplicationContextFactory.java @@ -47,7 +47,7 @@ public boolean requiresReload() { return false; } - public ClassLoader buildClassLoader(ClassLoader parentClassLoader) { + public URLClassLoader buildClassLoader(ClassLoader parentClassLoader) { if (logger.isDebugEnabled()) { logger.debug("Creating URLClassLoader from extracted local file {}", this.jar.getAbsolutePath()); } diff --git a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java index c4b4c6e36..f4bd3dd0c 100644 --- a/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java +++ b/step-functions/step-functions-handler/src/main/java/step/functions/handler/FunctionMessageHandler.java @@ -44,15 +44,13 @@ import java.io.File; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Proxy; +import java.net.URLClassLoader; import java.nio.file.Files; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; public class FunctionMessageHandler extends AbstractMessageHandler { @@ -62,7 +60,6 @@ public class FunctionMessageHandler extends AbstractMessageHandler { public static final String FUNCTION_HANDLER_KEY = "$functionhandler"; public static final String FUNCTION_TYPE_KEY = "$functionType"; - public static final String BRANCH_HANDLER_INITIALIZER = "handler-initializer"; // Cached object mapper for message payload serialization private final ObjectMapper mapper; @@ -74,6 +71,7 @@ public class FunctionMessageHandler extends AbstractMessageHandler { public FunctionHandlerFactory functionHandlerFactory; private File functionHandlerInitializerJar; + private URLClassLoader functionHandlerInitializerClassloader; public FunctionMessageHandler() { super(); @@ -115,10 +113,10 @@ public void init(AgentTokenServices agentTokenServices) { agentTokenServices.getApplicationContextBuilder().getApplicationContextConfiguration()); applicationContextBuilder.forkCurrentContext(AbstractFunctionHandler.FORKED_BRANCH); - applicationContextBuilder.forkCurrentContext(BRANCH_HANDLER_INITIALIZER); functionHandlerFactory = new FunctionHandlerFactory(applicationContextBuilder, agentTokenServices.getFileManagerClient()); functionHandlerInitializerJar = ResourceExtractor.extractResource(this.getClass().getClassLoader(), "step-functions-handler-initializer.jar"); + functionHandlerInitializerClassloader = new FileApplicationContextFactory(functionHandlerInitializerJar).buildClassLoader(this.getClass().getClassLoader()); } @Override @@ -182,11 +180,23 @@ public OutputMessage handle(AgentTokenWrapper token, InputMessage inputMessage) }); } + public T runInContext(ClassLoader classLoader, Callable runnable) throws Exception { + ClassLoader previousCl = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(classLoader); + + Object var4; + try { + var4 = runnable.call(); + } finally { + Thread.currentThread().setContextClassLoader(previousCl); + } + + return (T)var4; + } + private LiveReporting initializeLiveReporting(Map properties, TokenReservationSession tokenReservationSession) throws Exception { - ApplicationContextControl applicationContextControl = applicationContextBuilder.pushContext(BRANCH_HANDLER_INITIALIZER, new FileApplicationContextFactory(functionHandlerInitializerJar), false); - // The usage of this application context will be released when the session is closed - tokenReservationSession.registerObjectToBeClosedWithSession(applicationContextControl); - return applicationContextBuilder.runInContext(BRANCH_HANDLER_INITIALIZER, () -> { + + return runInContext(functionHandlerInitializerClassloader, () -> { // There's no easy way to do this in the AbstractFunctionHandler itself, because // the only place where the Input properties are guaranteed to be available is in the (abstract) // handle() method (which would then have to be implemented in all subclasses). So we do it here. @@ -204,7 +214,7 @@ private LiveReporting initializeLiveReporting(Map properties, To liveReportingClientClass.getClassLoader(), new Class[]{LiveReportingClient.class}, (proxy, method, args) -> { try { - return applicationContextBuilder.runInContext(BRANCH_HANDLER_INITIALIZER, () -> method.invoke(liveReportingClient, args)); + return runInContext(functionHandlerInitializerClassloader, () -> method.invoke(liveReportingClient, args)); } catch (InvocationTargetException ite) { // rethrow the original exception instead of InvocationTargetException, (usually) conforming to the method's throws signature unless it's a RuntimeException or similar throw ite.getCause(); @@ -277,6 +287,9 @@ public void close() throws Exception { if (liveReportingExecutor != null) { liveReportingExecutor.shutdownNow(); } + if (functionHandlerInitializerClassloader != null) { + functionHandlerInitializerClassloader.close(); + } if (functionHandlerInitializerJar != null) { Files.deleteIfExists(functionHandlerInitializerJar.toPath()); } From 38ea1cf9f63ad38e62002727ef464d4fa93d58d2 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 12 Dec 2025 10:17:47 +0100 Subject: [PATCH 15/31] SED-4445 bumping framework to 2.5.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b39e27a50..4b5649d10 100644 --- a/pom.xml +++ b/pom.xml @@ -47,7 +47,7 @@ 2025.6.25 2.5.0 - 2025.12.11-693aaad2a52e533162cf65cb + 2.5.1 3.0.23 From 2407ee238bd5a92c7d41b140c7f6f182c216ca33 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 19 Dec 2025 10:34:44 +0100 Subject: [PATCH 16/31] SED-4418 Calling Keywords in an after section does not release tokens (#575) * SED-4418 Calling Keywords in an after section does not release tokens * SED-4418 Calling Keywords in an after section does not release tokens * SED-4418 Adding a junit test --- .../handlers/FunctionGroupHandler.java | 10 +++++++- .../handlers/AbstractFunctionHandlerTest.java | 6 +++++ .../handlers/FunctionGroupHandlerTest.java | 24 +++++++++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionGroupHandler.java b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionGroupHandler.java index ab6658813..d60df1072 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionGroupHandler.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionGroupHandler.java @@ -114,7 +114,11 @@ protected void execute_(ReportNode node, FunctionGroup functionGroup) throws Exc consumer.accept(functionGroup, node); } } finally { - releaseTokens(context, true); + try { + releaseTokens(context, true); + } finally { + cleanupFunctionGroupContextFromContext(node); + } } } @@ -123,6 +127,10 @@ private void addFunctionGroupContextToContext(ReportNode node, FunctionGroupCont context.put(FunctionGroupHandle.class, this); } + private void cleanupFunctionGroupContextFromContext(ReportNode node) { + context.getVariablesManager().removeVariable(node, FUNCTION_GROUP_CONTEXT_KEY); + } + private FunctionGroupContext buildFunctionGroupContext(FunctionExecutionService functionExecutionService, FunctionGroup functionGroup) { Map additionalSelectionCriteria = tokenSelectorHelper.getTokenSelectionCriteria(functionGroup, getBindings()); return new FunctionGroupContext(functionExecutionService, additionalSelectionCriteria); diff --git a/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/AbstractFunctionHandlerTest.java b/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/AbstractFunctionHandlerTest.java index 1134a9fcd..d45044255 100644 --- a/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/AbstractFunctionHandlerTest.java +++ b/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/AbstractFunctionHandlerTest.java @@ -81,6 +81,12 @@ protected static void getLocalAndRemoteTokenFromSession(ExecutionContext t) { t.getCurrentReportNode().setStatus(ReportNodeStatus.PASSED); } + protected static void validateIsInSession(ExecutionContext t) { + if (t.getVariablesManager().getVariable(FunctionGroupHandler.FUNCTION_GROUP_CONTEXT_KEY) == null) { + throw new RuntimeException("Not is a session"); + } + } + private static void getLocalAndRemoteToken(FunctionGroupSession session) { session.getLocalToken(); try { diff --git a/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/FunctionGroupHandlerTest.java b/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/FunctionGroupHandlerTest.java index ee7037727..c6f50935d 100644 --- a/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/FunctionGroupHandlerTest.java +++ b/step-plans/step-plans-base-artefacts/src/test/java/step/artefacts/handlers/FunctionGroupHandlerTest.java @@ -184,4 +184,28 @@ public void executionFinally(ExecutionContext context) { assertEquals(REMOTE_URL, executionRef.get().getAgentsInvolved()); } + + /** + * this test verifies that the session group context of a session control is cleaned up before executing its after section + * @throws IOException + * @throws ExecutionEngineException + */ + @Test + public void testSessionAfterSection() throws IOException, ExecutionEngineException { + Plan plan = PlanBuilder.create().startBlock(new FunctionGroup()).withAfter(new CheckArtefact(FunctionGroupHandlerTest::validateIsInSession)).add(new CheckArtefact(FunctionGroupHandlerTest::getLocalAndRemoteTokenFromSession)).add(new Echo()).endBlock().build(); + + StringWriter writer = new StringWriter(); + try (ExecutionEngine engine = newEngineWithCustomTokenReleaseFunction(this::markTokenAsReleased, null)) { + engine.execute(plan).printTree(writer); + } + + // Assert that the token have been returned after Session execution + assertThatLocalAndRemoteTokenHaveBeenReleased(); + assertEquals("Session:TECHNICAL_ERROR:\n" + + " CheckArtefact:PASSED:\n" + + " Echo:PASSED:\n" + + " [AFTER]\n" + + " CheckArtefact:TECHNICAL_ERROR:Not is a session\n", writer.toString()); + } + } From b5e41d7f51722c4f02c9f45735ccfbd172fcd7b7 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 19 Dec 2025 11:53:30 +0100 Subject: [PATCH 17/31] SED-4287 ForEach may try to use uninitialized tempWriter (#576) * SED-4287 ForEach may try to use uninitialized tempWriter * SED-4287 propagating exception --- .../step/datapool/file/CSVReaderDataPool.java | 94 +++++++++++++------ 1 file changed, 64 insertions(+), 30 deletions(-) diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/datapool/file/CSVReaderDataPool.java b/step-plans/step-plans-base-artefacts/src/main/java/step/datapool/file/CSVReaderDataPool.java index fa2d5eb1d..b0208350c 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/datapool/file/CSVReaderDataPool.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/datapool/file/CSVReaderDataPool.java @@ -27,6 +27,7 @@ import java.util.Iterator; import java.util.LinkedHashSet; import java.util.Set; +import java.util.UUID; import java.util.Vector; import java.util.concurrent.atomic.AtomicBoolean; @@ -51,6 +52,8 @@ public class CSVReaderDataPool extends FileReaderDataPool { // written to protected File tempFile; protected PrintWriter tempFileWriter; + // unique identifier for this instance to avoid temp file collisions + private final String instanceId = UUID.randomUUID().toString(); public CSVReaderDataPool(CSVDataPool configuration) { super(configuration); @@ -62,14 +65,22 @@ public void init() { super.init(); // Write operations to rows (RowWrapper.put) are written to a temporary file - // which - // overrides the initial file when the data pool is closed - tempFile = new File(filePath + ".tmp"); - try { + // which overrides the initial file when the data pool is closed + // Using a unique instance ID to avoid collisions between multiple threads/instances + tempFile = new File(filePath + ".tmp." + instanceId); + } + + /** + * Lazily initializes the tempFileWriter when first write operation occurs. + * This is synchronized to ensure thread-safety during initialization. + * If initialization fails, the error is logged and tempFileWriter remains null. + */ + private synchronized void initializeTempFileWriterIfRequired() throws IOException { + if (tempFileWriter == null) { tempFileWriter = new PrintWriter(new BufferedWriter(new FileWriter(tempFile))); // write headers to the temporary file - if (headers!=null) { + if (headers != null) { Iterator iterator = headers.iterator(); while (iterator.hasNext()) { String header = iterator.next(); @@ -80,8 +91,6 @@ public void init() { } } tempFileWriter.println(); - } catch (IOException e) { - logger.error("Error while creating temporary file " + tempFile.getAbsolutePath(), e); } } @@ -90,18 +99,35 @@ public void close() { super.close(); try { - tempFileWriter.close(); - // persist the changes if necessary - if (isWriteEnabled() && hasChanges.get()) { - // move the initial file - File initialFile = new File(filePath + ".initial"); - Files.move(new File(filePath), initialFile); - // replace the initial file by the temporary file containing the changes - Files.move(tempFile, new File(filePath)); - // delete the initial file - initialFile.delete(); + // Close the temp file writer if it was initialized + if (tempFileWriter != null) { + tempFileWriter.close(); + + // persist the changes if necessary + // Only attempt to move files if tempFileWriter was successfully initialized + if (isWriteEnabled() && hasChanges.get()) { + // Synchronize on the file path to prevent concurrent file operations + // from multiple instances using the same file + synchronized (filePath.intern()) { + // move the initial file + File initialFile = new File(filePath + ".initial"); + Files.move(new File(filePath), initialFile); + // replace the initial file by the temporary file containing the changes + Files.move(tempFile, new File(filePath)); + // delete the initial file + if (!initialFile.delete()) { + logger.error("Could not delete the CSV initial file {}", initialFile.getAbsolutePath()); + } + } + } + } + + // Clean up temp file if it still exists + if (tempFile.exists()) { + if (!tempFile.delete()) { + logger.error("Could not delete the CSV temporary file {}", tempFile.getAbsolutePath()); + } } - tempFile.delete(); } catch (IOException e) { logger.error("Error while closing the CSV dataset", e); } @@ -114,20 +140,28 @@ public void writeRow(DataPoolRow row) throws IOException { if(isWriteEnabled()) { Object value = row.getValue(); if (value != null && value instanceof CSVRowWrapper) { - CSVRowWrapper csvRow = (CSVRowWrapper) value; - - Iterator iterator = headers.iterator(); - while (iterator.hasNext()) { - String header = iterator.next(); - Object object = csvRow.rowData.get(header); - if(object != null) { - tempFileWriter.print(object.toString()); - } - if (iterator.hasNext()) { - tempFileWriter.print(delimiter); + // Lazily initialize the temp file writer on first write + initializeTempFileWriterIfRequired(); + + // Only write if initialization succeeded + if (tempFileWriter != null) { + CSVRowWrapper csvRow = (CSVRowWrapper) value; + + Iterator iterator = headers.iterator(); + while (iterator.hasNext()) { + String header = iterator.next(); + Object object = csvRow.rowData.get(header); + if(object != null) { + tempFileWriter.print(object.toString()); + } + if (iterator.hasNext()) { + tempFileWriter.print(delimiter); + } } + tempFileWriter.println(); + } else { + throw new RuntimeException("Cannot write row: temporary file writer could not be initialized for file " + filePath); } - tempFileWriter.println(); } } } From 174c63878cf80a1b5a8196709b8f7e8d6366e9c6 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Thu, 8 Jan 2026 14:00:19 +0100 Subject: [PATCH 18/31] SED-4340 find-usages-of-keyword-and-plan-referenced-by-attribute-is-broken --- .../step/core/references/ReferenceFinder.java | 124 ++++++++++++++ .../references/ReferenceFinderServices.java | 141 +--------------- .../java/step/core/GlobalContextBuilder.java | 28 ++-- .../core/references/ReferenceFinderTest.java | 151 ++++++++++++++++++ .../entities/EntityDependencyTreeVisitor.java | 32 ++-- .../step/core/entities/EntityManager.java | 3 +- .../EntityDependencyTreeVisitorTest.java | 2 +- .../packages/FunctionPackageEntity.java | 7 +- .../artefacts/handlers/FunctionLocator.java | 73 ++++++--- .../artefacts/handlers/LocatorHelper.java | 4 +- .../step/artefacts/handlers/PlanLocator.java | 59 ++++++- .../main/java/step/core/plans/PlanEntity.java | 26 ++- .../functions/accessor/FunctionEntity.java | 28 +++- .../step/core/scheduler/ScheduleEntity.java | 2 +- 14 files changed, 471 insertions(+), 209 deletions(-) create mode 100644 step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java create mode 100644 step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java new file mode 100644 index 000000000..52f84fe64 --- /dev/null +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java @@ -0,0 +1,124 @@ +package step.core.references; + +import step.core.accessors.AbstractOrganizableObject; +import step.core.entities.EntityConstants; +import step.core.entities.EntityDependencyTreeVisitor; +import step.core.entities.EntityManager; +import step.core.plans.Plan; +import step.core.plans.PlanAccessor; +import step.functions.Function; +import step.functions.accessor.FunctionAccessor; +import step.resources.Resource; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class ReferenceFinder { + + private final EntityManager entityManager; + + public ReferenceFinder(EntityManager entityManager) { + this.entityManager = entityManager; + } + + public List findReferences(FindReferencesRequest request) { + if (request.searchType == null) { + throw new IllegalArgumentException("A valid searchType must be provided"); + } + if (request.searchValue == null || request.searchValue.trim().isEmpty()) { + throw new IllegalArgumentException("A non-empty searchValue must be provided"); + } + + List results = new ArrayList<>(); + + PlanAccessor planAccessor = (PlanAccessor) entityManager.getEntityByName(EntityConstants.plans).getAccessor(); + + // Find composite keywords containing requested usages; composite KWs are really just plans in disguise :-) + FunctionAccessor functionAccessor = (FunctionAccessor) entityManager.getEntityByName(EntityConstants.functions).getAccessor(); + + try (Stream functionStream = functionAccessor.streamLazy()) { + functionStream.forEach(function -> { + List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.functions, function, request); + if (!matchingObjects.isEmpty()) { + results.add(new FindReferencesResponse(function)); + } + }); + } + + // Find plans containing usages + try (Stream stream = (request.includeHiddenPlans) ? planAccessor.streamLazy() : planAccessor.getVisiblePlans()) { + stream.forEach(plan -> { + List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.plans, plan, request); + if (!matchingObjects.isEmpty()) { + results.add(new FindReferencesResponse(plan)); + } + }); + } + + // Sort the results by name + results.sort(Comparator.comparing(f -> f.name)); + return results; + } + + private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) { + List referencedObjects = getReferencedObjects(entityType, object).stream().filter(o -> (o != null && !o.equals(object))).collect(Collectors.toList()); + //System.err.println("objects referenced from plan: " + planToString(plan) + ": "+ referencedObjects.stream().map(ReferenceFinderServices::objectToString).collect(Collectors.toList())); + return referencedObjects.stream().filter(o -> doesRequestMatch(request, o)).collect(Collectors.toList()); + } + + // returns a (generic) set of objects referenced by a plan + private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) { + Set referencedObjects = new HashSet<>(); + + // The references can be filled in three different ways due to the implementation: + // 1. through the predicate (just below) + // 2. by (actual object) reference in the tree visitor (onResolvedEntity) + + // No context predicate is used by the reference finder, since we want to find all entities (i.e. if we search the usages of a Keyword from the Common project, we should be able + // to find plans using it in other projects. + // This unfortunately can return incorrect results, i.e. a keyword "MyKeyword" is created in ProjectA and ProjectB, A PlanA is created in ProjectA and is using the KW of the same project. + // Searching usage of "MyKeyword" in projectB will return the planA from projectA + EntityDependencyTreeVisitor entityDependencyTreeVisitor = new EntityDependencyTreeVisitor(entityManager, o -> true); + FindReferencesTreeVisitor entityTreeVisitor = new FindReferencesTreeVisitor(entityManager, referencedObjects); + entityDependencyTreeVisitor.visitEntityDependencyTree(entityType, object.getId().toString(), entityTreeVisitor, EntityDependencyTreeVisitor.VISIT_MODE.RESOLVE_ALL); + + return referencedObjects; + } + + private boolean doesRequestMatch(FindReferencesRequest req, Object o) { + if (o instanceof Plan) { + Plan p = (Plan) o; + switch (req.searchType) { + case PLAN_NAME: + return p.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); + case PLAN_ID: + return p.getId().toString().equals(req.searchValue); + default: + return false; + } + } else if (o instanceof Function) { + Function f = (Function) o; + switch (req.searchType) { + case KEYWORD_NAME: + return f.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); + case KEYWORD_ID: + return f.getId().toString().equals(req.searchValue); + default: + return false; + } + } else if (o instanceof Resource) { + Resource r = (Resource) o; + switch (req.searchType) { + case RESOURCE_NAME: + return r.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); + case RESOURCE_ID: + return r.getId().toString().equals(req.searchValue); + default: + return false; + } + } else { + return false; + } + } +} diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java index 910960109..b6110c07c 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java @@ -28,155 +28,20 @@ @Tag(name = "References") public class ReferenceFinderServices extends AbstractStepServices { - private EntityManager entityManager; + private ReferenceFinder referenceFinder; @PostConstruct public void init() throws Exception { super.init(); - entityManager = getContext().getEntityManager(); + referenceFinder = new ReferenceFinder(getContext().getEntityManager()); } - - // Uncomment for easier debugging (poor man's Unit Test), URL will be http://localhost:8080/rest/references/findReferencesDebug - /* - @GET - @Path("/findReferencesDebug") - @Produces(MediaType.APPLICATION_JSON) - public List findReferencesTest() { - List result = new ArrayList<>(); - result.addAll(findReferences(new FindReferencesRequest(PLAN_NAME, "TestXXX"))); -// result.addAll(findReferences(new FindReferencesRequest(PLAN_ID, "6195001c0a98d92da8a57830"))); - result.addAll(findReferences(new FindReferencesRequest(KEYWORD_NAME, "UnitTest"))); -// result.addAll(findReferences(new FindReferencesRequest(KEYWORD_ID, "60cca3488b81b227a5fe92d9"))); - return result; - } - //*/ - @Path("/findReferences") @POST @Secured @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public List findReferences(FindReferencesRequest request) { - if (request.searchType == null) { - throw new IllegalArgumentException("A valid searchType must be provided"); - } - if (request.searchValue == null || request.searchValue.trim().isEmpty()) { - throw new IllegalArgumentException("A non-empty searchValue must be provided"); - } - - List results = new ArrayList<>(); - - PlanAccessor planAccessor = (PlanAccessor) entityManager.getEntityByName(EntityConstants.plans).getAccessor(); - - // Find composite keywords containing requested usages; composite KWs are really just plans in disguise :-) - FunctionAccessor functionAccessor = (FunctionAccessor) entityManager.getEntityByName(EntityConstants.functions).getAccessor(); - - try (Stream functionStream = functionAccessor.streamLazy()) { - functionStream.forEach(function -> { - List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.functions, function, request); - if (!matchingObjects.isEmpty()) { - results.add(new FindReferencesResponse(function)); - } - }); - } - - // Find plans containing usages - try (Stream stream = (request.includeHiddenPlans) ? planAccessor.streamLazy() : planAccessor.getVisiblePlans()) { - stream.forEach(plan -> { - List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.plans, plan, request); - if (!matchingObjects.isEmpty()) { - results.add(new FindReferencesResponse(plan)); - } - }); - } - - // Sort the results by name - results.sort(Comparator.comparing(f -> f.name)); - return results; - } - - private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) { - List referencedObjects = getReferencedObjects(entityType, object).stream().filter(o -> (o != null && !o.equals(object))).collect(Collectors.toList()); - //System.err.println("objects referenced from plan: " + planToString(plan) + ": "+ referencedObjects.stream().map(ReferenceFinderServices::objectToString).collect(Collectors.toList())); - return referencedObjects.stream().filter(o -> doesRequestMatch(request, o)).collect(Collectors.toList()); + return referenceFinder.findReferences(request); } - - // returns a (generic) set of objects referenced by a plan - private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) { - Set referencedObjects = new HashSet<>(); - - // The references can be filled in three different ways due to the implementation: - // 1. through the predicate (just below) - // 2. by (actual object) reference in the tree visitor (onResolvedEntity) - // 3. by object ID in the tree visitor (onResolvedEntityId) - - ObjectPredicate visitedObjectPredicate = visitedObject -> { - referencedObjects.add(visitedObject); - return true; - }; - - EntityDependencyTreeVisitor entityDependencyTreeVisitor = new EntityDependencyTreeVisitor(entityManager, visitedObjectPredicate); - FindReferencesTreeVisitor entityTreeVisitor = new FindReferencesTreeVisitor(entityManager, referencedObjects); - entityDependencyTreeVisitor.visitEntityDependencyTree(entityType, object.getId().toString(), entityTreeVisitor, false); - - return referencedObjects; - } - - private boolean doesRequestMatch(FindReferencesRequest req, Object o) { - if (o instanceof Plan) { - Plan p = (Plan) o; - switch (req.searchType) { - case PLAN_NAME: - return p.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); - case PLAN_ID: - return p.getId().toString().equals(req.searchValue); - default: - return false; - } - } else if (o instanceof Function) { - Function f = (Function) o; - switch (req.searchType) { - case KEYWORD_NAME: - return f.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); - case KEYWORD_ID: - return f.getId().toString().equals(req.searchValue); - default: - return false; - } - } else if (o instanceof Resource) { - Resource r = (Resource) o; - switch (req.searchType) { - case RESOURCE_NAME: - return r.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); - case RESOURCE_ID: - return r.getId().toString().equals(req.searchValue); - default: - return false; - } - } else { - return false; - } - } - - // the following functions are only needed for debugging - private static String objectToString(Object o) { - if (o instanceof Plan) { - return planToString((Plan) o); - } else if (o instanceof Function) { - return functionToString((Function) o); - } else { - return o.getClass() + " " + o.toString(); - } - } - - private static String planToString(Plan plan) { - return "PLAN: " + plan.getAttributes().toString() + " id=" + plan.getId().toString(); - } - - private static String functionToString(Function function) { - return "FUNCTION: " + function.getAttributes().toString() + " id=" + function.getId().toString(); - } - - } diff --git a/step-controller/step-controller-server/src/test/java/step/core/GlobalContextBuilder.java b/step-controller/step-controller-server/src/test/java/step/core/GlobalContextBuilder.java index f20b27a98..06cf7e72c 100644 --- a/step-controller/step-controller-server/src/test/java/step/core/GlobalContextBuilder.java +++ b/step-controller/step-controller-server/src/test/java/step/core/GlobalContextBuilder.java @@ -22,6 +22,9 @@ import ch.exense.commons.io.FileHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import step.artefacts.handlers.FunctionLocator; +import step.artefacts.handlers.PlanLocator; +import step.artefacts.handlers.SelectorHelper; import step.core.access.InMemoryUserAccessor; import step.core.access.User; import step.core.access.UserAccessor; @@ -30,6 +33,8 @@ import step.core.artefacts.reports.ReportNode; import step.core.artefacts.reports.ReportNodeAccessor; import step.core.dynamicbeans.DynamicBeanResolver; +import step.core.dynamicbeans.DynamicJsonObjectResolver; +import step.core.dynamicbeans.DynamicJsonValueResolver; import step.core.dynamicbeans.DynamicValueResolver; import step.core.entities.Entity; import step.core.entities.EntityConstants; @@ -40,17 +45,19 @@ import step.core.plans.InMemoryPlanAccessor; import step.core.plans.Plan; import step.core.plans.PlanAccessor; +import step.core.plans.PlanEntity; import step.core.plugins.ControllerPluginManager; import step.core.plugins.PluginManager.Builder.CircularDependencyException; import step.core.repositories.RepositoryObjectManager; import step.core.scheduler.ExecutionTaskAccessor; import step.core.scheduler.ExecutiontTaskParameters; import step.core.scheduler.InMemoryExecutionTaskAccessor; +import step.core.scheduler.ScheduleEntity; import step.expressions.ExpressionHandler; import step.framework.server.ServerPluginManager; import step.framework.server.tables.TableRegistry; -import step.functions.Function; import step.functions.accessor.FunctionAccessor; +import step.functions.accessor.FunctionEntity; import step.functions.accessor.InMemoryFunctionAccessorImpl; import step.resources.*; @@ -101,20 +108,23 @@ public static GlobalContext createGlobalContext() throws CircularDependencyExcep logger.error("Unable to create temp folder for the resource manager", e); } - context.setEntityManager(new EntityManager()); + DynamicJsonObjectResolver dynamicJsonObjectResolver = new DynamicJsonObjectResolver(new DynamicJsonValueResolver(context.getExpressionHandler())); + SelectorHelper selectorHelper = new SelectorHelper(dynamicJsonObjectResolver); + PlanLocator planLocator = new PlanLocator(context.getPlanAccessor(), selectorHelper); + FunctionLocator functionLocator = new FunctionLocator(functionAccessor, selectorHelper); + + EntityManager entityManager = new EntityManager(); + context.setEntityManager(entityManager); context.getEntityManager() .register(new Entity(EntityConstants.executions, context.getExecutionAccessor(), Execution.class)) - .register(new Entity(EntityConstants.plans, context.getPlanAccessor(), Plan.class)) + .register(new PlanEntity(context.getPlanAccessor(), planLocator, entityManager)) .register(new Entity(EntityConstants.reports, context.getReportAccessor(), ReportNode.class)) - .register(new Entity(EntityConstants.tasks, - context.getScheduleAccessor(), ExecutiontTaskParameters.class)) + .register(new ScheduleEntity(context.getScheduleAccessor(), ExecutiontTaskParameters.class, entityManager)) .register(new Entity(EntityConstants.users, context.getUserAccessor(), User.class)) - .register(new Entity(EntityConstants.functions, - (FunctionAccessor) functionAccessor, Function.class)) - .register(new Entity(EntityConstants.resources, resourceAccessor, - Resource.class)) + .register(new FunctionEntity(functionAccessor, functionLocator, entityManager)) + .register(new ResourceEntity(resourceAccessor, entityManager)) .register(new Entity(EntityConstants.resourceRevisions, resourceRevisionAccessor, ResourceRevision.class)); diff --git a/step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java b/step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java new file mode 100644 index 000000000..5f2e5fcfa --- /dev/null +++ b/step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java @@ -0,0 +1,151 @@ +package step.core.references; + +import org.junit.Before; +import org.junit.Test; +import step.artefacts.CallFunction; +import step.artefacts.CallPlan; +import step.artefacts.ForEachBlock; +import step.attachments.FileResolver; +import step.core.GlobalContext; +import step.core.accessors.AbstractOrganizableObject; +import step.core.dynamicbeans.DynamicValue; +import step.core.plans.Plan; +import step.core.plans.builder.PlanBuilder; +import step.core.plugins.PluginManager; +import step.datapool.file.CSVDataPool; +import step.functions.Function; +import step.functions.accessor.FunctionAccessor; +import step.planbuilder.BaseArtefacts; +import step.planbuilder.FunctionArtefacts; +import step.plugins.functions.types.CompositeFunction; +import step.resources.Resource; + +import java.io.IOException; +import java.util.List; + +import static org.junit.Assert.*; +import static step.core.GlobalContextBuilder.createGlobalContext; +import static step.resources.ResourceManager.RESOURCE_TYPE_DATASOURCE; + +public class ReferenceFinderTest { + + public static final String CALLING_PLAN_BY_ID = "CALLING PLAN BY ID"; + public static final String CALLING_PLAN_BY_NAME = "CALLING PLAN BY NAME"; + public static final String CALLED_PLAN_NAME = "CALLED PLAN"; + public static final String CALLED_FUNCTION_NAME = "CALLED FUNCTION"; + public static final String COMPOSITE_KEYWORD = "COMPOSITE_KEYWORD"; + public static final String CSV_FILE = "CSV FILE"; + public static final String PLAN_USING_RESOURCE = "PLAN_USING_RESOURCE"; + private GlobalContext context; + private ReferenceFinder referenceFinder; + private FunctionAccessor functionAccessor; + + @Before + public void setup() throws ClassNotFoundException, PluginManager.Builder.CircularDependencyException, InstantiationException, IllegalAccessException { + context = createGlobalContext(); + referenceFinder = new ReferenceFinder(context.getEntityManager()); + functionAccessor = context.require(FunctionAccessor.class); + } + + @Test + public void findReferencesForPlansAndKeyword() { + Plan calledPlan = PlanBuilder.create().startBlock(BaseArtefacts.sequence()).endBlock().build(); + calledPlan.addAttribute(AbstractOrganizableObject.NAME, CALLED_PLAN_NAME); + context.getPlanAccessor().save(calledPlan); + + Function function = new Function(); + function.addAttribute(AbstractOrganizableObject.NAME, CALLED_FUNCTION_NAME); + functionAccessor.save(function); + CallFunction callFunction = FunctionArtefacts.keyword(function.getAttribute(AbstractOrganizableObject.NAME), "{\"key1\":\"val1\"}"); + + //Plan calling another plan by ID + CallPlan callPlanById = new CallPlan(); + String calledPlanId = calledPlan.getId().toString(); + callPlanById.setPlanId(calledPlanId); + Plan planCallingPlanById = PlanBuilder.create().startBlock(BaseArtefacts.sequence()).add(callPlanById).add(callFunction).endBlock().build(); + planCallingPlanById.addAttribute(AbstractOrganizableObject.NAME, CALLING_PLAN_BY_ID); + context.getPlanAccessor().save(planCallingPlanById); + + //Plan calling another plan by attributes + CallPlan callPlanByName = new CallPlan(); + callPlanByName.setSelectionAttributes(new DynamicValue("{\"name\":\"" + CALLED_PLAN_NAME + "\"}")); + Plan planCallingPlanByName = PlanBuilder.create().startBlock(BaseArtefacts.sequence()).add(callPlanByName).add(callFunction).endBlock().build(); + planCallingPlanByName.addAttribute(AbstractOrganizableObject.NAME, CALLING_PLAN_BY_NAME); + context.getPlanAccessor().save(planCallingPlanByName); + + //Composite Keyword calling plan and keyword by name + CompositeFunction compositeFunction = new CompositeFunction(); + compositeFunction.addAttribute(AbstractOrganizableObject.NAME, COMPOSITE_KEYWORD); + compositeFunction.setPlan(planCallingPlanByName); + functionAccessor.save(compositeFunction); + + //Search usage by Plan ID + List findReferencesResponse = + referenceFinder.findReferences(new FindReferencesRequest(FindReferencesRequest.Type.PLAN_ID, calledPlanId)); + assertPlansAndCompositesAreFound(findReferencesResponse, planCallingPlanById, planCallingPlanByName, compositeFunction); + + //Search usage by Plan name + findReferencesResponse = + referenceFinder.findReferences(new FindReferencesRequest(FindReferencesRequest.Type.PLAN_NAME, CALLED_PLAN_NAME)); + assertPlansAndCompositesAreFound(findReferencesResponse, planCallingPlanById, planCallingPlanByName, compositeFunction); + + //Search Keyword By ID + findReferencesResponse = + referenceFinder.findReferences(new FindReferencesRequest(FindReferencesRequest.Type.KEYWORD_ID, function.getId().toString())); + assertPlansAndCompositesAreFound(findReferencesResponse, planCallingPlanById, planCallingPlanByName, compositeFunction); + + //Search Keyword By name + findReferencesResponse = + referenceFinder.findReferences(new FindReferencesRequest(FindReferencesRequest.Type.KEYWORD_NAME, CALLED_FUNCTION_NAME)); + assertPlansAndCompositesAreFound(findReferencesResponse, planCallingPlanById, planCallingPlanByName, compositeFunction); + + } + + private static void assertPlansAndCompositesAreFound(List findReferencesResponse, Plan planCallingPlanById, + Plan planCallingPlanByName, CompositeFunction compositeFunction) { + assertFirstResponseReference(3, findReferencesResponse, planCallingPlanById, CALLING_PLAN_BY_ID); + FindReferencesResponse findReferencesResponse2 = findReferencesResponse.get(1); + assertEquals(planCallingPlanByName.getId().toString(), findReferencesResponse2.id); + assertEquals(CALLING_PLAN_BY_NAME, findReferencesResponse2.name); + FindReferencesResponse findReferencesResponse3 = findReferencesResponse.get(2); + assertEquals(compositeFunction.getId().toString(), findReferencesResponse3.id); + assertEquals(COMPOSITE_KEYWORD, findReferencesResponse3.name); + } + + @Test + public void findReferencesForResources() throws IOException { + Resource resource = new Resource(); + resource.setResourceName(CSV_FILE); + resource.setResourceType(RESOURCE_TYPE_DATASOURCE); + context.getResourceManager().saveResource(resource); + + ForEachBlock f = new ForEachBlock(); + CSVDataPool p = new CSVDataPool(); + p.setFile(new DynamicValue(FileResolver.createPathForResourceId(resource.getId().toString()))); + f.setDataSource(p); + f.setDataSourceType("excel"); + + Plan planUsingResource = PlanBuilder.create().startBlock(BaseArtefacts.sequence()).add(f).endBlock().build(); + planUsingResource.addAttribute(AbstractOrganizableObject.NAME, PLAN_USING_RESOURCE); + context.getPlanAccessor().save(planUsingResource); + + List findReferencesResponse = + referenceFinder.findReferences(new FindReferencesRequest(FindReferencesRequest.Type.RESOURCE_ID, resource.getId().toString())); + assertFirstResponseReference(1, findReferencesResponse, planUsingResource, PLAN_USING_RESOURCE); + + + findReferencesResponse = + referenceFinder.findReferences(new FindReferencesRequest(FindReferencesRequest.Type.RESOURCE_NAME, CSV_FILE)); + assertFirstResponseReference(1, findReferencesResponse, planUsingResource, PLAN_USING_RESOURCE); + + + } + + private static void assertFirstResponseReference(int expected, List findReferencesResponse, Plan planUsingResource, String planUsingResource1) { + assertEquals(expected, findReferencesResponse.size()); + FindReferencesResponse findReferencesResponse1 = findReferencesResponse.get(0); + assertEquals(planUsingResource.getId().toString(), findReferencesResponse1.id); + assertEquals(planUsingResource1, findReferencesResponse1.name); + } + +} \ No newline at end of file diff --git a/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java b/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java index 236b4550d..ad6bac876 100644 --- a/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java +++ b/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java @@ -28,6 +28,12 @@ public class EntityDependencyTreeVisitor { // TODO declare it as non-static to avoid potential leaks private static final Map, BeanInfo> beanInfoCache = new ConcurrentHashMap<>(); + public enum VISIT_MODE { + SINGLE, + RECURSIVE, + RESOLVE_ALL + } + public EntityDependencyTreeVisitor(EntityManager entityManager, ObjectPredicate objectPredicate) { super(); this.entityManager = entityManager; @@ -35,27 +41,27 @@ public EntityDependencyTreeVisitor(EntityManager entityManager, ObjectPredicate } public void visitEntityDependencyTree(String entityName, String entityId, EntityTreeVisitor visitor, - boolean recursive) { - EntityTreeVisitorContext context = new EntityTreeVisitorContext(objectPredicate, recursive, visitor); + VISIT_MODE visitMode) { + EntityTreeVisitorContext context = new EntityTreeVisitorContext(objectPredicate, visitMode, visitor); visitEntity(entityName, entityId, context); } public void visitSingleObject(Object object, EntityTreeVisitor visitor, Set messageCollector) { - EntityTreeVisitorContext context = new EntityTreeVisitorContext(objectPredicate, false, visitor); + EntityTreeVisitorContext context = new EntityTreeVisitorContext(objectPredicate, VISIT_MODE.SINGLE, visitor); resolveEntityDependencies(object, context); } public class EntityTreeVisitorContext { - private final boolean recursive; + private final VISIT_MODE visitMode; private final ObjectPredicate objectPredicate; private final EntityTreeVisitor visitor; private final Map stack = new HashMap<>(); - public EntityTreeVisitorContext(ObjectPredicate objectPredicate, boolean recursive, EntityTreeVisitor visitor) { + public EntityTreeVisitorContext(ObjectPredicate objectPredicate, VISIT_MODE visitMode, EntityTreeVisitor visitor) { super(); this.objectPredicate = objectPredicate; - this.recursive = recursive; + this.visitMode = visitMode; this.visitor = visitor; } @@ -64,11 +70,15 @@ public ObjectPredicate getObjectPredicate() { } public void visitEntity(String entityName, String entityId) { - if (recursive) { + if (VISIT_MODE.RECURSIVE.equals(visitMode)) { EntityDependencyTreeVisitor.this.visitEntity(entityName, entityId, this); } } - + + public void onResolvedEntity(String entityName, String entityId, Object entity) { + visitor.onResolvedEntity(entityName, entityId, entity); + } + public String resolvedEntityId(String entityName, String entityId) { return visitor.onResolvedEntityId(entityName, entityId); } @@ -77,8 +87,8 @@ public EntityTreeVisitor getVisitor() { return visitor; } - public boolean isRecursive() { - return recursive; + public VISIT_MODE getVisitMode() { + return visitMode; } protected Map getStack() { @@ -249,7 +259,7 @@ private String resolveEntityIdAndVisitResolvedEntity(String entityName, Object a } // Visit the resolved entity - if (resolvedEntityId != null && visitorContext.isRecursive()) { + if (resolvedEntityId != null && VISIT_MODE.RECURSIVE.equals(visitorContext.getVisitMode())) { visitEntity(entityName, resolvedEntityId, visitorContext); } diff --git a/step-core/src/main/java/step/core/entities/EntityManager.java b/step-core/src/main/java/step/core/entities/EntityManager.java index c6c98933a..d3a01d559 100644 --- a/step-core/src/main/java/step/core/entities/EntityManager.java +++ b/step-core/src/main/java/step/core/entities/EntityManager.java @@ -98,6 +98,7 @@ public void getEntitiesReferences(String entityType, ObjectPredicate objectPredi */ public void getEntitiesReferences(String entityName, String entityId, ObjectPredicate objectPredicate, EntityReferencesMap references, boolean recursive) { EntityDependencyTreeVisitor entityDependencyTreeVisitor = new EntityDependencyTreeVisitor(this, objectPredicate); + EntityDependencyTreeVisitor.VISIT_MODE visitMode = recursive ? EntityDependencyTreeVisitor.VISIT_MODE.RECURSIVE : EntityDependencyTreeVisitor.VISIT_MODE.SINGLE; entityDependencyTreeVisitor.visitEntityDependencyTree(entityName, entityId, new EntityTreeVisitor() { @Override @@ -114,7 +115,7 @@ public void onResolvedEntity(String entityName, String entityId, Object entity) public String onResolvedEntityId(String entityName, String resolvedEntityId) { return null; } - }, recursive); + }, visitMode); } public void updateReferences(Object entity, Map references, ObjectPredicate objectPredicate, Set messageCollector) { diff --git a/step-core/src/test/java/step/core/entities/EntityDependencyTreeVisitorTest.java b/step-core/src/test/java/step/core/entities/EntityDependencyTreeVisitorTest.java index df664803b..96798523f 100644 --- a/step-core/src/test/java/step/core/entities/EntityDependencyTreeVisitorTest.java +++ b/step-core/src/test/java/step/core/entities/EntityDependencyTreeVisitorTest.java @@ -73,7 +73,7 @@ public void onResolvedEntity(String entityName, String entityId, Object entity) public String onResolvedEntityId(String entityName, String resolvedEntityId) { return null; } - }, true); + }, EntityDependencyTreeVisitor.VISIT_MODE.RECURSIVE); assertEquals(9, entityIds.size()); diff --git a/step-functions/step-functions-package/src/main/java/step/functions/packages/FunctionPackageEntity.java b/step-functions/step-functions-package/src/main/java/step/functions/packages/FunctionPackageEntity.java index 8bdc174fe..6dd1de9b4 100644 --- a/step-functions/step-functions-package/src/main/java/step/functions/packages/FunctionPackageEntity.java +++ b/step-functions/step-functions-package/src/main/java/step/functions/packages/FunctionPackageEntity.java @@ -1,10 +1,7 @@ package step.functions.packages; -import step.core.entities.Entity; +import step.core.entities.*; import step.core.entities.EntityDependencyTreeVisitor.EntityTreeVisitorContext; -import step.core.entities.EntityManager; -import step.core.entities.DependencyTreeVisitorHook; -import step.core.entities.EntityManagerSupplier; import step.functions.Function; public class FunctionPackageEntity extends Entity { @@ -28,7 +25,7 @@ public void onVisitEntity(Object o, EntityTreeVisitorContext context) { Function f = (Function) o; String id = (String) f.getCustomField(FUNCTION_PACKAGE_ID); if (id != null) { - if(context.isRecursive()) { + if(EntityDependencyTreeVisitor.VISIT_MODE.RECURSIVE.equals(context.getVisitMode())) { context.visitEntity(entityName, id); } diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java index bb209d171..1a5483e07 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java @@ -45,56 +45,91 @@ public FunctionLocator(FunctionAccessor functionAccessor, SelectorHelper selecto /** * Resolve a {@link CallFunction} artefact to the underlying {@link Function} * - * @param callFunctionArtefact the CallFunction artefact + * @param callFunctionArtefact the CallFunction artifact to be resolved * @param objectPredicate the predicate to be used to filter the results out * @param bindings the bindings to be used for the evaluation of dynamic expressions (can be null) - * @return the {@link Function} referenced by this artefact + * @return the {@link Function} referenced by this artifact */ public Function getFunction(CallFunction callFunctionArtefact, ObjectPredicate objectPredicate, Map bindings) { + return selectAllFunctionsByPriority(callFunctionArtefact, objectPredicate, bindings, true).get(0); + } + + /** + * Resolve a {@link CallFunction} artefact to the underlying list of matching {@link Function} + * @param callFunctionArtefact the {@link CallFunction} artifact to resolve + * @param objectPredicate to filter out results + * @param bindings to be used for evaluation of selection criteria + * @param strictMode whether selection is strict and must find a result or we can ignore unresolvable dynamic selection criteria and bypass activation expression + * @return the list of resolved Keywords, can be empty when strictMode is false + */ + public List selectAllFunctionsByPriority(CallFunction callFunctionArtefact, ObjectPredicate objectPredicate, Map bindings, boolean strictMode) { Objects.requireNonNull(callFunctionArtefact, "The artefact must not be null"); Objects.requireNonNull(objectPredicate, "The object predicate must not be null"); - Function function; + String selectionAttributesJson = callFunctionArtefact.getFunction().get(); Map attributes; try { attributes = selectorHelper.buildSelectionAttributesMap(selectionAttributesJson, bindings); } catch (Exception e) { - throw new NoSuchElementException("Unable to find keyword with attributes "+selectionAttributesJson + ". Cause: " + e.getMessage()); + //We only throw exception for missing bindings when strictMode is ON + if (strictMode) { + throw new NoSuchElementException("Unable to find keyword with attributes " + selectionAttributesJson + ". Cause: " + e.getMessage()); + } else { + return List.of(); + } } - if(attributes.size()>0) { - + if (!attributes.isEmpty()) { Stream stream = StreamSupport.stream(functionAccessor.findManyByAttributes(attributes), false); stream = stream.filter(objectPredicate); List functionsMatchingByAttributes = stream.collect(Collectors.toList()); // reorder matching functions: the function from current AP has a priority - List orderedFunctions = LocatorHelper.prioritizeAndFilterApEntities(functionsMatchingByAttributes, bindings); + List orderedFunctions = LocatorHelper.prioritizeAndFilterApEntities(functionsMatchingByAttributes, bindings, !strictMode); + // In strict mode at least one match is required + if (strictMode && orderedFunctions.isEmpty()) { + throw new NoSuchElementException("Unable to find keyword with attributes " + selectionAttributesJson); + } - // after prioritization, we check the chosen active keyword version + // after prioritization, we either select only the one matching the active version whenever provided or returned all matching ones Set activeKeywordVersions = getActiveKeywordVersions(bindings); - if (activeKeywordVersions != null && activeKeywordVersions.size() > 0) { - // First try to find a function matching one of the active versions - function = orderedFunctions.stream().filter(f -> { + if (activeKeywordVersions != null && !activeKeywordVersions.isEmpty()) { + // First try to find the functions matching one of the active versions + List activeVersions = orderedFunctions.stream().filter(f -> { String version = f.getAttributes().get(AbstractOrganizableObject.VERSION); return version != null && activeKeywordVersions.contains(version); - }).findFirst().orElse(null); - // if no function has been found with one of the active versions, return the first function WITHOUT version - if (function == null) { - function = orderedFunctions.stream().filter(f -> { + }).collect(Collectors.toList()); + // if no function has been found with one of the active versions, return the functions WITHOUT versions + if (activeVersions.isEmpty()) { + activeVersions = orderedFunctions.stream().filter(f -> { String version = f.getAttributes().get(AbstractOrganizableObject.VERSION); return version == null || version.trim().isEmpty(); - }).findFirst().orElseThrow(() -> new NoSuchElementException("Unable to find keyword with attributes " + selectionAttributesJson + " matching on of the versions: " + activeKeywordVersions)); + }).collect(Collectors.toList()); + } + if (activeVersions.isEmpty() && strictMode) { + throw new NoSuchElementException("Unable to find keyword with attributes " + selectionAttributesJson + " matching on of the versions: " + activeKeywordVersions); + } else { + return activeVersions; } } else { - // No active versions defined. Return the first function - function = orderedFunctions.stream().findFirst().orElseThrow(() -> new NoSuchElementException("Unable to find keyword with attributes " + selectionAttributesJson)); + //No version defined with simply return the ordered function by priorities + return orderedFunctions; } - return function; } else { throw new NoSuchElementException("No selection attribute defined"); } + } + /** + * Resolve a {@link CallFunction} artefact to the underlying list of matching {@link Function} + * + * @param callFunctionArtefact the CallFunction artifact + * @param objectPredicate the predicate to be used to filter the results out + * @param bindings the bindings to be used for the evaluation of dynamic expressions (can be null) + * @return the {@link Function} list referenced by this artifact + */ + public List getMatchingFunctions(CallFunction callFunctionArtefact, ObjectPredicate objectPredicate, Map bindings) { + return selectAllFunctionsByPriority(callFunctionArtefact, objectPredicate, bindings, false); } private Set getActiveKeywordVersions(Map bindings) { diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/LocatorHelper.java b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/LocatorHelper.java index 973301eed..24484628e 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/LocatorHelper.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/LocatorHelper.java @@ -17,7 +17,7 @@ public class LocatorHelper { /** * Reorders and filters entities according to the current automation package and activation expression */ - public static List prioritizeAndFilterApEntities(List entities, Map bindings) { + public static List prioritizeAndFilterApEntities(List entities, Map bindings, boolean bypassActivation) { // reorder entities: entities from current AP have a priority List entitiesFromSameAP = new ArrayList<>(); List entitiesActivatedExplicitly = new ArrayList<>(); @@ -34,7 +34,7 @@ public static List prioritizeAndFilterA break; } } - if (evaluationExpressionIsDefined(entity)) { + if (!bypassActivation && evaluationExpressionIsDefined(entity)) { if (isActivated(bindings, entity)) { entitiesActivatedExplicitly.add(entity); } diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/PlanLocator.java b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/PlanLocator.java index 71165ae51..49ffa0093 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/PlanLocator.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/PlanLocator.java @@ -44,7 +44,7 @@ public PlanLocator(PlanAccessor accessor, SelectorHelper selectorHelper) { } /** - * Resolve a {@link CallPlan} artefact to the underlying {@link Plan}. Returns null if plan is not resolved by ID + * Resolve a {@link CallPlan} artefact to the underlying {@link Plan}. If multiple plans are resolved, the first one (ordered by priority) is returned) * * @param artefact the {@link CallPlan} artefact * @param objectPredicate the predicate to be used to filter the results out @@ -52,24 +52,69 @@ public PlanLocator(PlanAccessor accessor, SelectorHelper selectorHelper) { * @return the {@link Plan} referenced by the provided artefact */ public Plan selectPlan(CallPlan artefact, ObjectPredicate objectPredicate, Map bindings) { + return selectAlPlansByAttributesAndPriority(artefact, objectPredicate, bindings, true).get(0); + } + + /** + * Resolve a {@link CallPlan} artefact to the list of underlying matching {@link Plan}. + * @param artefact the {@link CallPlan} artefact + * @param objectPredicate the predicate to be used to filter the results out + * @param bindings the bindings to be used for the evaluation of dynamic expressions (can be null) + * @param strictMode whether selection is strict and must find a result or we can ignore unresolvable dynamic selection criteria and bypass activation expression + * @return the list of resolved Plan, can be empty when strictMode is false + */ + public List selectAlPlansByAttributesAndPriority(CallPlan artefact, ObjectPredicate objectPredicate, Map bindings, boolean strictMode) { Objects.requireNonNull(artefact, "The artefact must not be null"); Objects.requireNonNull(objectPredicate, "The object predicate must not be null"); - Plan a; + // Handle CallPlan with plan ID reference if(artefact.getPlanId()!=null) { - a = Optional.ofNullable(accessor.get(artefact.getPlanId())).orElseThrow(() -> new NoSuchElementException("Unable to find plan with id: " + artefact.getPlanId())); + Plan plan = accessor.get(artefact.getPlanId()); + if (plan != null && objectPredicate.test(plan)) { + return List.of(plan); + } else if (strictMode) { + throw new NoSuchElementException("Unable to find plan with id: " + artefact.getPlanId()); + } else { + return List.of(); + } } else { - Map selectionAttributes = selectorHelper.buildSelectionAttributesMap(artefact.getSelectionAttributes().get(), bindings); + // Handle Call Plan with call by attributes + String selectionAttributesJson = artefact.getSelectionAttributes().get(); + Map selectionAttributes; + try { + selectionAttributes = selectorHelper.buildSelectionAttributesMap(selectionAttributesJson, bindings); + } catch (Exception e) { + //In case bindings are missing, we only throw an exception in strict Mode (used in execution context) + if (strictMode) { + throw e; + } else { + return List.of(); + } + } Stream stream = StreamSupport.stream(accessor.findManyByAttributes(selectionAttributes), false); stream = stream.filter(objectPredicate); List matchingPlans = stream.collect(Collectors.toList()); // The same logic as for functions - plans from current automation package have priority in 'CallPlan' // We use prioritization by current automation package and filtering by activation expressions - List orderedPlans = LocatorHelper.prioritizeAndFilterApEntities(matchingPlans, bindings); - a = orderedPlans.stream().findFirst().orElseThrow(()->new NoSuchElementException("Unable to find plan with attributes: "+selectionAttributes.toString())); + List orderedPlans = LocatorHelper.prioritizeAndFilterApEntities(matchingPlans, bindings, !strictMode); + if (strictMode && orderedPlans.isEmpty()) { + throw new NoSuchElementException("Unable to find plan with attributes: "+ selectionAttributesJson); + } + return orderedPlans; } - return a; + } + + /** + * Resolve a {@link CallPlan} artefact to the list of underlying matching {@link Plan}. + * + * @param artefact the {@link CallPlan} artefact + * @param objectPredicate the predicate to be used to filter the results out + * @param bindings the bindings to be used for the evaluation of dynamic expressions (can be null) + * @return the list of {@link Plan} referenced by this artifact + */ + public List getMatchingPlans(CallPlan artefact, ObjectPredicate objectPredicate, Map bindings) { + return selectAlPlansByAttributesAndPriority(artefact, objectPredicate, bindings, false); } /** diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/core/plans/PlanEntity.java b/step-plans/step-plans-base-artefacts/src/main/java/step/core/plans/PlanEntity.java index a72d62db6..73e8ddb9d 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/core/plans/PlanEntity.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/core/plans/PlanEntity.java @@ -12,13 +12,25 @@ public class PlanEntity extends Entity> { public PlanEntity(Accessor accessor, PlanLocator planLocator, EntityManager entityManager) { super(EntityConstants.plans, accessor, Plan.class); entityManager.addDependencyTreeVisitorHook((entity, context) -> { - //This is only required to recursively visit the plans referenced by callPlan artefacts - if (entity instanceof CallPlan && context.isRecursive()) { - try { - Plan plan = planLocator.selectPlanNotNull((CallPlan) entity, context.getObjectPredicate(), null); - context.visitEntity(EntityConstants.plans, plan.getId().toString()); - } catch (PlanLocator.PlanLocatorException ex) { - context.getVisitor().onWarning(ex.getMessage()); + // Only apply the logic for CallPlan artifacts + if (entity instanceof CallPlan) { + CallPlan callPlan = (CallPlan) entity; + switch (context.getVisitMode()) { + case RECURSIVE: + // In recursive mode, we recursively visit the resolved plan. If multiple plans match, the one with the highest priority is chosen + try { + Plan plan = planLocator.selectPlanNotNull(callPlan, context.getObjectPredicate(), null); + context.visitEntity(EntityConstants.plans, plan.getId().toString()); + } catch (PlanLocator.PlanLocatorException ex) { + context.getVisitor().onWarning(ex.getMessage()); + } + break; + case RESOLVE_ALL: + // In resolve ALL mode, we resolve all matching plans but do not visit recursively + planLocator.getMatchingPlans(callPlan, context.getObjectPredicate(), null).forEach(p -> { + context.onResolvedEntity(EntityConstants.plans, p.getId().toHexString(), p); + }); + break; } } }); diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/functions/accessor/FunctionEntity.java b/step-plans/step-plans-base-artefacts/src/main/java/step/functions/accessor/FunctionEntity.java index 646e15204..d1cb4996e 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/functions/accessor/FunctionEntity.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/functions/accessor/FunctionEntity.java @@ -20,14 +20,26 @@ public class FunctionEntity extends Entity> { public FunctionEntity(Accessor accessor, FunctionLocator functionLocator, EntityManager entityManager) { super(EntityConstants.functions, accessor, Function.class); entityManager.addDependencyTreeVisitorHook((t, context) -> { - //This is only required to recursively visit the function referenced by callFunction artefacts - if (t instanceof CallFunction && context.isRecursive()) { - try { - Function function = functionLocator.getFunction((CallFunction) t, context.getObjectPredicate(), - null); - context.visitEntity(EntityConstants.functions, function.getId().toString()); - } catch (NoSuchElementException e) { - context.getVisitor().onWarning("The keyword referenced by the call keyword artefact '" + ((CallFunction) t).getAttribute(AbstractOrganizableObject.NAME) + "' could not be found"); + //Only apply logic is the entity is a CallFunction + if (t instanceof CallFunction) { + CallFunction callFunction = (CallFunction) t; + switch (context.getVisitMode()) { + case RECURSIVE: + //In recursive mode we visit the resolved entity recursively (the highest priority is chosen if multiple entity matches) + try { + Function function = functionLocator.getFunction(callFunction, context.getObjectPredicate(), + null); + context.visitEntity(EntityConstants.functions, function.getId().toString()); + } catch (NoSuchElementException e) { + context.getVisitor().onWarning("The keyword referenced by the call keyword artefact '" + (callFunction).getAttribute(AbstractOrganizableObject.NAME) + "' could not be found"); + } + break; + case RESOLVE_ALL: + //In resolve All mode we resolve all matching entities but do not visit recursively + functionLocator.getMatchingFunctions(callFunction, context.getObjectPredicate(), null).forEach(f -> { + context.onResolvedEntity(EntityConstants.functions, f.getId().toHexString(), f); + }); + break; } } }); diff --git a/step-plans/step-plans-core/src/main/java/step/core/scheduler/ScheduleEntity.java b/step-plans/step-plans-core/src/main/java/step/core/scheduler/ScheduleEntity.java index 8afcf62cc..fb78a5f81 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/scheduler/ScheduleEntity.java +++ b/step-plans/step-plans-core/src/main/java/step/core/scheduler/ScheduleEntity.java @@ -24,7 +24,7 @@ public void onVisitEntity(Object entity, EntityDependencyTreeVisitor.EntityTreeV if (repositoryObject != null && repositoryObject.getRepositoryID() != null && repositoryObject.getRepositoryID().equals(LOCAL_REPOSITORY_ID)) { String localPlanId = repositoryObject.getRepositoryParameters().get(RepositoryObjectReference.PLAN_ID); if (localPlanId != null) { - if (context.isRecursive()) { + if (EntityDependencyTreeVisitor.VISIT_MODE.RECURSIVE.equals(context.getVisitMode())) { context.visitEntity(EntityConstants.plans, localPlanId); } String newEntityId = context.resolvedEntityId(EntityConstants.plans, localPlanId); From 54984471ab80da198dcc087566130fb8a8af5ecd Mon Sep 17 00:00:00 2001 From: David Stephan Date: Thu, 8 Jan 2026 14:45:37 +0100 Subject: [PATCH 19/31] SED-4340 PR feedbacks --- .../java/step/core/references/ReferenceFinder.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java index 52f84fe64..d4ca7e139 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java @@ -62,9 +62,10 @@ public List findReferences(FindReferencesRequest request } private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) { - List referencedObjects = getReferencedObjects(entityType, object).stream().filter(o -> (o != null && !o.equals(object))).collect(Collectors.toList()); - //System.err.println("objects referenced from plan: " + planToString(plan) + ": "+ referencedObjects.stream().map(ReferenceFinderServices::objectToString).collect(Collectors.toList())); - return referencedObjects.stream().filter(o -> doesRequestMatch(request, o)).collect(Collectors.toList()); + return getReferencedObjects(entityType, object).stream() + .filter(o -> (o != null && !o.equals(object))) + .filter(o -> doesRequestMatch(request, o)) + .collect(Collectors.toList()); } // returns a (generic) set of objects referenced by a plan @@ -91,7 +92,7 @@ private boolean doesRequestMatch(FindReferencesRequest req, Object o) { Plan p = (Plan) o; switch (req.searchType) { case PLAN_NAME: - return p.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); + return req.searchValue.equals(p.getAttribute(AbstractOrganizableObject.NAME)); case PLAN_ID: return p.getId().toString().equals(req.searchValue); default: @@ -101,7 +102,7 @@ private boolean doesRequestMatch(FindReferencesRequest req, Object o) { Function f = (Function) o; switch (req.searchType) { case KEYWORD_NAME: - return f.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); + return req.searchValue.equals(f.getAttribute(AbstractOrganizableObject.NAME)); case KEYWORD_ID: return f.getId().toString().equals(req.searchValue); default: @@ -111,7 +112,7 @@ private boolean doesRequestMatch(FindReferencesRequest req, Object o) { Resource r = (Resource) o; switch (req.searchType) { case RESOURCE_NAME: - return r.getAttribute(AbstractOrganizableObject.NAME).equals(req.searchValue); + return req.searchValue.equals(r.getAttribute(AbstractOrganizableObject.NAME)); case RESOURCE_ID: return r.getId().toString().equals(req.searchValue); default: From a431643e726cfd1d1f9af0cdb6a1e9507f5a067b Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 9 Jan 2026 10:50:21 +0100 Subject: [PATCH 20/31] SED-4340 PR feedbacks --- .../main/java/step/core/references/ReferenceFinder.java | 6 +++--- .../step/core/entities/EntityDependencyTreeVisitor.java | 9 +++++++++ .../java/step/artefacts/handlers/FunctionLocator.java | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java index d4ca7e139..d39c5fd6a 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java @@ -72,9 +72,9 @@ private List getReferencedObjectsMatchingRequest(String entityType, Abst private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) { Set referencedObjects = new HashSet<>(); - // The references can be filled in three different ways due to the implementation: - // 1. through the predicate (just below) - // 2. by (actual object) reference in the tree visitor (onResolvedEntity) + // The references can be filled in two different ways due to the implementation: + // 1. by (actual object) reference in the tree visitor (onResolvedEntity) + // 2. by object ID in the tree visitor (onResolvedEntityId) // No context predicate is used by the reference finder, since we want to find all entities (i.e. if we search the usages of a Keyword from the Common project, we should be able // to find plans using it in other projects. diff --git a/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java b/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java index ad6bac876..29db0dc1c 100644 --- a/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java +++ b/step-core/src/main/java/step/core/entities/EntityDependencyTreeVisitor.java @@ -29,8 +29,17 @@ public class EntityDependencyTreeVisitor { private static final Map, BeanInfo> beanInfoCache = new ConcurrentHashMap<>(); public enum VISIT_MODE { + /** + * Visit one entity resolving its references. One reference is resolved to at most one entity. Resolved entities are NOT visited recursively. + */ SINGLE, + /** + * Visit one entity resolving its references. One reference is resolved to at most one entity. Resolved entities are visited recursively. + */ RECURSIVE, + /** + * Visit one entity resolving its references. One reference can be resolved to multiple entities. Resolved entities are NOT visited recursively. + */ RESOLVE_ALL } diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java index 1a5483e07..eae8c43ca 100644 --- a/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/artefacts/handlers/FunctionLocator.java @@ -112,7 +112,7 @@ public List selectAllFunctionsByPriority(CallFunction callFunctionArte return activeVersions; } } else { - //No version defined with simply return the ordered function by priorities + //No active version provided, we simply return the ordered function by priorities return orderedFunctions; } } else { From e0c4c97fffed6353c957f9658165f7cf120bd33a Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 9 Jan 2026 11:31:40 +0100 Subject: [PATCH 21/31] SED-4340 fixing incorrect context and predicate usage --- .../step/core/references/ReferenceFinder.java | 49 +++++++++++++------ .../references/ReferenceFinderServices.java | 15 +----- .../core/references/ReferenceFinderTest.java | 3 +- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java index d39c5fd6a..2e0c19bcc 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java @@ -1,9 +1,15 @@ package step.core.references; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import step.core.AbstractContext; import step.core.accessors.AbstractOrganizableObject; import step.core.entities.EntityConstants; import step.core.entities.EntityDependencyTreeVisitor; import step.core.entities.EntityManager; +import step.core.objectenricher.EnricheableObject; +import step.core.objectenricher.ObjectHookRegistry; +import step.core.objectenricher.ObjectPredicate; import step.core.plans.Plan; import step.core.plans.PlanAccessor; import step.functions.Function; @@ -16,10 +22,14 @@ public class ReferenceFinder { + private static final Logger logger = LoggerFactory.getLogger(ReferenceFinder.class); + private final EntityManager entityManager; + private final ObjectHookRegistry objectHookRegistry; - public ReferenceFinder(EntityManager entityManager) { + public ReferenceFinder(EntityManager entityManager, ObjectHookRegistry objectHookRegistry) { this.entityManager = entityManager; + this.objectHookRegistry = objectHookRegistry; } public List findReferences(FindReferencesRequest request) { @@ -39,9 +49,13 @@ public List findReferences(FindReferencesRequest request try (Stream functionStream = functionAccessor.streamLazy()) { functionStream.forEach(function -> { - List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.functions, function, request); - if (!matchingObjects.isEmpty()) { - results.add(new FindReferencesResponse(function)); + try { + List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.functions, function, request); + if (!matchingObjects.isEmpty()) { + results.add(new FindReferencesResponse(function)); + } + } catch (Exception e) { + logger.error("Unable to find references for function {}", function.getId(), e); } }); } @@ -49,9 +63,13 @@ public List findReferences(FindReferencesRequest request // Find plans containing usages try (Stream stream = (request.includeHiddenPlans) ? planAccessor.streamLazy() : planAccessor.getVisiblePlans()) { stream.forEach(plan -> { - List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.plans, plan, request); - if (!matchingObjects.isEmpty()) { - results.add(new FindReferencesResponse(plan)); + try { + List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.plans, plan, request); + if (!matchingObjects.isEmpty()) { + results.add(new FindReferencesResponse(plan)); + } + } catch (Exception e) { + logger.error("Unable to find references for plan {}", plan.getId(), e); } }); } @@ -61,7 +79,7 @@ public List findReferences(FindReferencesRequest request return results; } - private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) { + private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) throws Exception { return getReferencedObjects(entityType, object).stream() .filter(o -> (o != null && !o.equals(object))) .filter(o -> doesRequestMatch(request, o)) @@ -69,18 +87,21 @@ private List getReferencedObjectsMatchingRequest(String entityType, Abst } // returns a (generic) set of objects referenced by a plan - private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) { + private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) throws Exception { Set referencedObjects = new HashSet<>(); // The references can be filled in two different ways due to the implementation: // 1. by (actual object) reference in the tree visitor (onResolvedEntity) // 2. by object ID in the tree visitor (onResolvedEntityId) - // No context predicate is used by the reference finder, since we want to find all entities (i.e. if we search the usages of a Keyword from the Common project, we should be able - // to find plans using it in other projects. - // This unfortunately can return incorrect results, i.e. a keyword "MyKeyword" is created in ProjectA and ProjectB, A PlanA is created in ProjectA and is using the KW of the same project. - // Searching usage of "MyKeyword" in projectB will return the planA from projectA - EntityDependencyTreeVisitor entityDependencyTreeVisitor = new EntityDependencyTreeVisitor(entityManager, o -> true); + // When searching the references of a give entity we must apply the predicate as if we were in the context of this entity + ObjectPredicate predicate = o -> true; //default value for non enricheable objects + if (object instanceof EnricheableObject) { + AbstractContext context = new AbstractContext() {}; + objectHookRegistry.rebuildContext(context, (EnricheableObject) object); + predicate = objectHookRegistry.getObjectPredicate(context); + } + EntityDependencyTreeVisitor entityDependencyTreeVisitor = new EntityDependencyTreeVisitor(entityManager, predicate); FindReferencesTreeVisitor entityTreeVisitor = new FindReferencesTreeVisitor(entityManager, referencedObjects); entityDependencyTreeVisitor.visitEntityDependencyTree(entityType, object.getId().toString(), entityTreeVisitor, EntityDependencyTreeVisitor.VISIT_MODE.RESOLVE_ALL); diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java index b6110c07c..8ad874cbe 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinderServices.java @@ -1,18 +1,9 @@ package step.core.references; import io.swagger.v3.oas.annotations.tags.Tag; -import step.core.accessors.AbstractOrganizableObject; import step.core.deployment.AbstractStepServices; -import step.core.entities.EntityConstants; +import step.core.objectenricher.ObjectHookRegistry; import step.framework.server.security.Secured; -import step.core.entities.EntityDependencyTreeVisitor; -import step.core.entities.EntityManager; -import step.core.objectenricher.ObjectPredicate; -import step.core.plans.Plan; -import step.core.plans.PlanAccessor; -import step.functions.Function; -import step.functions.accessor.FunctionAccessor; -import step.resources.Resource; import jakarta.annotation.PostConstruct; import jakarta.inject.Singleton; @@ -20,8 +11,6 @@ import jakarta.ws.rs.core.MediaType; import java.util.*; -import java.util.stream.Collectors; -import java.util.stream.Stream; @Singleton @Path("references") @@ -33,7 +22,7 @@ public class ReferenceFinderServices extends AbstractStepServices { @PostConstruct public void init() throws Exception { super.init(); - referenceFinder = new ReferenceFinder(getContext().getEntityManager()); + referenceFinder = new ReferenceFinder(getContext().getEntityManager(), getContext().require(ObjectHookRegistry.class)); } @Path("/findReferences") diff --git a/step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java b/step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java index 5f2e5fcfa..ca1ae42df 100644 --- a/step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java +++ b/step-controller/step-controller-server/src/test/java/step/core/references/ReferenceFinderTest.java @@ -9,6 +9,7 @@ import step.core.GlobalContext; import step.core.accessors.AbstractOrganizableObject; import step.core.dynamicbeans.DynamicValue; +import step.core.objectenricher.ObjectHookRegistry; import step.core.plans.Plan; import step.core.plans.builder.PlanBuilder; import step.core.plugins.PluginManager; @@ -43,7 +44,7 @@ public class ReferenceFinderTest { @Before public void setup() throws ClassNotFoundException, PluginManager.Builder.CircularDependencyException, InstantiationException, IllegalAccessException { context = createGlobalContext(); - referenceFinder = new ReferenceFinder(context.getEntityManager()); + referenceFinder = new ReferenceFinder(context.getEntityManager(), new ObjectHookRegistry()); functionAccessor = context.require(FunctionAccessor.class); } From 340fc42379b574c19c2f039df469f95f4ad3c072 Mon Sep 17 00:00:00 2001 From: Christoph Langguth Date: Tue, 13 Jan 2026 14:57:57 +0100 Subject: [PATCH 22/31] SED-4471 Add audit logging to parameters, schedules, resources (#581) * SED-4471 Add audit logging to parameters and schedules * SED-4471 Code simplification * SED-4471 More logging --- .../parametermanager/ParameterServices.java | 7 +++++-- .../core/scheduler/SchedulerServices.java | 20 +++++++++++++----- .../java/step/resources/ResourceServices.java | 21 +++++++++++++++++-- 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/step-controller/step-controller-base-plugins/src/main/java/step/plugins/parametermanager/ParameterServices.java b/step-controller/step-controller-base-plugins/src/main/java/step/plugins/parametermanager/ParameterServices.java index 0e593c00e..b2d28e208 100644 --- a/step-controller/step-controller-base-plugins/src/main/java/step/plugins/parametermanager/ParameterServices.java +++ b/step-controller/step-controller-base-plugins/src/main/java/step/plugins/parametermanager/ParameterServices.java @@ -106,7 +106,9 @@ public Parameter save(Parameter newParameter) { private Parameter save(Parameter newParameter, Parameter sourceParameter) { assertRights(newParameter); try { - return maskProtectedValue(parameterManager.save(newParameter, sourceParameter, getSession().getUser().getUsername())); + Parameter result = maskProtectedValue(parameterManager.save(newParameter, sourceParameter, getSession().getUser().getUsername())); + auditLog("save", result, Map.of("key", result.getKey())); + return result; } catch (ParameterManagerException e) { throw new ControllerServiceException(e.getMessage()); } @@ -137,6 +139,7 @@ public Parameter clone(String id) { newParameter.setId(new ObjectId()); //Remove link to AP Optional.ofNullable(newParameter.getCustomFields()).ifPresent(fields -> fields.remove(AutomationPackageEntity.AUTOMATION_PACKAGE_ID)); + auditLog("clone", newParameter, Map.of("key", newParameter.getKey())); return save(newParameter, sourceParameter); } @@ -145,7 +148,7 @@ public void delete(String id) { Parameter parameter = get(id); assertEntityIsEditableInContext(parameter); assertRights(parameter); - + auditLog("delete", parameter, Map.of("key", parameter.getKey())); parameterAccessor.remove(new ObjectId(id)); } diff --git a/step-controller/step-controller-server/src/main/java/step/core/scheduler/SchedulerServices.java b/step-controller/step-controller-server/src/main/java/step/core/scheduler/SchedulerServices.java index 47abe9a93..b54a70aff 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/scheduler/SchedulerServices.java +++ b/step-controller/step-controller-server/src/main/java/step/core/scheduler/SchedulerServices.java @@ -32,13 +32,11 @@ import step.core.execution.model.ExecutionParameters; import step.core.repositories.RepositoryObjectReference; import step.framework.server.Session; +import step.framework.server.audit.AuditLogger; import step.framework.server.security.Secured; import step.framework.server.security.SecuredContext; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; +import java.util.*; @Singleton @Path("scheduler/task") @@ -93,6 +91,7 @@ public ExecutiontTaskParameters save(ExecutiontTaskParameters schedule) { } catch (Exception e) { throw new ControllerServiceException(e.getMessage()); } + auditLog("save", schedule); return schedule; } @@ -131,6 +130,8 @@ public void enableAllExecutionTasksSchedule(@QueryParam("enabled") Boolean enabl } else { scheduler.disableAllExecutionTasksSchedule(); } + // sample log: INFO AuditLogger - {"user":"admin","operation":"manage-scheduler","type":"scheduler","name":"scheduler","id":null,"attributes":{"enabled":"false"}} + AuditLogger.logEntityModification(getSession(), "manage-scheduler", "scheduler", null, "scheduler", Map.of("enabled", Boolean.toString(Boolean.TRUE.equals(enabled)))); } @Operation(description = "Enable/disable the given scheduler task.") @@ -139,10 +140,17 @@ public void enableAllExecutionTasksSchedule(@QueryParam("enabled") Boolean enabl @Secured(right = "{entity}-toggle") public void enableExecutionTask(@PathParam("id") String executionTaskID, @QueryParam("enabled") Boolean enabled) { try { + String auditOperation; if (enabled != null && enabled) { scheduler.enableExecutionTask(executionTaskID); + auditOperation = "enable"; } else { scheduler.disableExecutionTask(executionTaskID); + auditOperation = "disable"; + } + if (AuditLogger.isEntityModificationsLoggingEnabled()) { + ExecutiontTaskParameters auditTask = scheduler.get(executionTaskID); + auditLog(auditOperation, auditTask); } } catch (Exception e) { throw new ControllerServiceException(e.getMessage()); @@ -152,7 +160,9 @@ public void enableExecutionTask(@PathParam("id") String executionTaskID, @QueryP @Override @Secured(right = "{entity}-delete") public void delete(String id) { - assertEntityIsEditableInContext(getEntity(id)); + ExecutiontTaskParameters task = getEntity(id); + assertEntityIsEditableInContext(task); + auditLog("delete", task); scheduler.removeExecutionTask(id); } } diff --git a/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java b/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java index dac90d182..1065786bd 100644 --- a/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java +++ b/step-controller/step-controller-server/src/main/java/step/resources/ResourceServices.java @@ -30,10 +30,12 @@ import org.glassfish.jersey.media.multipart.FormDataParam; import step.controller.services.async.AsyncTaskStatus; import step.core.GlobalContext; +import step.core.accessors.AbstractOrganizableObject; import step.core.deployment.AbstractStepAsyncServices; import step.core.deployment.ControllerServiceException; import step.core.entities.EntityConstants; import step.core.objectenricher.ObjectEnricher; +import step.framework.server.audit.AuditLogger; import step.framework.server.security.Secured; import step.framework.server.tables.service.TableService; import step.framework.server.tables.service.bulk.TableBulkOperationReport; @@ -41,8 +43,7 @@ import java.io.IOException; import java.io.InputStream; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Consumer; @Path("/resources") @@ -63,6 +64,17 @@ public void init() throws Exception { tableService = globalContext.require(TableService.class); } + private void auditLog(String operation, Resource resource) { + if (resource == null || !AuditLogger.isEntityModificationsLoggingEnabled()) { + return; + } + String entityName = resource.getAttribute(AbstractOrganizableObject.NAME); + Map attributes = new LinkedHashMap<>(Objects.requireNonNullElse(getObjectEnricher().getAdditionalAttributes(), Map.of())); + attributes.put("resourceType", resource.getResourceType()); + AuditLogger.logEntityModification(getSession(), operation, "resources", resource.getId().toHexString(), entityName, attributes); + } + + @POST @Path("/content") @Secured(right = "resource-write") @@ -89,6 +101,7 @@ trackingAttribute, getSession().getUser().getUsername(), origin == null ? new UploadedResourceOrigin().toStringRepresentation() : origin, originTimestamp ); + auditLog("create", resource); return new ResourceUploadResponse(resource, null); } catch (InvalidResourceFormatException e) { throw uploadFileNotAnArchive(); @@ -104,6 +117,7 @@ private ControllerServiceException uploadFileNotAnArchive() { @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) public Resource saveResource(Resource resource) throws IOException { + auditLog("save", resource); return resourceManager.saveResource(resource); } @@ -119,6 +133,7 @@ public ResourceUploadResponse saveResourceContent(@PathParam("id") String resour try { Resource resource = resourceManager.saveResourceContent(resourceId, uploadedInputStream, fileDetail.getFileName(), null, getSession().getUser().getUsername()); + auditLog("save-content", resource); return new ResourceUploadResponse(resource, null); } catch (InvalidResourceFormatException e) { throw uploadFileNotAnArchive(); @@ -154,6 +169,7 @@ public Response getResourceContent(@PathParam("id") String resourceId, @QueryPar public void deleteResource(@PathParam("id") String resourceId) { Resource resource = resourceManager.getResource(resourceId); assertEntityIsEditableInContext(resource); + auditLog("delete", resource); resourceManager.deleteResource(resourceId); } @@ -164,6 +180,7 @@ public void deleteResource(@PathParam("id") String resourceId) { public void deleteResourceRevisions(@PathParam("id") String resourceId) { Resource resource = resourceManager.getResource(resourceId); assertEntityIsEditableInContext(resource); + auditLog("delete-revisions", resource); resourceManager.deleteResourceRevisionContent(resourceId); } From 8c255deae6fdfef7bbdbb63d9c1cb14714a3d069 Mon Sep 17 00:00:00 2001 From: Jonathan Rubiero <30461894+rubij@users.noreply.github.com> Date: Thu, 15 Jan 2026 14:02:15 +0100 Subject: [PATCH 23/31] EI-485 new nexus staging host --- build_parameters.json | 8 ++++---- pom.xml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build_parameters.json b/build_parameters.json index 7971aa5d0..75f49e6db 100644 --- a/build_parameters.json +++ b/build_parameters.json @@ -10,12 +10,12 @@ "PARAMETERS": [ { "NAME": "DEVELOPMENT", - "URL": "nexus-staging::https://nexus-enterprise.exense.ch/repository/staging-maven/", + "URL": "nexus-staging::https://nexus-enterprise-staging.stepcloud-test.ch/repository/staging-maven/", "CONFIG": "SkipJavadoc" }, { "NAME": "INTEGRATION", - "URL": "nexus-staging::https://nexus-enterprise.exense.ch/repository/staging-maven/", + "URL": "nexus-staging::https://nexus-enterprise-staging.stepcloud-test.ch/repository/staging-maven/", "CONFIG": "PerformanceTest" }, { @@ -32,11 +32,11 @@ "PARAMETERS": [ { "NAME": "DEVELOPMENT", - "URL": "nexus-staging https://nexus-enterprise-staging.exense.ch/repository/staging-npm/" + "URL": "nexus-staging https://nexus-enterprise-staging.stepcloud-test.ch/repository/staging-npm/" }, { "NAME": "INTEGRATION", - "URL": "nexus-staging https://nexus-enterprise-staging.exense.ch/repository/staging-npm/" + "URL": "nexus-staging https://nexus-enterprise-staging.stepcloud-test.ch/repository/staging-npm/" }, { "NAME": "PRODUCTION", diff --git a/pom.xml b/pom.xml index 4b5649d10..f06b6f81b 100644 --- a/pom.xml +++ b/pom.xml @@ -80,7 +80,7 @@ nexus-staging - https://nexus-enterprise.exense.ch/repository/staging-maven/ + https://nexus-enterprise-staging.stepcloud-test.ch/repository/staging-maven/ From d44013c8b8a675a24e209ed66d23cf87dc132977 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 16 Jan 2026 10:12:11 +0100 Subject: [PATCH 24/31] SED-4430 Improve Agent and CLI resilience in case of network errors (#580) * SED-4430 Improve Agent and CLI resilience in case of network errors * SED-4430 Reverting method signatures and correct comment * EI-485 new nexus staging host * SED-4430 bumping grid and FW after PR merge --------- Co-authored-by: Jonathan Rubiero <30461894+rubij@users.noreply.github.com> --- pom.xml | 4 +-- .../cli/ExecuteAutomationPackageToolTest.java | 2 +- .../executions/RemoteExecutionManager.java | 35 ++++++++++++++----- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index f06b6f81b..1312d1da4 100644 --- a/pom.xml +++ b/pom.xml @@ -46,8 +46,8 @@ 11 2025.6.25 - 2.5.0 - 2.5.1 + 0.0.0-25-SNAPSHOT + 0.0.0-25-SNAPSHOT 3.0.23 diff --git a/step-cli/step-cli-core/src/test/java/step/cli/ExecuteAutomationPackageToolTest.java b/step-cli/step-cli-core/src/test/java/step/cli/ExecuteAutomationPackageToolTest.java index 3a626c67d..59938a3e4 100644 --- a/step-cli/step-cli-core/src/test/java/step/cli/ExecuteAutomationPackageToolTest.java +++ b/step-cli/step-cli-core/src/test/java/step/cli/ExecuteAutomationPackageToolTest.java @@ -183,7 +183,7 @@ private static Execution getMockedExecution(ReportNodeStatus resultStatus, Strin return execution; } - private RemoteExecutionManager createExecutionManagerMock(List executions) throws TimeoutException, InterruptedException { + private RemoteExecutionManager createExecutionManagerMock(List executions) throws InterruptedException, TimeoutException { RemoteExecutionManager remoteExecutionManagerMock = Mockito.mock(RemoteExecutionManager.class); Mockito.when(remoteExecutionManagerMock.get(Mockito.any())).thenAnswer(invocationOnMock -> executions.stream().filter(e -> e.getId().toString().equals(invocationOnMock.getArgument(0))).findFirst().get()); Mockito.when(remoteExecutionManagerMock.waitForTermination(Mockito.anyList(), Mockito.anyLong())).thenReturn(executions); diff --git a/step-controller/step-controller-remote-client/src/main/java/step/client/executions/RemoteExecutionManager.java b/step-controller/step-controller-remote-client/src/main/java/step/client/executions/RemoteExecutionManager.java index 8acda042b..e9d6d9acc 100644 --- a/step-controller/step-controller-remote-client/src/main/java/step/client/executions/RemoteExecutionManager.java +++ b/step-controller/step-controller-remote-client/src/main/java/step/client/executions/RemoteExecutionManager.java @@ -19,6 +19,7 @@ package step.client.executions; import ch.exense.commons.io.Poller; +import ch.exense.commons.resilience.RetryHelper; import jakarta.ws.rs.client.Entity; import jakarta.ws.rs.client.Invocation.Builder; import jakarta.ws.rs.core.MediaType; @@ -241,25 +242,43 @@ public Execution waitForTermination(String executionID, long timeout) throws Tim public List waitForTermination(List executionIds, long timeout) throws TimeoutException, InterruptedException { Set pendingExecutions = new HashSet<>(executionIds); + Map completedExecutions = new HashMap<>(); Poller.waitFor(() -> { - Set completed = new HashSet<>(); - for (String e : pendingExecutions) { - if (get(e).getStatus().equals(ExecutionStatus.ENDED)) { - completed.add(e); + //Local set of completed executions found during this polling iteration + Set completedExecutionIDs = new HashSet<>(); + for (String executionId : pendingExecutions) { + Execution execution = getExecutionWithRetryOnError(executionId); + if (execution.getStatus().equals(ExecutionStatus.ENDED)) { + completedExecutionIDs.add(executionId); + completedExecutions.put(executionId, execution); } } - pendingExecutions.removeAll(completed); + //remove the newly completed executions from the pending ones to only process the remaining execution in next polling + pendingExecutions.removeAll(completedExecutionIDs); return pendingExecutions.isEmpty(); }, timeout); + //final iteration for missing execution results List res = new ArrayList<>(); - for (String e : executionIds) { - Execution executionObj = get(e); - res.add(executionObj); + for (String executionId : executionIds) { + Execution execution = completedExecutions.get(executionId); + if (execution == null) { + execution = getExecutionWithRetryOnError(executionId); + } + res.add(execution); } return res; } + private Execution getExecutionWithRetryOnError(String executionId) { + try { + return RetryHelper.executeWithRetryOnExceptions(() -> get(executionId), + 3, 1000, RetryHelper.COMMON_NETWORK_EXCEPTIONS, "Getting execution with ID: " + executionId); + } catch (Exception e) { + throw new RuntimeException("Unable to get execution for " + executionId, e); + } + } + /** * @param executionId the ID of the execution * @return the {@link Execution} From 9ad83df8450bba275fdc9dcc16d6cfd79494dc57 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 19 Jan 2026 09:51:19 +0100 Subject: [PATCH 25/31] SED-4480 BE Junit test flakiness --- .../automation/packages/AutomationPackageManagerOSTest.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java index 4778e0817..e1e2137ff 100644 --- a/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java +++ b/step-automation-packages/step-automation-packages-controller/src/test/java/step/automation/packages/AutomationPackageManagerOSTest.java @@ -1370,6 +1370,12 @@ private SampleUploadingResult uploadSample1WithAsserts(ObjectId explicitOldId, A if (async && expectedDelay) { //The results of createOrUpdateAutomationPackage must have the status UPDATE_DELAYED, and the AP status set to DELAYED_UPDATE assertEquals(AutomationPackageUpdateStatus.UPDATE_DELAYED, updateResult.getStatus()); + //The update being async the change of the AP's status may take some time + Awaitility.await().atMost(Duration.ofSeconds(5)).pollDelay(Duration.ofMillis(50)).until(() -> { + AutomationPackage automationPackage = automationPackageAccessor.get(updateResult.getId()); + log.info("Current status: {}", automationPackage.getStatus()); + return AutomationPackageStatus.DELAYED_UPDATE.equals(automationPackage.getStatus()); + }); assertEquals(AutomationPackageStatus.DELAYED_UPDATE, automationPackageAccessor.get(updateResult.getId()).getStatus()); //We then poll until the AP is updated before continuing with the assertion (its status should be reset to null Awaitility.await().atMost(Duration.ofSeconds(5)).pollDelay(Duration.ofMillis(50)).until(() -> { From 443eb0df8684efcb136d044701208160ce49991b Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 30 Jan 2026 10:33:52 +0100 Subject: [PATCH 26/31] SED-4498 Parameters priority not respected when deployed with AP (#584) * SED-4498 Parameters priority not respected when deployed with AP * SED-4498 PR feedbacks * SED-4498 fixing junit --- .../src/main/java/step/commons/activation/Activator.java | 6 +++++- .../plugins/parametermanager/ParameterManagerTest.java | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/step-commons/src/main/java/step/commons/activation/Activator.java b/step-commons/src/main/java/step/commons/activation/Activator.java index a9109c0a6..61f5e8ebd 100644 --- a/step-commons/src/main/java/step/commons/activation/Activator.java +++ b/step-commons/src/main/java/step/commons/activation/Activator.java @@ -111,7 +111,11 @@ public int compare(T o1, T o2) { } private int getPriority(T o1) { - return o1.getActivationExpression()==null?0:(o1.getPriority()==null?1:o1.getPriority()); + if (o1.getPriority() != null) { + return o1.getPriority(); + } else { + return o1.getActivationExpression() != null ? 1 : 0; + } } }); diff --git a/step-controller/step-controller-base-plugins/src/test/java/step/plugins/parametermanager/ParameterManagerTest.java b/step-controller/step-controller-base-plugins/src/test/java/step/plugins/parametermanager/ParameterManagerTest.java index c2dc1499d..f29663a36 100644 --- a/step-controller/step-controller-base-plugins/src/test/java/step/plugins/parametermanager/ParameterManagerTest.java +++ b/step-controller/step-controller-base-plugins/src/test/java/step/plugins/parametermanager/ParameterManagerTest.java @@ -83,7 +83,7 @@ public void test1Common(Configuration configuration) throws ScriptException { accessor.save(new Parameter(null, "key2", "defaultValue", "desc")); accessor.save(new Parameter(null, "key2", "defaultValue2", "desc")); - accessor.save(new Parameter(new Expression("user=='poire'"), "key2", "defaultValue2", "desc")); + accessor.save(new Parameter(new Expression("user=='poire'"), "key2", "defaultValue3", "desc")); accessor.save(new Parameter(null, "key3", "value1", "desc")); accessor.save(new Parameter(new Expression("user=='poire'"), "key3", "value2", "desc")); @@ -95,9 +95,9 @@ public void test1Common(Configuration configuration) throws ScriptException { bindings.put("user", "poire"); Map params = m.getAllParameterValues(bindings, null); - Assert.assertEquals(params.get("key1"),"poirier"); - Assert.assertEquals(params.get("key2"),"defaultValue2"); - Assert.assertEquals(params.get("key3"),"value3"); + Assert.assertEquals("poirier", params.get("key1")); + Assert.assertEquals("defaultValue3", params.get("key2")); + Assert.assertEquals("value3", params.get("key3")); params = m.getAllParameterValues(bindings, t -> false); Assert.assertEquals(0, params.size()); From ff8d80d4ae472868936cc222dffa194db1a8a074 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 30 Jan 2026 10:40:30 +0100 Subject: [PATCH 27/31] =?UTF-8?q?SED-4506=20Clean-up=20of=20isolated=20AP?= =?UTF-8?q?=20resources=20delete=20all=20revisions=20and=20f=E2=80=A6=20(#?= =?UTF-8?q?585)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SED-4506 Clean-up of isolated AP resources delete all revisions and files but keep the resource entry in DB --- .../packages/execution/IsolatedAutomationPackageRepository.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java index bc9411ac7..ae4921b13 100644 --- a/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java +++ b/step-automation-packages/step-automation-packages-manager/src/main/java/step/automation/packages/execution/IsolatedAutomationPackageRepository.java @@ -179,7 +179,7 @@ public void cleanUpOutdatedResources() { OffsetDateTime lastExecutionTime = OffsetDateTime.parse(lastExecutionTimeStr, DateTimeFormatter.ISO_DATE_TIME); if (lastExecutionTime.isBefore(minExecutionTime)) { log.info("Cleanup the outdated resource for automation package: {} ...", apResourceInfo); - resourceManager.deleteResourceRevisionContent(foundResource.getId().toString()); + resourceManager.deleteResource(foundResource.getId().toString()); removed++; } } else { From c6183e82bc3b6b55fe598e961b16aef2dc88bd05 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Fri, 30 Jan 2026 16:26:15 +0100 Subject: [PATCH 28/31] SED-4451 the performance of the analytics view is very poor on large datasets after re ingestion (#578) * SED-4451 The performance of the analytics view is very poor on large datasets after re-ingestion * SED-4451 The performance of the analytics view is very poor on large datasets after re-ingestion * SED-4451 The performance of the analytics view is very poor on large datasets after re-ingestion * SED-4451 adapting TimeSeriesExecutionPlugin * SED-4451 index optimizations and other perf fixes * SED-4451 bumping framework * SED-4451 PR Feedbacks * SED-4451 adding missing TS index and fixing default resolution * SED-4451 Bumping framework to 0.0.0-25-SNAPSHOT before merge --- .../measurements/MeasurementPlugin.java | 3 +- .../measurements/raw/MeasurementAccessor.java | 10 ++-- .../plugins/timeseries/MetricsConstants.java | 27 ++++++++++- .../TimeSeriesControllerPlugin.java | 23 ++++++++-- .../timeseries/TimeSeriesExecutionPlugin.java | 5 +- .../plugins/timeseries/TimeSeriesService.java | 26 +++++++---- .../timeseries/api/FetchBucketsRequest.java | 9 ++++ .../core/deployment/AbstractStepServices.java | 5 ++ .../tasks/MigrationManagerTasksPlugin.java | 1 + .../tasks/V29_2_TimeSeriesNewIndexes.java | 46 +++++++++++++++++++ .../src/main/java/step/core/Constants.java | 2 +- .../aggregated/ReportNodeTimeSeries.java | 13 ++++++ .../TimeSeriesCollectionsBuilder.java | 2 +- .../TimeSeriesCollectionsSettings.java | 1 + 14 files changed, 150 insertions(+), 23 deletions(-) create mode 100644 step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java diff --git a/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java b/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java index 11110a0b6..2dd2eff31 100644 --- a/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java +++ b/step-controller-plugins/step-controller-plugins-measurement/src/main/java/step/plugins/measurements/MeasurementPlugin.java @@ -56,6 +56,7 @@ public class MeasurementPlugin extends AbstractExecutionEnginePlugin { public static final String SCHEDULE = "schedule"; public static final String TEST_CASE = "testcase"; public static final String EXECUTION_DESCRIPTION = "execution"; + public static final String PROJECT = "project"; public static final String CTX_SCHEDULER_TASK_ID = "$schedulerTaskId"; public static final String CTX_SCHEDULE_NAME = "$scheduleName"; public static final String CTX_EXECUTION_DESCRIPTION = "$executionDescription"; @@ -66,7 +67,7 @@ public class MeasurementPlugin extends AbstractExecutionEnginePlugin { // These are used by the MeasurementControllerPlugin to "reconstruct" measures from measurements, and indicate the // "internal" fields which should NOT be added to the measure data field. Keep this in sync with the fields defined above. - static final Set MEASURE_NOT_DATA_KEYS = Set.of("_id", "project", "projectName", ATTRIBUTE_EXECUTION_ID, RN_ID, + static final Set MEASURE_NOT_DATA_KEYS = Set.of("_id", PROJECT, "projectName", ATTRIBUTE_EXECUTION_ID, RN_ID, ORIGIN, RN_STATUS, PLAN_ID, PLAN, AGENT_URL, TASK_ID, SCHEDULE, TEST_CASE, EXECUTION_DESCRIPTION); // Same use, but for defining which fields SHOULD be directly copied to the top-level fields of a measure. static final Set MEASURE_FIELDS = Set.of(NAME, BEGIN, VALUE, STATUS); diff --git a/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java b/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java index 6928a4fe7..f6fb513b9 100644 --- a/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java +++ b/step-controller-plugins/step-controller-plugins-raw-measurement/src/main/java/step/plugins/measurements/raw/MeasurementAccessor.java @@ -4,13 +4,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import step.core.collections.*; +import step.core.collections.Collection; import step.core.entities.EntityManager; import step.plugins.measurements.MeasurementPlugin; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Stream; public class MeasurementAccessor { @@ -97,4 +95,8 @@ private static Document getRawMeasurement(Map o) { document.remove(MeasurementPlugin.PLAN); return document; } + + public void createOrUpdateCompoundIndex(LinkedHashSet indexFields) { + coll.createOrUpdateCompoundIndex(indexFields); + } } diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java index e7b885334..1387db8c1 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/MetricsConstants.java @@ -5,10 +5,16 @@ import java.util.Arrays; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; public class MetricsConstants { - - public static final MetricAttribute STATUS_ATTRIBUTE = new MetricAttribute() + + // The static method getAllAttributeNames is placed at the end of this class (to make sure all constant are initialized). + // It is used to register all metric as supported timeseries fields. + // WARNING: Do not forget to update this method when adding new constants + + public static final MetricAttribute STATUS_ATTRIBUTE = new MetricAttribute() .setName("rnStatus") .setType(MetricAttributeType.TEXT) .setMetadata(Map.of("knownValues", Arrays.asList("PASSED", "FAILED", "TECHNICAL_ERROR", "INTERRUPTED"))) @@ -49,4 +55,21 @@ public class MetricsConstants { .setName("result") .setType(MetricAttributeType.TEXT) .setDisplayName("Result"); + + + public static String getAllAttributeNames() { + return Stream.of( + STATUS_ATTRIBUTE, + TYPE_ATRIBUTE, + TASK_ATTRIBUTE, + EXECUTION_ATTRIBUTE, + PLAN_ATTRIBUTE, + NAME_ATTRIBUTE, + ERROR_CODE_ATTRIBUTE, + EXECUTION_BOOLEAN_RESULT, + EXECUTION_RESULT + ) + .map(MetricAttribute::getName) + .collect(Collectors.joining(",")); + } } diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java index abcd97250..dcd8cf971 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesControllerPlugin.java @@ -35,6 +35,8 @@ import java.util.*; +import static step.core.timeseries.TimeSeriesConstants.ATTRIBUTES_PREFIX; +import static step.core.timeseries.TimeSeriesConstants.TIMESTAMP_ATTRIBUTE; import static step.plugins.measurements.MeasurementPlugin.ATTRIBUTE_EXECUTION_ID; import static step.plugins.timeseries.MetricsConstants.*; import static step.plugins.timeseries.TimeSeriesExecutionPlugin.*; @@ -45,7 +47,9 @@ public class TimeSeriesControllerPlugin extends AbstractControllerPlugin { private static final Logger logger = LoggerFactory.getLogger(TimeSeriesControllerPlugin.class); public static final String TIME_SERIES_MAIN_COLLECTION = "timeseries"; public static final String TIME_SERIES_ATTRIBUTES_PROPERTY = "timeseries.attributes"; - public static final String TIME_SERIES_ATTRIBUTES_DEFAULT = EXECUTION_ID + "," + TASK_ID + "," + PLAN_ID + ",metricType,origin,name,rnStatus,project,type"; + //We should review the usage of the following property default value, it is technically possible to set the value in properties: not sure if that really work. + //This is used to determine if we fall back to RAW measurement when we filter or group by fields that are not supported by time-series and when reingesting timeseries from RAW measurements + public static final String TIME_SERIES_ATTRIBUTES_DEFAULT = MetricsConstants.getAllAttributeNames() + ",metricType,origin,project"; // Following properties are used by the UI. In the future we could remove the prefix 'plugins.' to align with other properties public static final String PARAM_KEY_EXECUTION_DASHBOARD_ID = "plugins.timeseries.execution.dashboard.id"; @@ -123,10 +127,21 @@ public ExecutionEnginePlugin getExecutionEnginePlugin() { public void initializeData(GlobalContext context) throws Exception { super.initializeData(context); timeSeries.createIndexes(new LinkedHashSet<>(List.of(new IndexField(ATTRIBUTE_EXECUTION_ID, Order.ASC, String.class)))); + IndexField metricTypeIndexField = new IndexField(ATTRIBUTES_PREFIX + METRIC_TYPE, Order.ASC, String.class); + IndexField beginIndexField = new IndexField(TIMESTAMP_ATTRIBUTE, Order.ASC, Long.class); timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( - new IndexField("attributes.taskId", Order.ASC, String.class), - new IndexField("attributes.metricType", Order.ASC, String.class), - new IndexField("begin", Order.ASC, String.class) + metricTypeIndexField, + beginIndexField + ))); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + new IndexField(ATTRIBUTES_PREFIX + TASK_ATTRIBUTE.getName(), Order.ASC, String.class), + metricTypeIndexField, + beginIndexField + ))); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + new IndexField(ATTRIBUTES_PREFIX + PLAN_ATTRIBUTE.getName(), Order.ASC, String.class), + metricTypeIndexField, + beginIndexField ))); List metrics = createOrUpdateMetrics(context.require(MetricTypeAccessor.class)); diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java index e5972f323..9ee61212b 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesExecutionPlugin.java @@ -59,7 +59,10 @@ public void initializeExecutionContext(ExecutionEngineContext executionEngineCon super.initializeExecutionContext(executionEngineContext, executionContext); TimeSeriesIngestionPipeline mainIngestionPipeline = timeSeries.getIngestionPipeline(); TreeMap additionalAttributes = executionContext.getObjectEnricher().getAdditionalAttributes(); - TimeSeriesIngestionPipeline ingestionPipeline = new TimeSeriesIngestionPipeline(null, new TimeSeriesIngestionPipelineSettings()) { + //Crete a wrapper of the ingestion pipeline to automatically enrich data with execution attributes + //This approach is quite error-prone and should be refactored + TimeSeriesIngestionPipeline ingestionPipeline = new TimeSeriesIngestionPipeline(null, + new TimeSeriesIngestionPipelineSettings().setResolution(mainIngestionPipeline.getResolution())) { @Override public void ingestPoint(Map attributes, long timestamp, long value) { attributes.putAll(additionalAttributes); diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java index 72c2b3a3f..010a009ce 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/TimeSeriesService.java @@ -101,14 +101,19 @@ public TimeSeriesAPIResponse getMeasurements(@NotNull FetchBucketsRequest reques } private void enrichRequest(FetchBucketsRequest request) { - request.setOqlFilter(enrichOqlFilter(request.getOqlFilter())); + request.setOqlFilter(enrichOqlFilter(request.getOqlFilter(), request.isIncludeGlobalEntities())); if (request.getMaxNumberOfSeries() <= 0) { request.setMaxNumberOfSeries(maxNumberOfSeries); } } - private String enrichOqlFilter(String oqlFilter) { - String additionalOqlFilter = getObjectFilter().getOQLFilter(); + private String enrichOqlFilter(String oqlFilter, boolean includeGlobalEntities) { + String additionalOqlFilter = ""; + if (includeGlobalEntities) { + additionalOqlFilter = getObjectFilter().getOQLFilter(); + } else { + additionalOqlFilter = getRestrictedObjectFilter().getOQLFilter(); + } if (StringUtils.isNotEmpty(additionalOqlFilter)) { return (StringUtils.isNotEmpty(oqlFilter)) ? oqlFilter + " and (" + additionalOqlFilter + ")" : @@ -155,8 +160,9 @@ public boolean timeSeriesIsBuilt(@PathParam("executionId") String executionId) { @Path("/measurements-fields") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - public Set getMeasurementsAttributes(@QueryParam("filter") String oqlFilter) { - oqlFilter = enrichOqlFilter(oqlFilter); + public Set getMeasurementsAttributes(@QueryParam("filter") String oqlFilter, + @DefaultValue("false") @QueryParam("includeGlobalEntities") boolean includeGlobalEntities) { + oqlFilter = enrichOqlFilter(oqlFilter, includeGlobalEntities); return handler.getMeasurementsAttributes(oqlFilter); } @@ -168,9 +174,10 @@ public Set getMeasurementsAttributes(@QueryParam("filter") String oqlFil public List discoverMeasurements( @QueryParam("filter") String oqlFilter, @QueryParam("limit") int limit, - @QueryParam("skip") int skip + @QueryParam("skip") int skip, + @DefaultValue("false") @QueryParam("includeGlobalEntities") boolean includeGlobalEntities ) { - oqlFilter = enrichOqlFilter(oqlFilter); + oqlFilter = enrichOqlFilter(oqlFilter, includeGlobalEntities); return handler.getRawMeasurements(oqlFilter, skip, limit); } @@ -179,8 +186,9 @@ public List discoverMeasurements( @Path("/raw-measurements/stats") @Consumes(MediaType.APPLICATION_JSON) @Produces(MediaType.APPLICATION_JSON) - public MeasurementsStats getRawMeasurementsStats(@QueryParam("filter") String oqlFilter) { - oqlFilter = enrichOqlFilter(oqlFilter); + public MeasurementsStats getRawMeasurementsStats(@QueryParam("filter") String oqlFilter, + @DefaultValue("false") @QueryParam("includeGlobalEntities") boolean includeGlobalEntities) { + oqlFilter = enrichOqlFilter(oqlFilter, includeGlobalEntities); return handler.getRawMeasurementsStats(oqlFilter); } diff --git a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java index ac04c8763..a22e6e058 100644 --- a/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java +++ b/step-controller-plugins/step-controller-plugins-timeseries/src/main/java/step/plugins/timeseries/api/FetchBucketsRequest.java @@ -14,6 +14,7 @@ public class FetchBucketsRequest { private Set collectAttributeKeys; private int collectAttributesValuesLimit; private int maxNumberOfSeries; + private boolean includeGlobalEntities; //If not set default is false public Long getStart() { return start; @@ -104,4 +105,12 @@ public int getMaxNumberOfSeries() { public void setMaxNumberOfSeries(int maxNumberOfSeries) { this.maxNumberOfSeries = maxNumberOfSeries; } + + public boolean isIncludeGlobalEntities() { + return includeGlobalEntities; + } + + public void setIncludeGlobalEntities(boolean includeGlobalEntities) { + this.includeGlobalEntities = includeGlobalEntities; + } } diff --git a/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java b/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java index 84869aa23..3d59feaa4 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java +++ b/step-controller/step-controller-server/src/main/java/step/core/deployment/AbstractStepServices.java @@ -79,6 +79,11 @@ protected ObjectFilter getObjectFilter() { return objectHookRegistry.getObjectFilter(getSession()); } + protected ObjectFilter getRestrictedObjectFilter() { + Session session = new RestrictedScopeSession(getSession()); + return objectHookRegistry.getObjectFilter(session); + } + protected ObjectPredicate getObjectPredicate(){ return objectHookRegistry.getObjectPredicate(getSession()); } diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java index 1a591be5a..44760db29 100644 --- a/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/MigrationManagerTasksPlugin.java @@ -54,6 +54,7 @@ public void serverStart(GlobalContext context) throws Exception { migrationManager.register(V27_4_DropResolvedPlanNodesIndexForPSQLMigrationTask.class); migrationManager.register(V28_0_FixEmptyDefaultMavenSettingsMigrationTask.class); migrationManager.register(V29_0_UpdateAutomationPackageModel.class); + migrationManager.register(V29_2_TimeSeriesNewIndexes.class); } @Override diff --git a/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java new file mode 100644 index 000000000..f6ece6125 --- /dev/null +++ b/step-controller/step-controller-server/src/main/java/step/migration/tasks/V29_2_TimeSeriesNewIndexes.java @@ -0,0 +1,46 @@ +package step.migration.tasks; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import step.core.Version; + +import step.core.collections.Collection; +import step.core.collections.CollectionFactory; +import step.core.collections.Document; +import step.core.collections.postgresql.PostgreSQLCollection; +import step.migration.MigrationContext; +import step.migration.MigrationTask; + +import java.util.List; + +public class V29_2_TimeSeriesNewIndexes extends MigrationTask { + + + private static final Logger log = LoggerFactory.getLogger(V29_2_TimeSeriesNewIndexes.class); + + public V29_2_TimeSeriesNewIndexes(CollectionFactory collectionFactory, MigrationContext migrationContext) { + super(new Version(3,29,2), collectionFactory, migrationContext); + } + + @Override + public void runUpgradeScript() { + // Compound index created for timeseries collection with metricType, taskId and begin where wrongly using a string type for begin, we need to remove them. + // The correct ones are then created automatically in the initializeData hook as for a fresh setup + // This type of the field is only relevant for the PSQL indexes + List suffix = List.of("", "_minute", "_hour", "_day", "_week"); + suffix.forEach(s -> { + String collectionName = "timeseries" + s; + String indexName = "idx_" + collectionName + "_attributes_taskidasc_attributes_metrictypeasc_beginasc"; + Collection collection = collectionFactory.getCollection(collectionName, Document.class); + if (collection instanceof PostgreSQLCollection) { + collection.dropIndex(indexName); + log.info("Time-series index migration - dropped index {}", indexName); + } + }); + } + + @Override + public void runDowngradeScript() { + + } +} diff --git a/step-core/src/main/java/step/core/Constants.java b/step-core/src/main/java/step/core/Constants.java index 3762b32ae..66c025b3a 100644 --- a/step-core/src/main/java/step/core/Constants.java +++ b/step-core/src/main/java/step/core/Constants.java @@ -19,7 +19,7 @@ package step.core; public interface Constants { - String STEP_API_VERSION_STRING = "3.29.1"; + String STEP_API_VERSION_STRING = "3.29.2"; Version STEP_API_VERSION = new Version(STEP_API_VERSION_STRING); String STEP_YAML_SCHEMA_VERSION_STRING = "1.2.0"; diff --git a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java index 11358d3ee..7d3293176 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java +++ b/step-plans/step-plans-core/src/main/java/step/core/artefacts/reports/aggregated/ReportNodeTimeSeries.java @@ -11,11 +11,15 @@ import step.core.timeseries.bucket.BucketAttributes; import step.core.timeseries.ingestion.TimeSeriesIngestionPipeline; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static step.core.timeseries.TimeSeriesConstants.ATTRIBUTES_PREFIX; +import static step.core.timeseries.TimeSeriesConstants.TIMESTAMP_ATTRIBUTE; + public class ReportNodeTimeSeries implements AutoCloseable { public static final String CONF_KEY_REPORT_NODE_TIME_SERIES_ENABLED = "execution.engine.reportnodes.timeseries.enabled"; @@ -40,6 +44,15 @@ public ReportNodeTimeSeries(CollectionFactory collectionFactory, TimeSeriesColle timeSeries = new TimeSeriesBuilder().registerCollections(timeSeriesCollections).build(); ingestionPipeline = timeSeries.getIngestionPipeline(); timeSeries.createIndexes(Set.of(new IndexField(EXECUTION_ID, Order.ASC, String.class))); + IndexField beginIndexField = new IndexField(TIMESTAMP_ATTRIBUTE, Order.ASC, Long.class); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + new IndexField(ATTRIBUTES_PREFIX + "taskId", Order.ASC, String.class), + beginIndexField + ))); + timeSeries.createCompoundIndex(new LinkedHashSet<>(List.of( + new IndexField(ATTRIBUTES_PREFIX + "planId", Order.ASC, String.class), + beginIndexField + ))); this.ingestionEnabled = ingestionEnabled; } diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java index 5eb8215e0..93e333b8a 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsBuilder.java @@ -48,7 +48,7 @@ public List getTimeSeriesCollections(String mainCollection List enabledCollections = new ArrayList<>(); int flushSeriesQueueSize = collectionsSettings.getFlushSeriesQueueSize(); int flushAsyncQueueSize = collectionsSettings.getFlushAsyncQueueSize(); - addIfEnabled(enabledCollections, mainCollectionName, Duration.ofSeconds(1), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); + addIfEnabled(enabledCollections, mainCollectionName, Duration.ofMillis(collectionsSettings.getMainResolution()), collectionsSettings.getMainFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, true); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_PER_MINUTE, Duration.ofMinutes(1), collectionsSettings.getPerMinuteFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize,null, collectionsSettings.isPerMinuteEnabled()); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_HOURLY, Duration.ofHours(1), collectionsSettings.getHourlyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isHourlyEnabled()); addIfEnabled(enabledCollections, mainCollectionName + TIME_SERIES_SUFFIX_DAILY, Duration.ofDays(1), collectionsSettings.getDailyFlushInterval(), flushSeriesQueueSize, flushAsyncQueueSize, ignoredAttributesForHighResolution, collectionsSettings.isDailyEnabled()); diff --git a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java index de8d2cf70..d027785f5 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java +++ b/step-plans/step-plans-core/src/main/java/step/core/timeseries/TimeSeriesCollectionsSettings.java @@ -215,6 +215,7 @@ public static TimeSeriesCollectionsSettings buildSingleResolutionSettings(long m timeSeriesCollectionsSettings.setWeeklyEnabled(false); timeSeriesCollectionsSettings.setMainResolution(mainResolution); timeSeriesCollectionsSettings.setMainFlushInterval(mainFlushInterval); + timeSeriesCollectionsSettings.setFlushSeriesQueueSize(20000); return timeSeriesCollectionsSettings; } From e459cf60eddd3f84aada18e60515b7dd940f0e75 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 2 Feb 2026 09:47:39 +0100 Subject: [PATCH 29/31] SED-4340 fixing logging --- .../src/main/java/step/core/references/ReferenceFinder.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java index 2e0c19bcc..9ce9885df 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java @@ -46,7 +46,6 @@ public List findReferences(FindReferencesRequest request // Find composite keywords containing requested usages; composite KWs are really just plans in disguise :-) FunctionAccessor functionAccessor = (FunctionAccessor) entityManager.getEntityByName(EntityConstants.functions).getAccessor(); - try (Stream functionStream = functionAccessor.streamLazy()) { functionStream.forEach(function -> { try { @@ -55,7 +54,7 @@ public List findReferences(FindReferencesRequest request results.add(new FindReferencesResponse(function)); } } catch (Exception e) { - logger.error("Unable to find references for function {}", function.getId(), e); + logger.warn("Unable to inspect the keyword {} while searching usages for {}", function.getId(), request.searchValue, e); } }); } @@ -69,7 +68,7 @@ public List findReferences(FindReferencesRequest request results.add(new FindReferencesResponse(plan)); } } catch (Exception e) { - logger.error("Unable to find references for plan {}", plan.getId(), e); + logger.warn("Unable to inspect the plan {} while searching usages for {}", plan.getId(), request.searchValue, e); } }); } From 36fb4ed6b93d66d53aa3ea1ab3b4aad071a4e287 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 2 Feb 2026 09:57:44 +0100 Subject: [PATCH 30/31] SED-4340 fixing error handling --- .../step/core/references/ReferenceFinder.java | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java index 9ce9885df..a92ebde69 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java @@ -48,13 +48,9 @@ public List findReferences(FindReferencesRequest request FunctionAccessor functionAccessor = (FunctionAccessor) entityManager.getEntityByName(EntityConstants.functions).getAccessor(); try (Stream functionStream = functionAccessor.streamLazy()) { functionStream.forEach(function -> { - try { - List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.functions, function, request); - if (!matchingObjects.isEmpty()) { - results.add(new FindReferencesResponse(function)); - } - } catch (Exception e) { - logger.warn("Unable to inspect the keyword {} while searching usages for {}", function.getId(), request.searchValue, e); + List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.functions, function, request); + if (!matchingObjects.isEmpty()) { + results.add(new FindReferencesResponse(function)); } }); } @@ -62,13 +58,9 @@ public List findReferences(FindReferencesRequest request // Find plans containing usages try (Stream stream = (request.includeHiddenPlans) ? planAccessor.streamLazy() : planAccessor.getVisiblePlans()) { stream.forEach(plan -> { - try { - List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.plans, plan, request); - if (!matchingObjects.isEmpty()) { - results.add(new FindReferencesResponse(plan)); - } - } catch (Exception e) { - logger.warn("Unable to inspect the plan {} while searching usages for {}", plan.getId(), request.searchValue, e); + List matchingObjects = getReferencedObjectsMatchingRequest(EntityConstants.plans, plan, request); + if (!matchingObjects.isEmpty()) { + results.add(new FindReferencesResponse(plan)); } }); } @@ -78,7 +70,7 @@ public List findReferences(FindReferencesRequest request return results; } - private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) throws Exception { + private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) { return getReferencedObjects(entityType, object).stream() .filter(o -> (o != null && !o.equals(object))) .filter(o -> doesRequestMatch(request, o)) @@ -86,7 +78,7 @@ private List getReferencedObjectsMatchingRequest(String entityType, Abst } // returns a (generic) set of objects referenced by a plan - private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) throws Exception { + private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) { Set referencedObjects = new HashSet<>(); // The references can be filled in two different ways due to the implementation: @@ -97,7 +89,14 @@ private Set getReferencedObjects(String entityType, AbstractOrganizableO ObjectPredicate predicate = o -> true; //default value for non enricheable objects if (object instanceof EnricheableObject) { AbstractContext context = new AbstractContext() {}; - objectHookRegistry.rebuildContext(context, (EnricheableObject) object); + try { + objectHookRegistry.rebuildContext(context, (EnricheableObject) object); + } catch (Exception e) { + //The getReferencedObjects method is invoked for all entities found in the system, for some entities (for example plans that belongs to a deleted project), the context cannot be rebuilt. + //These expected errors are ignored + logger.warn("Unable to inspect the {} with id {}", entityType, object.getId(), e); + return referencedObjects; + } predicate = objectHookRegistry.getObjectPredicate(context); } EntityDependencyTreeVisitor entityDependencyTreeVisitor = new EntityDependencyTreeVisitor(entityManager, predicate); From 25fd2b821755631f2b5310e41748b35356692973 Mon Sep 17 00:00:00 2001 From: David Stephan Date: Mon, 2 Feb 2026 11:02:17 +0100 Subject: [PATCH 31/31] SED-4430 improving warning message --- .../src/main/java/step/core/references/ReferenceFinder.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java index a92ebde69..fe7374a74 100644 --- a/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java +++ b/step-controller/step-controller-server/src/main/java/step/core/references/ReferenceFinder.java @@ -71,14 +71,14 @@ public List findReferences(FindReferencesRequest request } private List getReferencedObjectsMatchingRequest(String entityType, AbstractOrganizableObject object, FindReferencesRequest request) { - return getReferencedObjects(entityType, object).stream() + return getReferencedObjects(entityType, object, request.searchValue).stream() .filter(o -> (o != null && !o.equals(object))) .filter(o -> doesRequestMatch(request, o)) .collect(Collectors.toList()); } // returns a (generic) set of objects referenced by a plan - private Set getReferencedObjects(String entityType, AbstractOrganizableObject object) { + private Set getReferencedObjects(String entityType, AbstractOrganizableObject object, String searchValue) { Set referencedObjects = new HashSet<>(); // The references can be filled in two different ways due to the implementation: @@ -94,7 +94,7 @@ private Set getReferencedObjects(String entityType, AbstractOrganizableO } catch (Exception e) { //The getReferencedObjects method is invoked for all entities found in the system, for some entities (for example plans that belongs to a deleted project), the context cannot be rebuilt. //These expected errors are ignored - logger.warn("Unable to inspect the {} with id {}", entityType, object.getId(), e); + logger.warn("Unable to inspect the {} with id {} while searching for usages of {}", entityType, object.getId(), searchValue, e); return referencedObjects; } predicate = objectHookRegistry.getObjectPredicate(context);