From e7337b0f3e325b474608ae2de9d187c97ba86e53 Mon Sep 17 00:00:00 2001 From: "luke.campbell" Date: Tue, 24 Jun 2025 09:54:12 +0100 Subject: [PATCH 1/6] feat: add support for opensearch containers --- pom.xml | 16 ++++++++++ .../configuration/ConfigurationKeys.java | 15 +++++++++ .../configuration/ConfigurationLogger.java | 10 ++++++ .../TestcontainersConfiguration.java | 28 +++++++++++++++++ .../management/TestcontainersManager.java | 31 +++++++++++++++++++ .../framework/resource/Resource.java | 1 + ...nfigurationLoaderSystemPropertiesTest.java | 7 +++++ 7 files changed, 108 insertions(+) diff --git a/pom.xml b/pom.xml index 439b6e0..9c10406 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,11 @@ kafka ${testcontainers.version} + + org.opensearch + opensearch-testcontainers + 1.0.0 + org.testcontainers elasticsearch @@ -192,6 +197,17 @@ provided + + org.opensearch.client + opensearch-java + 2.15.0 + + + org.opensearch.client + opensearch-rest-high-level-client + 2.15.0 + + org.elasticsearch elasticsearch diff --git a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java index 10b0090..75bc516 100644 --- a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java +++ b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java @@ -131,6 +131,14 @@ public final class ConfigurationKeys { public static final String ELASTICSEARCH_DISCOVERY_TYPE_KEY = "elasticsearch.discovery.type"; public static final String ELASTICSEARCH_CONTAINER_LOGGING_ENABLED_KEY = "elasticsearch.container.logging.enabled"; + // --- Opensearch configuration keys --- + public static final String OPENSEARCH_ENABLED_KEY = "opensearch.enabled"; + public static final String OPENSEARCH_IMAGE_TAG_KEY = "opensearch.image.tag"; + public static final String OPENSEARCH_PASSWORD_KEY = "opensearch.password"; + public static final String OPENSEARCH_CLUSTER_NAME_KEY = "opensearch.cluster.name"; + public static final String OPENSEARCH_DISCOVERY_TYPE_KEY = "opensearch.discovery.type"; + public static final String OPENSEARCH_CONTAINER_LOGGING_ENABLED_KEY = "opensearch.container.logging.enabled"; + // --- Ambar configuration keys --- public static final String AMBAR_ENABLED_KEY = "ambar.enabled"; public static final String AMBAR_IMAGE_TAG_KEY = "ambar.image.tag"; @@ -251,6 +259,13 @@ public final class ConfigurationKeys { ELASTICSEARCH_DISCOVERY_TYPE_KEY, ELASTICSEARCH_CONTAINER_LOGGING_ENABLED_KEY, + OPENSEARCH_ENABLED_KEY, + OPENSEARCH_IMAGE_TAG_KEY, + OPENSEARCH_PASSWORD_KEY, + OPENSEARCH_CLUSTER_NAME_KEY, + OPENSEARCH_DISCOVERY_TYPE_KEY, + OPENSEARCH_CONTAINER_LOGGING_ENABLED_KEY, + AMBAR_ENABLED_KEY, AMBAR_IMAGE_TAG_KEY, AMBAR_CONTAINER_LOGGING_ENABLED_KEY, diff --git a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java index acaab5c..de01959 100644 --- a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java +++ b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java @@ -169,6 +169,16 @@ public static void log() { log.info("elasticsearch.container.logging.enabled: " + ELASTICSEARCH_CONTAINER_LOGGING_ENABLED); } + log.info("opensearch.enabled: " + OPENSEARCH_ENABLED); + if(OPENSEARCH_ENABLED) { + log.info("opensearch.image.tag: " + OPENSEARCH_IMAGE_TAG); + log.info("opensearch.port: " + OPENSEARCH_PORT); + log.info("opensearch.password: " + OPENSEARCH_PASSWORD); + log.info("opensearch.cluster.name: " + OPENSEARCH_CLUSTER_NAME); + log.info("opensearch.discovery.type: " + OPENSEARCH_DISCOVERY_TYPE); + log.info("opensearch.container.logging.enabled: " + OPENSEARCH_CONTAINER_LOGGING_ENABLED); + } + log.info("ambar.enabled: " + AMBAR_ENABLED); if(AMBAR_ENABLED) { log.info("ambar.image.tag: " + AMBAR_IMAGE_TAG); diff --git a/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java b/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java index 1435a3c..7524de3 100644 --- a/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java +++ b/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java @@ -149,6 +149,15 @@ public final class TestcontainersConfiguration { private static final String DEFAULT_ELASTICSEARCH_CLUSTER_NAME = "elasticsearch"; private static final String DEFAULT_ELASTICSEARCH_DISCOVERY_TYPE = "single-node"; private static final String DEFAULT_ELASTICSEARCH_CONTAINER_LOGGING_ENABLED = "false"; + + // --- Opensearch configuration --- + private static final String DEFAULT_OPENSEARCH_ENABLED = "false"; + private static final String DEFAULT_OPENSEARCH_IMAGE_TAG = "2.15.0"; + private static final String DEFAULT_OPENSEARCH_PORT = "9200"; + private static final String DEFAULT_OPENSEARCH_PASSWORD = null; + private static final String DEFAULT_OPENSEARCH_CLUSTER_NAME = "opensearch"; + private static final String DEFAULT_OPENSEARCH_DISCOVERY_TYPE = "single-node"; + private static final String DEFAULT_OPENSEARCH_CONTAINER_LOGGING_ENABLED = "false"; // --- Ambar configuration --- private static final String DEFAULT_AMBAR_ENABLED = "false"; @@ -300,6 +309,15 @@ public final class TestcontainersConfiguration { public static String ELASTICSEARCH_DISCOVERY_TYPE; public static boolean ELASTICSEARCH_CONTAINER_LOGGING_ENABLED; + // --- Opensearch configuration --- + public static boolean OPENSEARCH_ENABLED; + public static String OPENSEARCH_IMAGE_TAG; + public static int OPENSEARCH_PORT; + public static String OPENSEARCH_PASSWORD; + public static String OPENSEARCH_CLUSTER_NAME; + public static String OPENSEARCH_DISCOVERY_TYPE; + public static boolean OPENSEARCH_CONTAINER_LOGGING_ENABLED; + // --- Ambar configuration --- public static boolean AMBAR_ENABLED; public static String AMBAR_IMAGE_TAG; @@ -460,6 +478,16 @@ public static void configure(Properties properties) { ELASTICSEARCH_DISCOVERY_TYPE = properties.getProperty("elasticsearch.discovery.type", DEFAULT_ELASTICSEARCH_DISCOVERY_TYPE); ELASTICSEARCH_CONTAINER_LOGGING_ENABLED = Boolean.valueOf(properties.getProperty("elasticsearch.container.logging.enabled", DEFAULT_ELASTICSEARCH_CONTAINER_LOGGING_ENABLED)); + // --- Opensearch configuration --- + OPENSEARCH_ENABLED = Boolean.valueOf(properties.getProperty("opensearch.enabled", DEFAULT_OPENSEARCH_ENABLED)); + OPENSEARCH_IMAGE_TAG = properties.getProperty("opensearch.image.tag", DEFAULT_OPENSEARCH_IMAGE_TAG); + // Port cannot be overridden in the Elasticsearch Testcontainer. + OPENSEARCH_PORT = Integer.parseInt(DEFAULT_OPENSEARCH_PORT); + OPENSEARCH_PASSWORD = properties.getProperty("opensearch.password", DEFAULT_OPENSEARCH_PASSWORD); + OPENSEARCH_CLUSTER_NAME = properties.getProperty("opensearch.cluster.name", DEFAULT_OPENSEARCH_CLUSTER_NAME); + OPENSEARCH_DISCOVERY_TYPE = properties.getProperty("opensearch.discovery.type", DEFAULT_OPENSEARCH_DISCOVERY_TYPE); + OPENSEARCH_CONTAINER_LOGGING_ENABLED = Boolean.valueOf(properties.getProperty("opensearch.container.logging.enabled", DEFAULT_OPENSEARCH_CONTAINER_LOGGING_ENABLED)); + // --- Ambar configuration --- AMBAR_ENABLED = Boolean.valueOf(properties.getProperty("ambar.enabled", DEFAULT_AMBAR_ENABLED)); AMBAR_IMAGE_TAG = properties.getProperty("ambar.image.tag", DEFAULT_AMBAR_IMAGE_TAG); diff --git a/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java b/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java index 7ada45c..67a900d 100644 --- a/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java +++ b/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java @@ -24,6 +24,7 @@ import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.security.plain.PlainLoginModule; +import org.opensearch.testcontainers.OpensearchContainer; import org.slf4j.LoggerFactory; import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; @@ -48,6 +49,7 @@ import static dev.lydtech.component.framework.resource.Resource.KAFKA_SCHEMA_REGISTRY; import static dev.lydtech.component.framework.resource.Resource.LOCALSTACK; import static dev.lydtech.component.framework.resource.Resource.MONGODB; +import static dev.lydtech.component.framework.resource.Resource.OPENSEARCH; import static dev.lydtech.component.framework.resource.Resource.POSTGRES; import static dev.lydtech.component.framework.resource.Resource.MARIADB; import static dev.lydtech.component.framework.resource.Resource.WIREMOCK; @@ -73,6 +75,7 @@ public final class TestcontainersManager { private GenericContainer conduktorContainer; private GenericContainer conduktorGatewayContainer; private GenericContainer elasticSearchContainer; + private GenericContainer openSearchContainer; private GenericContainer ambarContainer; private TestcontainersManager(){} @@ -184,6 +187,9 @@ private void createContainers() { if (ELASTICSEARCH_ENABLED) { elasticSearchContainer = createElasticsearchContainer(); } + if (OPENSEARCH_ENABLED && !ELASTICSEARCH_ENABLED) { + openSearchContainer = createOpensearchContainer(); + } if(AMBAR_ENABLED) { ambarContainer = createAmbarContainer(); } @@ -248,6 +254,9 @@ private void startContainers() { if(ELASTICSEARCH_ENABLED) { elasticSearchContainer.start(); } + if(OPENSEARCH_ENABLED) { + openSearchContainer.start(); + } if(AMBAR_ENABLED) { ambarContainer.start(); } @@ -743,6 +752,28 @@ private GenericContainer createElasticsearchContainer() { return container; } + private GenericContainer createOpensearchContainer() { + String containerName = OPENSEARCH.toString(); + DockerImageName opensearchImage = DockerImageName.parse("opensearchproject/opensearch" + ":" + OPENSEARCH_IMAGE_TAG); + OpensearchContainer container = new OpensearchContainer(opensearchImage.withTag(OPENSEARCH_IMAGE_TAG)) + .withNetwork(network) + .withNetworkAliases(containerName) + .withEnv("cluster.name", OPENSEARCH_CLUSTER_NAME) + .withEnv("discovery.type", OPENSEARCH_DISCOVERY_TYPE) + .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true") + .withEnv("DISABLE_SECURITY_PLUGIN", "true") + .withEnv("cluster.routing.allocation.disk.threshold_enabled", "false") + .withReuse(true) + .withCreateContainerCmdModifier(cmd -> { + String containerCmdModifier = CONTAINER_APPEND_GROUP_ID ?CONTAINER_NAME_PREFIX + "-" + containerName + "-" + CONTAINER_GROUP_ID :CONTAINER_NAME_PREFIX + "-" + containerName; + cmd.withName(containerCmdModifier); + }); + if(OPENSEARCH_CONTAINER_LOGGING_ENABLED) { + container.withLogConsumer(getLogConsumer(containerName)); + } + return container; + } + private GenericContainer createAmbarContainer() { String containerName = AMBAR.toString(); GenericContainer container = new GenericContainer<>("ambarltd/emulator:" + AMBAR_IMAGE_TAG) diff --git a/src/main/java/dev/lydtech/component/framework/resource/Resource.java b/src/main/java/dev/lydtech/component/framework/resource/Resource.java index 98a7aa5..d728833 100644 --- a/src/main/java/dev/lydtech/component/framework/resource/Resource.java +++ b/src/main/java/dev/lydtech/component/framework/resource/Resource.java @@ -14,6 +14,7 @@ public enum Resource { LOCALSTACK, MONGODB, ELASTICSEARCH, + OPENSEARCH, MARIADB, AMBAR; diff --git a/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java b/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java index 8595cd5..55e9e26 100644 --- a/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java +++ b/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java @@ -147,6 +147,13 @@ public void clearProperties() { System.clearProperty("elasticsearch.discovery.type"); System.clearProperty("elasticsearch.container.logging.enabled"); + System.clearProperty("opensearch.enabled"); + System.clearProperty("opensearch.image.tag"); + System.clearProperty("opensearch.password"); + System.clearProperty("opensearch.cluster.name"); + System.clearProperty("opensearch.discovery.type"); + System.clearProperty("opensearch.container.logging.enabled"); + System.clearProperty("ambar.enabled"); System.clearProperty("ambar.image.tag"); System.clearProperty("ambar.config.file.path"); From dc5b8259fce0cdc0de8b31ac436984a4e8e813e8 Mon Sep 17 00:00:00 2001 From: "luke.campbell" Date: Tue, 24 Jun 2025 11:13:31 +0100 Subject: [PATCH 2/6] chore: add opensearch readme and re-introduce OpensearchCtfClient --- documentation/README-opensearch.md | 34 +++++++++++++ .../opensearch/OpensearchCtfClient.java | 50 +++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 documentation/README-opensearch.md create mode 100644 src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java diff --git a/documentation/README-opensearch.md b/documentation/README-opensearch.md new file mode 100644 index 0000000..aa3661b --- /dev/null +++ b/documentation/README-opensearch.md @@ -0,0 +1,34 @@ +# Opensearch + +Enable Opensearch via the property `opensearch.enabled`. Opensearch is available on port `9200`. + +_note that `elasticsearch.enabled` must be `false` for the opensearch to container to start_ + +## Related properties + +| Property | Usage | Default | +|--------------------------------------|-----------------------------------------------------------------|---------------| +| opensearch.enabled | Whether a Docker Opensearch container should be started. | `false` | +| opensearch.image.tag | The image tag of the Opensearch Docker container to use. | `2.15.0` | +| opensearch.password | The Opensearch password to use. | | +| opensearch.cluster.name | The name of the Opensearch cluster. | `opensearch` | +| opensearch.discovery.type | Whether to form a single node or multi node Opensearch cluster. | `single-node` | +| opensearch.container.logging.enabled | Whether to output the Opensearch Docker logs to the console. | `false` | + +The container base URL can be obtained using the `OpensearchClient`: + +``` +import dev.lydtech.component.framework.client.open.OpensearchCtfClient; + +String baseUrl = OpensearchCtfClient.getInstance().getBaseUrl(); +``` + +The `org.opensearch.client.opensearch` can be obtained which can then be used to query the dockerised +Opensearch. Note this method is overloaded, also taking a `JsonpMapper`. + +``` +OpenSearchClient osClient = OpensearchCtfClient.getInstance().getOpensearchClient(); + GetResponse getResponse = osClient.get(s -> s + .index("item") + .id(location), Item.class); +``` diff --git a/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java b/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java new file mode 100644 index 0000000..a2e306e --- /dev/null +++ b/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java @@ -0,0 +1,50 @@ +package dev.lydtech.component.framework.client.opensearch; + +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpHost; +import org.opensearch.client.RestClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.rest_client.RestClientTransport; + +@Slf4j +public class OpensearchCtfClient { + + private static OpensearchCtfClient instance; + private static String baseUrl; + + private OpensearchCtfClient() { + String host = Optional.ofNullable(System.getProperty("docker.host")) + .orElse("localhost"); + String port = Optional.ofNullable(System.getProperty("elasticsearch.mapped.port")) + .orElseThrow(() -> new RuntimeException("opensearch.mapped.port property not found")); + baseUrl = "http://" + host + ":" + port; + log.info("Opensearch base URL is: " + baseUrl); + } + + public synchronized static OpensearchCtfClient getInstance() { + if(instance==null) { + instance = new OpensearchCtfClient(); + } + return instance; + } + + public String getBaseUrl() { + return baseUrl; + } + + public OpenSearchClient getOpensearchClient() { + return getOpensearchClient(new JacksonJsonpMapper()); + } + + + public OpenSearchClient getOpensearchClient(JsonpMapper mapper) { + RestClient restClient = RestClient.builder(HttpHost.create(baseUrl)).build(); + OpenSearchTransport transport = new RestClientTransport(restClient, mapper); + return new OpenSearchClient(transport); + } + +} \ No newline at end of file From cd8c3701dfb2b60d6e2374b33ddfbc7e5eca9426 Mon Sep 17 00:00:00 2001 From: "luke.campbell" Date: Tue, 24 Jun 2025 11:14:10 +0100 Subject: [PATCH 3/6] chore: add opensearch readme and re-introduce OpensearchCtfClient --- documentation/README-opensearch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/README-opensearch.md b/documentation/README-opensearch.md index aa3661b..faca653 100644 --- a/documentation/README-opensearch.md +++ b/documentation/README-opensearch.md @@ -18,7 +18,7 @@ _note that `elasticsearch.enabled` must be `false` for the opensearch to contain The container base URL can be obtained using the `OpensearchClient`: ``` -import dev.lydtech.component.framework.client.open.OpensearchCtfClient; +import dev.lydtech.component.framework.client.opensearch.OpensearchCtfClient; String baseUrl = OpensearchCtfClient.getInstance().getBaseUrl(); ``` From 953dd87e806a678914189d17648c7094d8390b0e Mon Sep 17 00:00:00 2001 From: "luke.campbell" Date: Tue, 24 Jun 2025 11:16:21 +0100 Subject: [PATCH 4/6] chore: typo in OpensearchCtfClient --- .../framework/client/opensearch/OpensearchCtfClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java b/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java index a2e306e..dd7623f 100644 --- a/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java +++ b/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java @@ -19,7 +19,7 @@ public class OpensearchCtfClient { private OpensearchCtfClient() { String host = Optional.ofNullable(System.getProperty("docker.host")) .orElse("localhost"); - String port = Optional.ofNullable(System.getProperty("elasticsearch.mapped.port")) + String port = Optional.ofNullable(System.getProperty("opensearch.mapped.port")) .orElseThrow(() -> new RuntimeException("opensearch.mapped.port property not found")); baseUrl = "http://" + host + ":" + port; log.info("Opensearch base URL is: " + baseUrl); From 3207272251fc25809cfae54ffe2afa9eb11eb4d3 Mon Sep 17 00:00:00 2001 From: "luke.campbell" Date: Tue, 24 Jun 2025 11:18:55 +0100 Subject: [PATCH 5/6] chore: update main ReadMe --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 0b1d6b6..5706a2d 100644 --- a/README.md +++ b/README.md @@ -761,6 +761,7 @@ com.github.dockerjava.api.exception.NotFoundException: Status 404: {"message":"c - [Wiremock](./documentation/README-wiremock.md) - [Localstack](./documentation/README-localstack.md) - [ElasticSearch](./documentation/README-elastic.md) +- [OpenSearch](./documentation/README-opensearch.md) - [Ambar](./documentation/README-ambar.md) # Troubleshooting From 51d9f7fb23d8275d4719148f2f96ed3b458fd2a8 Mon Sep 17 00:00:00 2001 From: "luke.campbell" Date: Tue, 24 Jun 2025 13:32:16 +0100 Subject: [PATCH 6/6] feat: add opensearch container support - address PR comments --- CHANGELOG.md | 4 ++++ documentation/README-opensearch.md | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bdb5df..eb8e22a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [3.10.0] - 2025-06-24 +### Added +- Supports Opensearch. + ## [3.9.0] - 2025-02-15 ### Added - Supports loading configuration overrides from a properties or YAML file. diff --git a/documentation/README-opensearch.md b/documentation/README-opensearch.md index faca653..99212dc 100644 --- a/documentation/README-opensearch.md +++ b/documentation/README-opensearch.md @@ -2,7 +2,7 @@ Enable Opensearch via the property `opensearch.enabled`. Opensearch is available on port `9200`. -_note that `elasticsearch.enabled` must be `false` for the opensearch to container to start_ +_note that `elasticsearch.enabled` must be `false` for the Opensearch container to start_ ## Related properties @@ -15,7 +15,7 @@ _note that `elasticsearch.enabled` must be `false` for the opensearch to contain | opensearch.discovery.type | Whether to form a single node or multi node Opensearch cluster. | `single-node` | | opensearch.container.logging.enabled | Whether to output the Opensearch Docker logs to the console. | `false` | -The container base URL can be obtained using the `OpensearchClient`: +The container base URL can be obtained using the `OpensearchCtfClient`: ``` import dev.lydtech.component.framework.client.opensearch.OpensearchCtfClient;