diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bdb5df..eb8e22a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [3.10.0] - 2025-06-24 +### Added +- Supports Opensearch. + ## [3.9.0] - 2025-02-15 ### Added - Supports loading configuration overrides from a properties or YAML file. diff --git a/README.md b/README.md index 0b1d6b6..5706a2d 100644 --- a/README.md +++ b/README.md @@ -761,6 +761,7 @@ com.github.dockerjava.api.exception.NotFoundException: Status 404: {"message":"c - [Wiremock](./documentation/README-wiremock.md) - [Localstack](./documentation/README-localstack.md) - [ElasticSearch](./documentation/README-elastic.md) +- [OpenSearch](./documentation/README-opensearch.md) - [Ambar](./documentation/README-ambar.md) # Troubleshooting diff --git a/documentation/README-opensearch.md b/documentation/README-opensearch.md new file mode 100644 index 0000000..99212dc --- /dev/null +++ b/documentation/README-opensearch.md @@ -0,0 +1,34 @@ +# Opensearch + +Enable Opensearch via the property `opensearch.enabled`. Opensearch is available on port `9200`. + +_note that `elasticsearch.enabled` must be `false` for the Opensearch container to start_ + +## Related properties + +| Property | Usage | Default | +|--------------------------------------|-----------------------------------------------------------------|---------------| +| opensearch.enabled | Whether a Docker Opensearch container should be started. | `false` | +| opensearch.image.tag | The image tag of the Opensearch Docker container to use. | `2.15.0` | +| opensearch.password | The Opensearch password to use. | | +| opensearch.cluster.name | The name of the Opensearch cluster. | `opensearch` | +| opensearch.discovery.type | Whether to form a single node or multi node Opensearch cluster. | `single-node` | +| opensearch.container.logging.enabled | Whether to output the Opensearch Docker logs to the console. | `false` | + +The container base URL can be obtained using the `OpensearchCtfClient`: + +``` +import dev.lydtech.component.framework.client.opensearch.OpensearchCtfClient; + +String baseUrl = OpensearchCtfClient.getInstance().getBaseUrl(); +``` + +The `org.opensearch.client.opensearch` can be obtained which can then be used to query the dockerised +Opensearch. Note this method is overloaded, also taking a `JsonpMapper`. + +``` +OpenSearchClient osClient = OpensearchCtfClient.getInstance().getOpensearchClient(); + GetResponse getResponse = osClient.get(s -> s + .index("item") + .id(location), Item.class); +``` diff --git a/pom.xml b/pom.xml index 439b6e0..9c10406 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,11 @@ kafka ${testcontainers.version} + + org.opensearch + opensearch-testcontainers + 1.0.0 + org.testcontainers elasticsearch @@ -192,6 +197,17 @@ provided + + org.opensearch.client + opensearch-java + 2.15.0 + + + org.opensearch.client + opensearch-rest-high-level-client + 2.15.0 + + org.elasticsearch elasticsearch diff --git a/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java b/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java new file mode 100644 index 0000000..dd7623f --- /dev/null +++ b/src/main/java/dev/lydtech/component/framework/client/opensearch/OpensearchCtfClient.java @@ -0,0 +1,50 @@ +package dev.lydtech.component.framework.client.opensearch; + +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpHost; +import org.opensearch.client.RestClient; +import org.opensearch.client.json.JsonpMapper; +import org.opensearch.client.json.jackson.JacksonJsonpMapper; +import org.opensearch.client.opensearch.OpenSearchClient; +import org.opensearch.client.transport.OpenSearchTransport; +import org.opensearch.client.transport.rest_client.RestClientTransport; + +@Slf4j +public class OpensearchCtfClient { + + private static OpensearchCtfClient instance; + private static String baseUrl; + + private OpensearchCtfClient() { + String host = Optional.ofNullable(System.getProperty("docker.host")) + .orElse("localhost"); + String port = Optional.ofNullable(System.getProperty("opensearch.mapped.port")) + .orElseThrow(() -> new RuntimeException("opensearch.mapped.port property not found")); + baseUrl = "http://" + host + ":" + port; + log.info("Opensearch base URL is: " + baseUrl); + } + + public synchronized static OpensearchCtfClient getInstance() { + if(instance==null) { + instance = new OpensearchCtfClient(); + } + return instance; + } + + public String getBaseUrl() { + return baseUrl; + } + + public OpenSearchClient getOpensearchClient() { + return getOpensearchClient(new JacksonJsonpMapper()); + } + + + public OpenSearchClient getOpensearchClient(JsonpMapper mapper) { + RestClient restClient = RestClient.builder(HttpHost.create(baseUrl)).build(); + OpenSearchTransport transport = new RestClientTransport(restClient, mapper); + return new OpenSearchClient(transport); + } + +} \ No newline at end of file diff --git a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java index 10b0090..75bc516 100644 --- a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java +++ b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationKeys.java @@ -131,6 +131,14 @@ public final class ConfigurationKeys { public static final String ELASTICSEARCH_DISCOVERY_TYPE_KEY = "elasticsearch.discovery.type"; public static final String ELASTICSEARCH_CONTAINER_LOGGING_ENABLED_KEY = "elasticsearch.container.logging.enabled"; + // --- Opensearch configuration keys --- + public static final String OPENSEARCH_ENABLED_KEY = "opensearch.enabled"; + public static final String OPENSEARCH_IMAGE_TAG_KEY = "opensearch.image.tag"; + public static final String OPENSEARCH_PASSWORD_KEY = "opensearch.password"; + public static final String OPENSEARCH_CLUSTER_NAME_KEY = "opensearch.cluster.name"; + public static final String OPENSEARCH_DISCOVERY_TYPE_KEY = "opensearch.discovery.type"; + public static final String OPENSEARCH_CONTAINER_LOGGING_ENABLED_KEY = "opensearch.container.logging.enabled"; + // --- Ambar configuration keys --- public static final String AMBAR_ENABLED_KEY = "ambar.enabled"; public static final String AMBAR_IMAGE_TAG_KEY = "ambar.image.tag"; @@ -251,6 +259,13 @@ public final class ConfigurationKeys { ELASTICSEARCH_DISCOVERY_TYPE_KEY, ELASTICSEARCH_CONTAINER_LOGGING_ENABLED_KEY, + OPENSEARCH_ENABLED_KEY, + OPENSEARCH_IMAGE_TAG_KEY, + OPENSEARCH_PASSWORD_KEY, + OPENSEARCH_CLUSTER_NAME_KEY, + OPENSEARCH_DISCOVERY_TYPE_KEY, + OPENSEARCH_CONTAINER_LOGGING_ENABLED_KEY, + AMBAR_ENABLED_KEY, AMBAR_IMAGE_TAG_KEY, AMBAR_CONTAINER_LOGGING_ENABLED_KEY, diff --git a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java index acaab5c..de01959 100644 --- a/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java +++ b/src/main/java/dev/lydtech/component/framework/configuration/ConfigurationLogger.java @@ -169,6 +169,16 @@ public static void log() { log.info("elasticsearch.container.logging.enabled: " + ELASTICSEARCH_CONTAINER_LOGGING_ENABLED); } + log.info("opensearch.enabled: " + OPENSEARCH_ENABLED); + if(OPENSEARCH_ENABLED) { + log.info("opensearch.image.tag: " + OPENSEARCH_IMAGE_TAG); + log.info("opensearch.port: " + OPENSEARCH_PORT); + log.info("opensearch.password: " + OPENSEARCH_PASSWORD); + log.info("opensearch.cluster.name: " + OPENSEARCH_CLUSTER_NAME); + log.info("opensearch.discovery.type: " + OPENSEARCH_DISCOVERY_TYPE); + log.info("opensearch.container.logging.enabled: " + OPENSEARCH_CONTAINER_LOGGING_ENABLED); + } + log.info("ambar.enabled: " + AMBAR_ENABLED); if(AMBAR_ENABLED) { log.info("ambar.image.tag: " + AMBAR_IMAGE_TAG); diff --git a/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java b/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java index 1435a3c..7524de3 100644 --- a/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java +++ b/src/main/java/dev/lydtech/component/framework/configuration/TestcontainersConfiguration.java @@ -149,6 +149,15 @@ public final class TestcontainersConfiguration { private static final String DEFAULT_ELASTICSEARCH_CLUSTER_NAME = "elasticsearch"; private static final String DEFAULT_ELASTICSEARCH_DISCOVERY_TYPE = "single-node"; private static final String DEFAULT_ELASTICSEARCH_CONTAINER_LOGGING_ENABLED = "false"; + + // --- Opensearch configuration --- + private static final String DEFAULT_OPENSEARCH_ENABLED = "false"; + private static final String DEFAULT_OPENSEARCH_IMAGE_TAG = "2.15.0"; + private static final String DEFAULT_OPENSEARCH_PORT = "9200"; + private static final String DEFAULT_OPENSEARCH_PASSWORD = null; + private static final String DEFAULT_OPENSEARCH_CLUSTER_NAME = "opensearch"; + private static final String DEFAULT_OPENSEARCH_DISCOVERY_TYPE = "single-node"; + private static final String DEFAULT_OPENSEARCH_CONTAINER_LOGGING_ENABLED = "false"; // --- Ambar configuration --- private static final String DEFAULT_AMBAR_ENABLED = "false"; @@ -300,6 +309,15 @@ public final class TestcontainersConfiguration { public static String ELASTICSEARCH_DISCOVERY_TYPE; public static boolean ELASTICSEARCH_CONTAINER_LOGGING_ENABLED; + // --- Opensearch configuration --- + public static boolean OPENSEARCH_ENABLED; + public static String OPENSEARCH_IMAGE_TAG; + public static int OPENSEARCH_PORT; + public static String OPENSEARCH_PASSWORD; + public static String OPENSEARCH_CLUSTER_NAME; + public static String OPENSEARCH_DISCOVERY_TYPE; + public static boolean OPENSEARCH_CONTAINER_LOGGING_ENABLED; + // --- Ambar configuration --- public static boolean AMBAR_ENABLED; public static String AMBAR_IMAGE_TAG; @@ -460,6 +478,16 @@ public static void configure(Properties properties) { ELASTICSEARCH_DISCOVERY_TYPE = properties.getProperty("elasticsearch.discovery.type", DEFAULT_ELASTICSEARCH_DISCOVERY_TYPE); ELASTICSEARCH_CONTAINER_LOGGING_ENABLED = Boolean.valueOf(properties.getProperty("elasticsearch.container.logging.enabled", DEFAULT_ELASTICSEARCH_CONTAINER_LOGGING_ENABLED)); + // --- Opensearch configuration --- + OPENSEARCH_ENABLED = Boolean.valueOf(properties.getProperty("opensearch.enabled", DEFAULT_OPENSEARCH_ENABLED)); + OPENSEARCH_IMAGE_TAG = properties.getProperty("opensearch.image.tag", DEFAULT_OPENSEARCH_IMAGE_TAG); + // Port cannot be overridden in the Elasticsearch Testcontainer. + OPENSEARCH_PORT = Integer.parseInt(DEFAULT_OPENSEARCH_PORT); + OPENSEARCH_PASSWORD = properties.getProperty("opensearch.password", DEFAULT_OPENSEARCH_PASSWORD); + OPENSEARCH_CLUSTER_NAME = properties.getProperty("opensearch.cluster.name", DEFAULT_OPENSEARCH_CLUSTER_NAME); + OPENSEARCH_DISCOVERY_TYPE = properties.getProperty("opensearch.discovery.type", DEFAULT_OPENSEARCH_DISCOVERY_TYPE); + OPENSEARCH_CONTAINER_LOGGING_ENABLED = Boolean.valueOf(properties.getProperty("opensearch.container.logging.enabled", DEFAULT_OPENSEARCH_CONTAINER_LOGGING_ENABLED)); + // --- Ambar configuration --- AMBAR_ENABLED = Boolean.valueOf(properties.getProperty("ambar.enabled", DEFAULT_AMBAR_ENABLED)); AMBAR_IMAGE_TAG = properties.getProperty("ambar.image.tag", DEFAULT_AMBAR_IMAGE_TAG); diff --git a/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java b/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java index 7ada45c..67a900d 100644 --- a/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java +++ b/src/main/java/dev/lydtech/component/framework/management/TestcontainersManager.java @@ -24,6 +24,7 @@ import org.apache.kafka.clients.admin.NewTopic; import org.apache.kafka.common.config.SaslConfigs; import org.apache.kafka.common.security.plain.PlainLoginModule; +import org.opensearch.testcontainers.OpensearchContainer; import org.slf4j.LoggerFactory; import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; @@ -48,6 +49,7 @@ import static dev.lydtech.component.framework.resource.Resource.KAFKA_SCHEMA_REGISTRY; import static dev.lydtech.component.framework.resource.Resource.LOCALSTACK; import static dev.lydtech.component.framework.resource.Resource.MONGODB; +import static dev.lydtech.component.framework.resource.Resource.OPENSEARCH; import static dev.lydtech.component.framework.resource.Resource.POSTGRES; import static dev.lydtech.component.framework.resource.Resource.MARIADB; import static dev.lydtech.component.framework.resource.Resource.WIREMOCK; @@ -73,6 +75,7 @@ public final class TestcontainersManager { private GenericContainer conduktorContainer; private GenericContainer conduktorGatewayContainer; private GenericContainer elasticSearchContainer; + private GenericContainer openSearchContainer; private GenericContainer ambarContainer; private TestcontainersManager(){} @@ -184,6 +187,9 @@ private void createContainers() { if (ELASTICSEARCH_ENABLED) { elasticSearchContainer = createElasticsearchContainer(); } + if (OPENSEARCH_ENABLED && !ELASTICSEARCH_ENABLED) { + openSearchContainer = createOpensearchContainer(); + } if(AMBAR_ENABLED) { ambarContainer = createAmbarContainer(); } @@ -248,6 +254,9 @@ private void startContainers() { if(ELASTICSEARCH_ENABLED) { elasticSearchContainer.start(); } + if(OPENSEARCH_ENABLED) { + openSearchContainer.start(); + } if(AMBAR_ENABLED) { ambarContainer.start(); } @@ -743,6 +752,28 @@ private GenericContainer createElasticsearchContainer() { return container; } + private GenericContainer createOpensearchContainer() { + String containerName = OPENSEARCH.toString(); + DockerImageName opensearchImage = DockerImageName.parse("opensearchproject/opensearch" + ":" + OPENSEARCH_IMAGE_TAG); + OpensearchContainer container = new OpensearchContainer(opensearchImage.withTag(OPENSEARCH_IMAGE_TAG)) + .withNetwork(network) + .withNetworkAliases(containerName) + .withEnv("cluster.name", OPENSEARCH_CLUSTER_NAME) + .withEnv("discovery.type", OPENSEARCH_DISCOVERY_TYPE) + .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true") + .withEnv("DISABLE_SECURITY_PLUGIN", "true") + .withEnv("cluster.routing.allocation.disk.threshold_enabled", "false") + .withReuse(true) + .withCreateContainerCmdModifier(cmd -> { + String containerCmdModifier = CONTAINER_APPEND_GROUP_ID ?CONTAINER_NAME_PREFIX + "-" + containerName + "-" + CONTAINER_GROUP_ID :CONTAINER_NAME_PREFIX + "-" + containerName; + cmd.withName(containerCmdModifier); + }); + if(OPENSEARCH_CONTAINER_LOGGING_ENABLED) { + container.withLogConsumer(getLogConsumer(containerName)); + } + return container; + } + private GenericContainer createAmbarContainer() { String containerName = AMBAR.toString(); GenericContainer container = new GenericContainer<>("ambarltd/emulator:" + AMBAR_IMAGE_TAG) diff --git a/src/main/java/dev/lydtech/component/framework/resource/Resource.java b/src/main/java/dev/lydtech/component/framework/resource/Resource.java index 98a7aa5..d728833 100644 --- a/src/main/java/dev/lydtech/component/framework/resource/Resource.java +++ b/src/main/java/dev/lydtech/component/framework/resource/Resource.java @@ -14,6 +14,7 @@ public enum Resource { LOCALSTACK, MONGODB, ELASTICSEARCH, + OPENSEARCH, MARIADB, AMBAR; diff --git a/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java b/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java index 8595cd5..55e9e26 100644 --- a/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java +++ b/src/test/java/dev/lydtech/component/framework/configuration/ConfigurationLoaderSystemPropertiesTest.java @@ -147,6 +147,13 @@ public void clearProperties() { System.clearProperty("elasticsearch.discovery.type"); System.clearProperty("elasticsearch.container.logging.enabled"); + System.clearProperty("opensearch.enabled"); + System.clearProperty("opensearch.image.tag"); + System.clearProperty("opensearch.password"); + System.clearProperty("opensearch.cluster.name"); + System.clearProperty("opensearch.discovery.type"); + System.clearProperty("opensearch.container.logging.enabled"); + System.clearProperty("ambar.enabled"); System.clearProperty("ambar.image.tag"); System.clearProperty("ambar.config.file.path");