Skip to content

Commit d74743d

Browse files
jsvdjsvd
authored
Update testing to docker (#873)
* create a docker image one for logstash * if it's an integration test create another docker container for Elasticsearch * integration tests run against host 'elasticsearch' so tests were updated * certificates were recreated to validate the 'elasticsearch' host * removed any prior dependency to running tests, only docker is needed * it's possible to pass options to rspec using "SPEC_OPTS=' ...' ./ci/docker-run.sh` * ensure openssl is installed * use return code from logstash container * reduces test matrix to same version testing
1 parent 1216130 commit d74743d

21 files changed

+420
-447
lines changed

‎.ruby-version‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎.travis.yml‎

Lines changed: 17 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,19 @@
11
---
2-
sudo: false
3-
language: ruby
4-
cache: bundler
2+
sudo: required
3+
services: docker
4+
addons:
5+
apt:
6+
packages:
7+
- docker-ce
58
env:
6-
- INTEGRATION=false
7-
- DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=1.7.6 TEST_DEBUG=true
8-
- DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=2.4.4 TEST_DEBUG=true
9-
- DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=5.6.13 TEST_DEBUG=true
10-
- DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=6.0.0 TEST_DEBUG=true
11-
- DISTRIBUTION=oss INTEGRATION=true ES_VERSION=6.6.0 TEST_DEBUG=true
12-
- DISTRIBUTION=default INTEGRATION=true ES_VERSION=6.6.0 TEST_DEBUG=true
13-
- DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-6 TEST_DEBUG=true
14-
- DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-7 ARCH=LINUX TEST_DEBUG=true
15-
- DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-8 ARCH=LINUX TEST_DEBUG=true
16-
- DISTRIBUTION=legacyxpack SECURE_INTEGRATION=true INTEGRATION=true ES_VERSION=5.6.13
17-
TEST_DEBUG=true
18-
- DISTRIBUTION=default SECURE_INTEGRATION=true INTEGRATION=true ES_VERSION=6.6.0 TEST_DEBUG=true
19-
rvm:
20-
- jruby-1.7.25
21-
matrix:
22-
include:
23-
- rvm: jruby-9.1.13.0
24-
env: LOGSTASH_BRANCH=master
25-
- rvm: jruby-9.1.13.0
26-
env: LOGSTASH_BRANCH=7.x
27-
- rvm: jruby-9.1.13.0
28-
env: LOGSTASH_BRANCH=6.7
29-
- rvm: jruby-1.7.27
30-
env: LOGSTASH_BRANCH=5.6
31-
- rvm: jruby-1.7.27
32-
env: DISTRIBUTION=legacy INTEGRATION=true ES_VERSION=5.6.13 LOGSTASH_BRANCH=5.6
33-
TEST_DEBUG=true
34-
- rvm: jruby-9.1.13.0
35-
env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-6 LOGSTASH_BRANCH=6.7
36-
TEST_DEBUG=true
37-
- rvm: jruby-9.1.13.0
38-
env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-7 ARCH=LINUX
39-
LOGSTASH_BRANCH=7.x TEST_DEBUG=true
40-
- rvm: jruby-9.1.13.0
41-
env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-8 ARCH=LINUX
42-
LOGSTASH_BRANCH=master TEST_DEBUG=true
43-
allow_failures:
44-
- env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-8 ARCH=LINUX
45-
TEST_DEBUG=true
46-
- rvm: jruby-9.1.13.0
47-
env: DISTRIBUTION=default INTEGRATION=true ES_VERSION=LATEST-SNAPSHOT-8 ARCH=LINUX
48-
LOGSTASH_BRANCH=master TEST_DEBUG=true
49-
fast_finish: true
50-
install: true
51-
script: ci/build.sh
52-
jdk: openjdk8
53-
before_install: gem install bundler -v '< 2'
9+
- INTEGRATION=false ELASTIC_STACK_VERSION=5.x
10+
- INTEGRATION=false ELASTIC_STACK_VERSION=6.x
11+
- INTEGRATION=false ELASTIC_STACK_VERSION=7.x
12+
- DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=6.x
13+
- DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=7.x
14+
- DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
15+
- DISTRIBUTION=default INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
16+
- DISTRIBUTION=default SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x
17+
- DISTRIBUTION=oss INTEGRATION=true ELASTIC_STACK_VERSION=7.x
18+
install: ci/docker-setup.sh
19+
script: ci/docker-run.sh

‎ci/Dockerfile.elasticsearch‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
ARG elastic_stack_version
2+
ARG distribution_suffix
3+
FROM docker.elastic.co/elasticsearch/elasticsearch$distribution_suffix:$elastic_stack_version
4+
5+
ARG plugin_path=/usr/share/plugins/this
6+
ARG es_path=/usr/share/elasticsearch
7+
ARG es_yml=$es_path/config/elasticsearch.yml
8+
ARG secure_integration
9+
10+
RUN rm -f $es_path/config/scripts
11+
12+
COPY --chown=elasticsearch:elasticsearch spec/fixtures/scripts/groovy/* $es_path/config/scripts/
13+
COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/config/test_certs/
14+
COPY --chown=elasticsearch:elasticsearch ci/elasticsearch-run.sh $es_path/
15+
16+
RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.enabled: $secure_integration" >> $es_yml; fi
17+
RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.key: $es_path/config/test_certs/test.key" >> $es_yml; fi
18+
RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/test.crt" >> $es_yml; fi
19+
RUN if [ "$secure_integration" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi
20+
RUN if [ "$secure_integration" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi
21+
RUN if [ "$secure_integration" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi

‎ci/Dockerfile.logstash‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
ARG elastic_stack_version
2+
ARG distribution_suffix
3+
FROM docker.elastic.co/logstash/logstash$distribution_suffix:$elastic_stack_version
4+
USER root
5+
RUN yum install -y openssl
6+
USER logstash
7+
ENV JARS_SKIP="true"
8+
ENV LOGSTASH_SOURCE=1
9+
ENV LS_JAVA_OPTS="-Xmx256m -Xms256m"
10+
ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin"
11+
COPY --chown=logstash:logstash Gemfile /usr/share/plugins/this/Gemfile
12+
COPY --chown=logstash:logstash *.gemspec /usr/share/plugins/this/
13+
RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml
14+
RUN gem install bundler -v '< 2'
15+
WORKDIR /usr/share/plugins/this
16+
RUN bundle install --path=vendor/bundler
17+
COPY --chown=logstash:logstash . /usr/share/plugins/this
18+
ARG elastic_stack_version
19+
ARG distribution
20+
ARG integration
21+
ARG secure_integration
22+
ENV INTEGRATION $integration
23+
ENV SECURE_INTEGRATION $secure_integration
24+
ENV DISTRIBUTION $distribution
25+
ENV ELASTIC_STACK_VERSION $elastic_stack_version

‎ci/build.sh‎

Lines changed: 0 additions & 21 deletions
This file was deleted.

‎ci/docker-compose.yml‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
version: '3'
2+
3+
# run tests: cd ci; docker-compose up --build --force-recreate
4+
# manual: cd ci; docker-compose run logstash bash
5+
services:
6+
7+
logstash:
8+
build:
9+
context: ../
10+
dockerfile: ci/Dockerfile.logstash
11+
args:
12+
- elastic_stack_version=$ELASTIC_STACK_VERSION
13+
- distribution=${DISTRIBUTION:-default}
14+
- distribution_suffix=${DISTRIBUTION_SUFFIX}
15+
- integration=${INTEGRATION:-false}
16+
- secure_integration=${SECURE_INTEGRATION:-false}
17+
command: /usr/share/plugins/this/ci/logstash-run.sh
18+
environment:
19+
- SPEC_OPTS
20+
tty: true
21+
#volumes:
22+
# - ./:/usr/share/plugins/this
23+
24+
elasticsearch:
25+
build:
26+
context: ../
27+
dockerfile: ci/Dockerfile.elasticsearch
28+
args:
29+
- elastic_stack_version=$ELASTIC_STACK_VERSION
30+
- distribution=${DISTRIBUTION:-default}
31+
- distribution_suffix=${DISTRIBUTION_SUFFIX}
32+
- integration=${INTEGRATION:-false}
33+
- secure_integration=${SECURE_INTEGRATION:-false}
34+
command: /usr/share/elasticsearch/elasticsearch-run.sh
35+
tty: true
36+
ports:
37+
- "9200:9200"
38+
user: elasticsearch
39+
40+
#volumes:
41+
# - ./:/usr/share/plugins/this
42+

‎ci/docker-run.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
3+
# This is intended to be run inside the docker container as the command of the docker-compose.
4+
set -ex
5+
if [ "$INTEGRATION" == "true" ]; then
6+
docker-compose -f ci/docker-compose.yml up --exit-code-from logstash
7+
else
8+
docker-compose -f ci/docker-compose.yml up --exit-code-from logstash logstash
9+
fi

‎ci/docker-setup.sh‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
#!/bin/bash
2+
3+
# This is intended to be run the plugin's root directory. `ci/docker-test.sh`
4+
# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable.
5+
set -e
6+
7+
VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json"
8+
9+
if [ "$ELASTIC_STACK_VERSION" ]; then
10+
echo "Fetching versions from $VERSION_URL"
11+
VERSIONS=$(curl --silent $VERSION_URL)
12+
if [[ "$SNAPSHOT" = "true" ]]; then
13+
ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"')
14+
echo $ELASTIC_STACK_RETRIEVED_VERSION
15+
else
16+
ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"')
17+
fi
18+
if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then
19+
# remove starting and trailing double quotes
20+
ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}"
21+
ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}"
22+
echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}"
23+
export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION
24+
fi
25+
26+
if [[ "$DISTRIBUTION" = "oss" ]]; then
27+
export DISTRIBUTION_SUFFIX="-oss"
28+
else
29+
export DISTRIBUTION_SUFFIX=""
30+
fi
31+
32+
echo "Testing against version: $ELASTIC_STACK_VERSION"
33+
34+
if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then
35+
cd /tmp
36+
37+
jq=".build.projects.\"logstash\".packages.\"logstash$DISTRIBUTION_SUFFIX-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url"
38+
result=$(curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq)
39+
echo $result
40+
curl $result > logstash-docker-image.tar.gz
41+
tar xfvz logstash-docker-image.tar.gz repositories
42+
echo "Loading docker image: "
43+
cat repositories
44+
docker load < logstash-docker-image.tar.gz
45+
rm logstash-docker-image.tar.gz
46+
cd -
47+
48+
if [ "$INTEGRATION" == "true" ]; then
49+
50+
cd /tmp
51+
52+
jq=".build.projects.\"elasticsearch\".packages.\"elasticsearch$DISTRIBUTION_SUFFIX-$ELASTIC_STACK_VERSION-docker-image.tar.gz\".url"
53+
result=$(curl --silent https://artifacts-api.elastic.co/v1/versions/$ELASTIC_STACK_VERSION/builds/latest | jq -r $jq)
54+
echo $result
55+
curl $result > elasticsearch-docker-image.tar.gz
56+
tar xfvz elasticsearch-docker-image.tar.gz repositories
57+
echo "Loading docker image: "
58+
cat repositories
59+
docker load < elasticsearch-docker-image.tar.gz
60+
rm elasticsearch-docker-image.tar.gz
61+
cd -
62+
63+
fi
64+
fi
65+
66+
if [ -f Gemfile.lock ]; then
67+
rm Gemfile.lock
68+
fi
69+
70+
if [ "$INTEGRATION" == "true" ]; then
71+
docker-compose -f ci/docker-compose.yml down
72+
docker-compose -f ci/docker-compose.yml build
73+
else
74+
docker-compose -f ci/docker-compose.yml down
75+
docker-compose -f ci/docker-compose.yml build logstash
76+
fi
77+
else
78+
echo "Please set the ELASTIC_STACK_VERSION environment variable"
79+
echo "For example: export ELASTIC_STACK_VERSION=6.2.4"
80+
exit 1
81+
fi
82+

‎ci/elasticsearch-run.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
/usr/share/elasticsearch/bin/elasticsearch -Ediscovery.type=single-node

‎ci/logstash-run.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
set -ex
3+
4+
export PATH=$BUILD_DIR/gradle/bin:$PATH
5+
6+
wait_for_es() {
7+
echo "Waiting for elasticsearch to respond..."
8+
es_url="http://elasticsearch:9200"
9+
if [[ "$SECURE_INTEGRATION" == "true" ]]; then
10+
es_url="https://elasticsearch:9200 -k"
11+
fi
12+
count=120
13+
while ! curl --silent $es_url && [[ $count -ne 0 ]]; do
14+
count=$(( $count - 1 ))
15+
[[ $count -eq 0 ]] && return 1
16+
sleep 1
17+
done
18+
echo "Elasticsearch is Up !"
19+
20+
return 0
21+
}
22+
23+
if [[ "$INTEGRATION" != "true" ]]; then
24+
bundle exec rspec -fd spec/unit -t ~integration -t ~secure_integration
25+
else
26+
27+
if [[ "$SECURE_INTEGRATION" == "true" ]]; then
28+
extra_tag_args="--tag secure_integration"
29+
else
30+
extra_tag_args="--tag ~secure_integration --tag integration"
31+
fi
32+
33+
if [[ "$DISTRIBUTION" == "oss" ]]; then
34+
extra_tag_args="$extra_tag_args --tag distribution:oss --tag ~distribution:xpack"
35+
elif [[ "$DISTRIBUTION" == "default" ]]; then
36+
extra_tag_args="$extra_tag_args --tag ~distribution:oss --tag distribution:xpack"
37+
fi
38+
wait_for_es
39+
bundle exec rspec -fd $extra_tag_args --tag update_tests:painless --tag update_tests:groovy --tag es_version:$ELASTIC_STACK_VERSION spec/integration
40+
fi

0 commit comments

Comments
 (0)