From 7e6d68409a122c5fdeca7574a149ac90f8f69b88 Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Tue, 15 Jun 2021 11:17:44 +0200 Subject: [PATCH 1/8] upgraded livy to 0.7.1 --- Dockerfile | 15 +++++++++------ init/entrypoint | 6 +++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index e6d0f19..c8d0808 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,19 +30,22 @@ RUN mkdir /var/apache-spark-binaries/ # binaries # apache livy -RUN wget http://mirror.23media.de/apache/incubator/livy/0.5.0-incubating/livy-0.5.0-incubating-bin.zip -O /tmp/livy.zip -RUN unzip /tmp/livy.zip -d /opt/ +RUN wget http://mirror.23media.de/apache/incubator/livy/0.7.1-incubating/apache-livy-0.7.1-incubating-bin.zip -O /tmp/livy.zip +RUN unzip /tmp/livy.zip -d /opt/ && mv /opt/apache-livy-0.7.1-incubating-bin /opt/livy + # Logging dir -RUN mkdir /opt/livy-0.5.0-incubating-bin/logs +RUN mkdir /opt/livy/logs # apache spark -RUN wget https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz -O /tmp/spark-2.3.1-bin-hadoop2.7.tgz -RUN tar -xvzf /tmp/spark-2.3.1-bin-hadoop2.7.tgz -C /opt/ +RUN wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz -O /tmp/spark-2.4.8-bin-hadoop2.7.tgz +RUN tar -xvzf /tmp/spark-2.4.8-bin-hadoop2.7.tgz -C /opt/ && mv /opt/spark-2.4.8-bin-hadoop2.7 /opt/spark # set Python3 as default RUN rm /usr/bin/python RUN ln -s /usr/bin/python3 /usr/bin/python +# install R +RUN DEBIAN_FRONTEND="noninteractive" apt-get install r-base-dev -y # expose ports @@ -50,4 +53,4 @@ EXPOSE 8998 # start from init folder WORKDIR /opt/docker-init -ENTRYPOINT ["./entrypoint"] \ No newline at end of file +ENTRYPOINT ["./entrypoint"] diff --git a/init/entrypoint b/init/entrypoint index 53252eb..3285a44 100755 --- a/init/entrypoint +++ b/init/entrypoint @@ -35,7 +35,7 @@ function writeConfigOptions(){ export SPARK_MASTER_PORT=$SPARK_MASTER_PORT export DEPLOY_MODE=$DEPLOY_MODE - cat /opt/docker-conf/livy.conf | envsubst > /opt/livy-0.5.0-incubating-bin/conf/livy.conf + cat /opt/docker-conf/livy.conf | envsubst > /opt/livy/conf/livy.conf } function init(){ @@ -58,9 +58,9 @@ function init(){ function livy_server_service(){ - export SPARK_HOME=/opt/spark-2.3.1-bin-hadoop2.7/ + export SPARK_HOME=/opt/spark/ echo "starting Livy Server!" - /opt/livy-0.5.0-incubating-bin/bin/livy-server start + /opt/livy/bin/livy-server start # whatever blocking call tail -f /dev/null From bc4bede14eb3cfe0b2d143f2de66debefed7c9db Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Tue, 15 Jun 2021 11:30:42 +0200 Subject: [PATCH 2/8] changed jars directory --- Dockerfile | 2 ++ conf/livy.conf | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c8d0808..2301c4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,8 @@ RUN ln -s /usr/bin/python3 /usr/bin/python # install R RUN DEBIAN_FRONTEND="noninteractive" apt-get install r-base-dev -y +# make jars directory (from livy.conf) +RUN mkdir /jars # expose ports EXPOSE 8998 diff --git a/conf/livy.conf b/conf/livy.conf index 99d3029..75756f5 100644 --- a/conf/livy.conf +++ b/conf/livy.conf @@ -77,7 +77,7 @@ livy.server.session.state-retain.sec = 1800s # List of local directories from where files are allowed to be added to user sessions. By # default it's empty, meaning users can only reference remote URIs when starting their # sessions. -livy.file.local-dir-whitelist = /tmp/ +livy.file.local-dir-whitelist = /jars/ # Whether to enable csrf protection, by default it is false. If it is enabled, client should add # http-header "X-Requested-By" in request if the http method is POST/DELETE/PUT/PATCH. From 75faf985df2b82a8d839875ac72e4e585065e045 Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Fri, 10 Sep 2021 22:41:30 +0200 Subject: [PATCH 3/8] upgraded to spark3 --- Dockerfile | 6 ++++-- build.sh | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 build.sh diff --git a/Dockerfile b/Dockerfile index 2301c4e..835c3b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,8 +37,10 @@ RUN unzip /tmp/livy.zip -d /opt/ && mv /opt/apache-livy-0.7.1-incubating-bin /op RUN mkdir /opt/livy/logs # apache spark -RUN wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz -O /tmp/spark-2.4.8-bin-hadoop2.7.tgz -RUN tar -xvzf /tmp/spark-2.4.8-bin-hadoop2.7.tgz -C /opt/ && mv /opt/spark-2.4.8-bin-hadoop2.7 /opt/spark +RUN wget https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz -O /tmp/spark-3.1.1-bin-hadoop3.2.tgz +#RUN wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz -O /tmp/spark-2.4.8-bin-hadoop2.7.tgz +RUN tar -xvzf /tmp/spark-3.1.1-bin-hadoop3.2.tgz -C /opt/ && mv /opt/spark-3.1.1-bin-hadoop3.2 /opt/spark +#RUN tar -xvzf /tmp/spark-2.4.8-bin-hadoop2.7.tgz -C /opt/ && mv /opt/spark-2.4.8-bin-hadoop2.7 /opt/spark # set Python3 as default RUN rm /usr/bin/python diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..550632e --- /dev/null +++ b/build.sh @@ -0,0 +1,3 @@ +#/bin/sh + +docker build . -t stellars/livy-server-docker From 9077bfa82965d0da7e643aa0e8ba57524f29408d Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Fri, 10 Sep 2021 23:06:19 +0200 Subject: [PATCH 4/8] updated versions --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 835c3b3..ef65aa2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # select operating system FROM ubuntu:16.04 +ENV SPARK_VERSION=spark-3.1.1-bin-hadoop3.2 +ENV LIVY_VERSION=apache-livy-0.7.1-incubating-bin + # install operating system packages RUN apt-get update -y && apt-get install git curl gettext unzip wget software-properties-common python python-software-properties python-pip python3-pip dnsutils make -y From 68d65275e4d39d79eb92556cfb9a6580a9c150a4 Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Sat, 11 Sep 2021 12:57:46 +0200 Subject: [PATCH 5/8] updated mirror for livy --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef65aa2..0e74656 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,7 +33,7 @@ RUN mkdir /var/apache-spark-binaries/ # binaries # apache livy -RUN wget http://mirror.23media.de/apache/incubator/livy/0.7.1-incubating/apache-livy-0.7.1-incubating-bin.zip -O /tmp/livy.zip +RUN wget https://apache.mirrors.tworzy.net/incubator/livy/0.7.1-incubating/apache-livy-0.7.1-incubating-bin.zip -O /tmp/livy.zip RUN unzip /tmp/livy.zip -d /opt/ && mv /opt/apache-livy-0.7.1-incubating-bin /opt/livy # Logging dir From 76a6f9d2c2a0f1ebe41d372f99da1a71b42e2f7f Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Sun, 12 Sep 2021 14:56:29 +0200 Subject: [PATCH 6/8] removed version 3.1.1 from distro build. spark3 is incompatible with livy0.7.1 --- Dockerfile | 13 +++++-------- build.sh | 3 ++- push.sh | 2 ++ 3 files changed, 9 insertions(+), 9 deletions(-) create mode 100644 push.sh diff --git a/Dockerfile b/Dockerfile index 0e74656..ef5a6a9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,6 @@ # select operating system FROM ubuntu:16.04 -ENV SPARK_VERSION=spark-3.1.1-bin-hadoop3.2 -ENV LIVY_VERSION=apache-livy-0.7.1-incubating-bin - # install operating system packages RUN apt-get update -y && apt-get install git curl gettext unzip wget software-properties-common python python-software-properties python-pip python3-pip dnsutils make -y @@ -33,17 +30,17 @@ RUN mkdir /var/apache-spark-binaries/ # binaries # apache livy -RUN wget https://apache.mirrors.tworzy.net/incubator/livy/0.7.1-incubating/apache-livy-0.7.1-incubating-bin.zip -O /tmp/livy.zip +RUN wget https://apache.mirrors.tworzy.net/incubator/livy/0.7.1-incubating/apache-0.7.1-incubating-bin.zip -O /tmp/livy.zip RUN unzip /tmp/livy.zip -d /opt/ && mv /opt/apache-livy-0.7.1-incubating-bin /opt/livy # Logging dir RUN mkdir /opt/livy/logs # apache spark -RUN wget https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz -O /tmp/spark-3.1.1-bin-hadoop3.2.tgz -#RUN wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz -O /tmp/spark-2.4.8-bin-hadoop2.7.tgz -RUN tar -xvzf /tmp/spark-3.1.1-bin-hadoop3.2.tgz -C /opt/ && mv /opt/spark-3.1.1-bin-hadoop3.2 /opt/spark -#RUN tar -xvzf /tmp/spark-2.4.8-bin-hadoop2.7.tgz -C /opt/ && mv /opt/spark-2.4.8-bin-hadoop2.7 /opt/spark +#RUN wget https://archive.apache.org/dist/spark/spark-3.1.1/spark-3.1.1-bin-hadoop3.2.tgz -O /tmp/spark-3.1.1-bin-hadoop3.2.tgz +RUN wget https://archive.apache.org/dist/spark/spark-2.4.8/spark-2.4.8-bin-hadoop2.7.tgz -O /tmp/spark-2.4.8-bin-hadoop2.7.tgz +#RUN tar -xvzf /tmp/spark-3.1.1-bin-hadoop3.2.tgz -C /opt/ && mv /opt/spark-3.1.1-bin-hadoop3.2 /opt/spark +RUN tar -xvzf /tmp/spark-2.4.8-bin-hadoop2.7.tgz -C /opt/ && mv /opt/spark-2.4.8-bin-hadoop2.7 /opt/spark # set Python3 as default RUN rm /usr/bin/python diff --git a/build.sh b/build.sh index 550632e..55b5b3f 100644 --- a/build.sh +++ b/build.sh @@ -1,3 +1,4 @@ #/bin/sh -docker build . -t stellars/livy-server-docker +docker build . -t stellars/livy-server-docker:spark2.4.8 +#docker build . -t stellars/livy-server-docker:spark3.1.1 diff --git a/push.sh b/push.sh new file mode 100644 index 0000000..7f0a0df --- /dev/null +++ b/push.sh @@ -0,0 +1,2 @@ +#/bin/sh +docker push stellars/livy-server-docker:spark3.1.1 From 25fa051bfca17b8081455dacc65301f169ddde18 Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Sun, 12 Sep 2021 15:15:43 +0200 Subject: [PATCH 7/8] fixed typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef5a6a9..5a2be59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN mkdir /var/apache-spark-binaries/ # binaries # apache livy -RUN wget https://apache.mirrors.tworzy.net/incubator/livy/0.7.1-incubating/apache-0.7.1-incubating-bin.zip -O /tmp/livy.zip +RUN wget https://apache.mirrors.tworzy.net/incubator/livy/0.7.1-incubating/apache-livy-0.7.1-incubating-bin.zip -O /tmp/livy.zip RUN unzip /tmp/livy.zip -d /opt/ && mv /opt/apache-livy-0.7.1-incubating-bin /opt/livy # Logging dir From 9594f8c9900bd1f1e3c7d4aa5ef82b1df23609cf Mon Sep 17 00:00:00 2001 From: stellarshenson Date: Sun, 12 Sep 2021 15:17:36 +0200 Subject: [PATCH 8/8] updated pushscript --- push.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/push.sh b/push.sh index 7f0a0df..365ea50 100644 --- a/push.sh +++ b/push.sh @@ -1,2 +1,3 @@ #/bin/sh -docker push stellars/livy-server-docker:spark3.1.1 +#docker push stellars/livy-server-docker:spark3.1.1 +docker push stellars/livy-server-docker:spark2.4.8