This is an automated email from the ASF dual-hosted git repository. lresende pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-toree.git
commit 3846b04f95a7c6a80b83139a6daf931c68e7ca13 Author: Luciano Resende <[email protected]> AuthorDate: Fri Aug 14 21:11:49 2020 -0700 Update docker images to use Spark 3.0.2 --- Dockerfile | 17 +++++++---------- Dockerfile.toree-dev | 15 ++++++++------- Makefile | 2 +- Vagrantfile | 2 +- etc/kernel.json | 2 +- 5 files changed, 18 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index 54dad17..dbbd8bb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - && \ npm install -g bower # for Apache Spark demos -ENV APACHE_SPARK_VERSION 2.3.4 +ENV APACHE_SPARK_VERSION 3.0.2 RUN apt-get -y update && \ apt-get -y install software-properties-common @@ -46,11 +46,11 @@ RUN echo "===> install Java" && \ update-java-alternatives -s java-8-oracle RUN cd /tmp && \ - wget -q http://apache.claz.org/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz && \ - tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz -C /usr/local && \ - rm spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz + wget -q http://apache.claz.org/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7.tgz && \ + tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7.tgz -C /usr/local && \ + rm spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7.tgz -RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6 spark +RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7 spark # R support RUN apt-get update && \ @@ -61,16 +61,13 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* ENV SPARK_HOME /usr/local/spark -ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.1-src.zip +ENV PYTHONPATH $SPARK_HOME/python:$SPARK_HOME/python/lib/py4j-0.10.9-src.zip ENV PYSPARK_PYTHON /home/main/anaconda2/envs/python3/bin/python ENV R_LIBS_USER $SPARK_HOME/R/lib USER main -ENV DASHBOARDS_VERSION ==0.4.1 -ENV DASHBOARDS_BUNDLERS_VERSION ==0.2.2 - -ENV TOREE_VERSION >=0.1.0.dev0, <=0.1.0 +ENV TOREE_VERSION >=0.6.0.dev1, <=0.6.0 # get to the latest jupyter release and necessary libraries RUN conda install -y jupyter seaborn futures && \ diff --git a/Dockerfile.toree-dev b/Dockerfile.toree-dev index fe0504d..409e19c 100644 --- a/Dockerfile.toree-dev +++ b/Dockerfile.toree-dev @@ -23,24 +23,25 @@ FROM jupyter/all-spark-notebook USER root # Spark dependencies -ENV APACHE_SPARK_VERSION 2.3.4 +ENV APACHE_SPARK_VERSION 3.0.2 RUN apt-get -y update && \ - apt-get install -y --no-install-recommends openjdk-8-jre-headless && \ + apt-get install -y --no-install-recommends openjdk-8-jdk ca-certificates-java && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* && \ + update-ca-certificates -f && \ update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java -# Installing Spark2 +# Installing Spark3 RUN cd /tmp && \ - wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz && \ - tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz -C /usr/local && \ - rm spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6.tgz + wget -q https://archive.apache.org/dist/spark/spark-${APACHE_SPARK_VERSION}/spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7.tgz && \ + tar xzf spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7.tgz -C /usr/local && \ + rm spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7.tgz # Overwrite symlink RUN cd /usr/local && \ rm spark && \ - ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop2.6 spark + ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop2.7 spark # Remove other scala kernels RUN cd /opt/conda/share/jupyter/kernels/ && \ diff --git a/Makefile b/Makefile index 4cd7a27..5e1e2a6 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ IS_SNAPSHOT?=true SNAPSHOT:=-SNAPSHOT endif -APACHE_SPARK_VERSION?=3.0.0 +APACHE_SPARK_VERSION?=3.0.2 SCALA_VERSION?=2.12 IMAGE?=jupyter/all-spark-notebook:latest EXAMPLE_IMAGE?=apache/toree-examples diff --git a/Vagrantfile b/Vagrantfile index 2f9d804..f419d00 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -136,7 +136,7 @@ cat << EOF > /home/vagrant/.ipython/kernels/toree-kernel/kernel.json "CAPTURE_STANDARD_ERR": "true", "SEND_EMPTY_OUTPUT": "false", "SPARK_HOME": "/opt/spark", - "PYTHONPATH": "/opt/spark/python:/opt/spark/python/lib/py4j-0.8.2.1-src.zip" + "PYTHONPATH": "/opt/spark/python:/opt/spark/python/lib/py4j-0.10.9-src.zip" } } EOF diff --git a/etc/kernel.json b/etc/kernel.json index dfcf9ee..a428a57 100644 --- a/etc/kernel.json +++ b/etc/kernel.json @@ -4,7 +4,7 @@ }, "display_name": "Apache Toree (development)", "env": { - "PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10-1-src.zip", + "PYTHONPATH": "/usr/local/spark/python:/usr/local/spark/python/lib/py4j-0.10.9-src.zip", "SPARK_HOME": "/usr/local/spark", "CAPTURE_STANDARD_ERR": "true", "MAX_INTERPRETER_THREADS": "16",
