fgreg closed pull request #51: SDAP-137 Webapp should have a mesos and a
standalone image
URL: https://github.com/apache/incubator-sdap-nexus/pull/51
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/analysis/.gitignore b/analysis/.gitignore
index 2081316..559285c 100644
--- a/analysis/.gitignore
+++ b/analysis/.gitignore
@@ -1,3 +1,5 @@
+*.nc
+
# Created by .ignore support plugin (hsz.mobi)
### VirtualEnv template
# Virtualenv
@@ -194,4 +196,3 @@ com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
-
diff --git a/docker/nexus-webapp/Readme.md b/docker/nexus-webapp/Readme.md
new file mode 100644
index 0000000..7aeb962
--- /dev/null
+++ b/docker/nexus-webapp/Readme.md
@@ -0,0 +1,41 @@
+
+# How to Build
+
+All docker builds should happen from this directory. For copy/paste ability,
first export the environment variable `BUILD_VERSION` to the version number you
would like to tag images as.
+
+## spark-mesos-base
+
+ docker build -t sdap/spark-mesos-base:${BUILD_VERSION} -f
mesos/base/Dockerfile .
+
+## spark-mesos-master
+
+Builds from `spark-mesos-base` and supports `tag_version` build argument which
specifies the version of base to build from.
+
+ docker build -t sdap/spark-mesos-master:${BUILD_VERSION} -f
mesos/master/Dockerfile .
+
+## spark-mesos-agent
+
+Builds from `spark-mesos-base` and supports `tag_version` build argument which
specifies the version of base to build from.
+
+ docker build -t sdap/spark-mesos-agent:${BUILD_VERSION} -f
mesos/agent/Dockerfile .
+
+## nexus-webapp:mesos
+
+Builds from `spark-mesos-base` and supports `tag_version` build argument which
specifies the version of base to build from.
+
+ docker build -t sdap/nexus-webapp:mesos.${BUILD_VERSION} -f
mesos/webapp/Dockerfile .
+
+## nexus-webapp:standalone
+
+ docker build -t sdap/nexus-webapp:standalone.${BUILD_VERSION} -f
standalone/Dockerfile .
+
+# Push Images
+
+Push the images to the `sdap` organization on DockerHub
+
+ docker push sdap/spark-mesos-base:${BUILD_VERSION}
+ docker push sdap/spark-mesos-master:${BUILD_VERSION}
+ docker push sdap/spark-mesos-agent:${BUILD_VERSION}
+ docker push sdap/nexus-webapp:mesos.${BUILD_VERSION}
+ docker push sdap/nexus-webapp:standalone.${BUILD_VERSION}
+
diff --git a/docker/spark-mesos-base/install_nexusproto.sh
b/docker/nexus-webapp/install_conda.sh
similarity index 70%
rename from docker/spark-mesos-base/install_nexusproto.sh
rename to docker/nexus-webapp/install_conda.sh
index ce44c70..7d99ef2 100755
--- a/docker/spark-mesos-base/install_nexusproto.sh
+++ b/docker/nexus-webapp/install_conda.sh
@@ -13,23 +13,15 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -e
-APACHE_NEXUSPROTO="https://github.com/apache/incubator-sdap-nexusproto.git";
-MASTER="master"
+set -ebx
-GIT_REPO=${1:-$APACHE_NEXUSPROTO}
-GIT_BRANCH=${2:-$MASTER}
+URL=${1:-"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh"}
+CONDA=${2:-"/usr/local/anaconda2"}
-mkdir nexusproto
-pushd nexusproto
-git init
-git pull ${GIT_REPO} ${GIT_BRANCH}
-
-./gradlew pythonInstall --info
-
-./gradlew install --info
-
-rm -rf /root/.gradle
+pushd /tmp
+wget -q ${URL} -O install_anaconda.sh
+/bin/bash install_anaconda.sh -b -p ${CONDA}
+rm install_anaconda.sh
+${CONDA}/bin/conda update -n base conda
popd
-rm -rf nexusproto
diff --git a/docker/nexus-webapp/install_java.sh
b/docker/nexus-webapp/install_java.sh
new file mode 100755
index 0000000..b9ce92b
--- /dev/null
+++ b/docker/nexus-webapp/install_java.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ebx
+
+URL=${1:-"http://download.oracle.com/otn-pub/java/jdk/8u191-b12/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64.rpm"}
+RPM_PACKAGE=${URL##*/}
+
+# Install Oracle JDK
+wget -q --no-cookies --no-check-certificate --header "Cookie:
gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie" ${URL}
+yum install -y ${RPM_PACKAGE}
+rm ${RPM_PACKAGE}
diff --git a/docker/nexus-webapp/install_spark.sh
b/docker/nexus-webapp/install_spark.sh
new file mode 100755
index 0000000..1e668cc
--- /dev/null
+++ b/docker/nexus-webapp/install_spark.sh
@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ebx
+
+DL_HOST=${1:-"http://d3kbcqa49mib13.cloudfront.net"}
+VERSION=${2:-"2.2.0"}
+DIR=${3:-"spark-${VERSION}"}
+INSTALL_DIR=${4:-"/usr/local"}
+
+pushd ${INSTALL_DIR}
+wget --quiet ${DL_HOST}/spark-${VERSION}-bin-hadoop2.7.tgz
+tar -xzf spark-${VERSION}-bin-hadoop2.7.tgz
+chown -R root.root spark-${VERSION}-bin-hadoop2.7.tgz
+ln -s spark-${VERSION}-bin-hadoop2.7 ${DIR}
+# Do not remove the package, mesos requires it
+# rm spark-${VERSION}-bin-hadoop2.7.tgz
+popd
diff --git a/docker/nexus-webapp/mesos/agent/Dockerfile
b/docker/nexus-webapp/mesos/agent/Dockerfile
new file mode 100644
index 0000000..feca35f
--- /dev/null
+++ b/docker/nexus-webapp/mesos/agent/Dockerfile
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG tag_version=1.0.0-SNAPSHOT
+FROM sdap/spark-mesos-base:${tag_version}
+
+MAINTAINER Apache SDAP "dev@sdap.apache.org"
+
+# Run a Mesos slave.
+
+WORKDIR /tmp
+
+COPY mesos/agent/docker-entrypoint.sh /tmp/docker-entrypoint.sh
+
+
+# This will run docker-entrypoint.sh with the value of CMD as default
arguments. However, if any arguments are supplied
+# to the docker run command when launching this image, the command line
arguments will override these CMD arguments
+ENTRYPOINT ["/bin/bash", "-c", "/tmp/docker-entrypoint.sh $(eval echo $@)",
"$@"]
+CMD ["--master=${MESOS_MASTER_NAME}:${MESOS_MASTER_PORT}",
"--port=${MESOS_AGENT_PORT}", "--work_dir=${MESOS_WORKDIR}"]
diff --git a/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
new file mode 100755
index 0000000..2228b0d
--- /dev/null
+++ b/docker/nexus-webapp/mesos/agent/docker-entrypoint.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ebx
+
+if [ -n "$TORNADO_HOST" ]; then
+ sed -i "s/server.socket_host = .*/server.socket_host = '${TORNADO_HOST}'/g"
${NEXUS_SRC}/analysis/webservice/config/web.ini
+fi
+sed -i "s/host=127.0.0.1/host=$CASSANDRA_CONTACT_POINTS/g"
${NEXUS_SRC}/data-access/nexustiles/config/datastores.ini && \
+sed -i "s/local_datacenter=.*/local_datacenter=$CASSANDRA_LOCAL_DATACENTER/g"
${NEXUS_SRC}/data-access/nexustiles/config/datastores.ini && \
+sed -i "s/host=localhost:8983/host=$SOLR_URL_PORT/g"
${NEXUS_SRC}/data-access/nexustiles/config/datastores.ini
+
+# DOMS
+sed -i
"s/module_dirs=.*/module_dirs=webservice.algorithms,webservice.algorithms_spark,webservice.algorithms.doms/g"
${NEXUS_SRC}/analysis/webservice/config/web.ini && \
+sed -i "s/host=.*/host=$CASSANDRA_CONTACT_POINTS/g"
${NEXUS_SRC}/analysis/webservice/algorithms/doms/domsconfig.ini && \
+sed -i "s/local_datacenter=.*/local_datacenter=$CASSANDRA_LOCAL_DATACENTER/g"
${NEXUS_SRC}/analysis/webservice/algorithms/doms/domsconfig.ini
+
+cd ${NEXUS_SRC}/data-access
+python setup.py install --force
+
+cd ${NEXUS_SRC}/analysis
+python setup.py install --force
+
+# Set PROJ_LIB env var as workaround for missing environment variable for
basemap https://github.com/conda-forge/basemap-feedstock/issues/30
+${MESOS_HOME}/build/bin/mesos-agent.sh --no-systemd_enable_support
--launcher=posix --no-switch_user
--executor_environment_variables='{"PYTHON_EGG_CACHE": "/tmp",
"PROJ_LIB":"/usr/local/anaconda2/share/proj"}' "$@"
diff --git a/docker/nexus-webapp/mesos/base/Dockerfile
b/docker/nexus-webapp/mesos/base/Dockerfile
new file mode 100644
index 0000000..2cce5d1
--- /dev/null
+++ b/docker/nexus-webapp/mesos/base/Dockerfile
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM centos:7
+
+MAINTAINER Apache SDAP "dev@sdap.apache.org"
+
+WORKDIR /tmp
+
+RUN yum -y update && \
+ yum -y install \
+ bzip2 \
+ gcc \
+ git \
+ mesa-libGL.x86_64 \
+ python-devel \
+ wget \
+ which && \
+ yum clean all
+
+# Set environment variables. For Mesos, I used MESOS_VER because MESOS_VERSION
+# is expected to be a logical TRUE/FALSE flag that tells Mesos whether or not
+# to simply print the version number and exit.
+
+ENV INSTALL_LOC=/usr/local \
+ HADOOP_VERSION=2.7.3 \
+ SPARK_VERSION=2.2.0 \
+ MESOS_VER=1.5.1 \
+ MESOS_MASTER_PORT=5050 \
+ MESOS_AGENT_PORT=5051 \
+ MESOS_WORKDIR=/var/lib/mesos \
+ MESOS_IP=0.0.0.0 \
+ MESOS_MASTER_NAME=mesos-master \
+ PYTHON_EGG_CACHE=/tmp \
+ JAVA_HOME=/usr/java/default \
+ NEXUS_SRC=/tmp/incubator-sdap-nexus
+
+ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \
+ MESOS_HOME=${INSTALL_LOC}/mesos-${MESOS_VER} \
+ SPARK_DIR=spark-${SPARK_VERSION} \
+ SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \
+ MESOS_MASTER=mesos://${MESOS_IP}:${MESOS_PORT} \
+ MESOS_PACKAGE=mesos-${MESOS_VER}.tar.gz
+
+ENV SPARK_HOME=${INSTALL_LOC}/${SPARK_DIR} \
+ PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/bin/python \
+ PYSPARK_PYTHON=${CONDA_HOME}/bin/python \
+ PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \
+ # Workaround for missing environment variable for basemap
https://github.com/conda-forge/basemap-feedstock/issues/30
+ PROJ_LIB=${CONDA_HOME}/share/proj
+
+ENV
PYTHONPATH=${PYTHONPATH}:${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${SPARK_HOME}/python/lib/pyspark.zip
\
+ MESOS_NATIVE_JAVA_LIBRARY=${INSTALL_LOC}/lib/libmesos.so \
+ SPARK_EXECUTOR_URI=${INSTALL_LOC}/${SPARK_PACKAGE}.tgz \
+ PATH=${CONDA_HOME}/bin:${PATH}
+
+# Install Oracle JDK
+COPY install_java.sh ./install_java.sh
+RUN ./install_java.sh
"http://download.oracle.com/otn-pub/java/jdk/8u191-b12/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64.rpm";
+
+# Set up Mesos
+COPY mesos/base/install_mesos.sh ./install_mesos.sh
+RUN source ./install_mesos.sh
+
+# Install Spark
+COPY install_spark.sh ./install_spark.sh
+RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net"; ${SPARK_VERSION}
${SPARK_DIR} ${INSTALL_LOC}
+
+# Install Anaconda
+COPY install_conda.sh ./install_conda.sh
+RUN ./install_conda.sh
"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh";
${CONDA_HOME}
+
+# Conda dependencies for nexus
+RUN conda install -c conda-forge -y netCDF4 && \
+ conda install -y numpy cython mpld3 scipy basemap gdal matplotlib && \
+ pip install shapely==1.5.16 cassandra-driver==3.5.0 && \
+ conda install -c conda-forge backports.functools_lru_cache=1.3 && \
+ cd /usr/lib64 && ln -s libcom_err.so.2 libcom_err.so.3 && \
+ cd /usr/local/anaconda2/lib && \
+ ln -s libnetcdf.so.11 libnetcdf.so.7 && \
+ ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \
+ ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
+ ln -s libjpeg.so.9 libjpeg.so.8
+
+# Install nexusproto and nexus
+ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
+ARG APACHE_NEXUSPROTO_BRANCH=master
+ARG APACHE_NEXUS=https://github.com/apache/incubator-sdap-nexus.git
+ARG APACHE_NEXUS_BRANCH=master
+ARG REBUILD_CODE=1
+COPY install_nexusproto.sh ./install_nexusproto.sh
+COPY install_nexus.sh ./install_nexus.sh
+RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH &&
\
+ /tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC
+
+CMD ["/bin/bash"]
diff --git a/docker/spark-mesos-base/install_mesos.sh
b/docker/nexus-webapp/mesos/base/install_mesos.sh
old mode 100644
new mode 100755
similarity index 92%
rename from docker/spark-mesos-base/install_mesos.sh
rename to docker/nexus-webapp/mesos/base/install_mesos.sh
index 655d737..fc0c475
--- a/docker/spark-mesos-base/install_mesos.sh
+++ b/docker/nexus-webapp/mesos/base/install_mesos.sh
@@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+set -ebx
+
# Install a few utility tools
yum install -y tar wget git
@@ -56,10 +58,15 @@ tar -zxf ${MESOS_PACKAGE} -C ${INSTALL_LOC}
rm -f ${MESOS_PACKAGE}
# Configure and build.
-cd ${MESOS_HOME}
+mkdir -p ${MESOS_WORKDIR}
+mkdir -p ${MESOS_HOME}
+pushd ${MESOS_HOME}
mkdir build
-cd build
+pushd build
../configure
make
-make check
+# Can't run make check until this is fixed:
https://issues.apache.org/jira/browse/MESOS-8608
+# make check
make install
+popd
+popd
diff --git a/docker/spark-mesos-master/Dockerfile
b/docker/nexus-webapp/mesos/master/Dockerfile
similarity index 69%
rename from docker/spark-mesos-master/Dockerfile
rename to docker/nexus-webapp/mesos/master/Dockerfile
index c1d7d39..a6f7f91 100644
--- a/docker/spark-mesos-master/Dockerfile
+++ b/docker/nexus-webapp/mesos/master/Dockerfile
@@ -24,4 +24,7 @@ EXPOSE ${MESOS_MASTER_PORT}
WORKDIR ${MESOS_HOME}/build
-CMD ["/bin/bash", "-c", "./bin/mesos-master.sh --ip=${MESOS_IP}
--hostname=${MESOS_MASTER_NAME} --port=${MESOS_MASTER_PORT}
--work_dir=${MESOS_WORKDIR}"]
+# This will run mesos-master.sh with the value of CMD as default arguments.
However, if any arguments are supplied
+# to the docker run command when launching this image, the command line
arguments will override these CMD arguments
+ENTRYPOINT ["/bin/bash", "-c", "./bin/mesos-master.sh $(eval echo $@)", "$@"]
+CMD ["--ip=${MESOS_IP}", "--hostname=${MESOS_MASTER_NAME}",
"--port=${MESOS_MASTER_PORT}", "--work_dir=${MESOS_WORKDIR}"]
diff --git a/docker/spark-mesos-agent/Dockerfile
b/docker/nexus-webapp/mesos/webapp/Dockerfile
similarity index 88%
rename from docker/spark-mesos-agent/Dockerfile
rename to docker/nexus-webapp/mesos/webapp/Dockerfile
index 471d63f..aaa64b2 100644
--- a/docker/spark-mesos-agent/Dockerfile
+++ b/docker/nexus-webapp/mesos/webapp/Dockerfile
@@ -18,10 +18,7 @@ FROM sdap/spark-mesos-base:${tag_version}
MAINTAINER Apache SDAP "dev@sdap.apache.org"
-# Run a Mesos slave.
-
-WORKDIR ${MESOS_HOME}/build
-
-COPY docker-entrypoint.sh /tmp/docker-entrypoint.sh
+COPY /mesos/webapp/docker-entrypoint.sh /tmp/docker-entrypoint.sh
+ENV MASTER=mesos://${MESOS_MASTER_NAME}:${MESOS_MASTER_PORT}
ENTRYPOINT ["/tmp/docker-entrypoint.sh"]
diff --git a/docker/nexus-webapp/docker-entrypoint.sh
b/docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh
similarity index 100%
rename from docker/nexus-webapp/docker-entrypoint.sh
rename to docker/nexus-webapp/mesos/webapp/docker-entrypoint.sh
diff --git a/docker/nexus-webapp/Dockerfile
b/docker/nexus-webapp/standalone/Dockerfile
similarity index 71%
rename from docker/nexus-webapp/Dockerfile
rename to docker/nexus-webapp/standalone/Dockerfile
index 94ea293..06073fc 100644
--- a/docker/nexus-webapp/Dockerfile
+++ b/docker/nexus-webapp/standalone/Dockerfile
@@ -41,23 +41,25 @@ ENV SPARK_LOCAL_IP=127.0.0.1 \
PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell" \
PYTHONPATH=${PYTHONPATH}:/usr/local/spark-${SPARK_VERSION}/python:/usr/local/spark-${SPARK_VERSION}/python/lib/py4j-0.10.4-src.zip:/usr/local/spark-${SPARK_VERSION}/python/lib/pyspark.zip
\
SPARK_EXECUTOR_URI=/usr/local/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz \
- NEXUS_SRC=/tmp/incubator-sdap-nexus
+ NEXUS_SRC=/tmp/incubator-sdap-nexus \
+ PATH=/usr/local/anaconda2/bin:$PATH \
+ # Workaround for missing environment variable for basemap
https://github.com/conda-forge/basemap-feedstock/issues/30
+ PROJ_LIB=/usr/local/anaconda2/share/proj
+
+WORKDIR /tmp
+
+# Install Oracle JDK
+COPY install_java.sh ./install_java.sh
+RUN ./install_java.sh
"http://download.oracle.com/otn-pub/java/jdk/8u191-b12/2787e4a523244c269598db4e85c51e0c/jdk-8u191-linux-x64.rpm";
# Install Spark
-RUN cd /usr/local && \
- wget --quiet
http://d3kbcqa49mib13.cloudfront.net/spark-${SPARK_VERSION}-bin-hadoop2.7.tgz
&& \
- tar -xzf spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- chown -R root.root spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- ln -s spark-${SPARK_VERSION}-bin-hadoop2.7 ${SPARK_DIR} && \
- rm spark-${SPARK_VERSION}-bin-hadoop2.7.tgz && \
- cd /
+COPY install_spark.sh ./install_spark.sh
+RUN ./install_spark.sh "http://d3kbcqa49mib13.cloudfront.net"; ${SPARK_VERSION}
${SPARK_DIR} ${INSTALL_LOC}
+
+# Install Anaconda
+COPY install_conda.sh ./install_conda.sh
+RUN ./install_conda.sh
"https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh";
/usr/local/anaconda2
-# Install Miniconda
-RUN wget -q
https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O
install_anaconda.sh && \
- /bin/bash install_anaconda.sh -b -p /usr/local/anaconda2 && \
- rm install_anaconda.sh && \
- /usr/local/anaconda2/bin/conda update -n base conda
-ENV PATH /usr/local/anaconda2/bin:$PATH
# Conda dependencies for nexus
RUN conda install -c conda-forge -y netCDF4 && \
conda install -y numpy cython mpld3 scipy basemap gdal matplotlib && \
@@ -70,22 +72,16 @@ RUN conda install -c conda-forge -y netCDF4 && \
ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
ln -s libjpeg.so.9 libjpeg.so.8
-# Install Oracle JDK 1.8u182-b13
-RUN wget -q --no-cookies --no-check-certificate --header "Cookie:
gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie"
"http://download.oracle.com/otn-pub/java/jdk/8u181-b13/96a7b8442fe848ef90c96a2fad6ed6d1/jdk-8u181-linux-x64.rpm";
&& \
- yum install -y jdk-8u181-linux-x64.rpm && \
- rm jdk-8u181-linux-x64.rpm
-
-COPY *.sh /tmp/
-
# Install nexusproto and nexus
ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
ARG APACHE_NEXUSPROTO_BRANCH=master
ARG APACHE_NEXUS=https://github.com/apache/incubator-sdap-nexus.git
ARG APACHE_NEXUS_BRANCH=master
ARG REBUILD_CODE=1
+COPY install_nexusproto.sh ./install_nexusproto.sh
+COPY install_nexus.sh ./install_nexus.sh
RUN /tmp/install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH &&
\
/tmp/install_nexus.sh $APACHE_NEXUS $APACHE_NEXUS_BRANCH $NEXUS_SRC
-EXPOSE 8083
-
+COPY standalone/docker-entrypoint.sh ./docker-entrypoint.sh
ENTRYPOINT ["/tmp/docker-entrypoint.sh"]
diff --git a/docker/spark-mesos-agent/docker-entrypoint.sh
b/docker/nexus-webapp/standalone/docker-entrypoint.sh
similarity index 87%
rename from docker/spark-mesos-agent/docker-entrypoint.sh
rename to docker/nexus-webapp/standalone/docker-entrypoint.sh
index 36d608b..0589fb2 100755
--- a/docker/spark-mesos-agent/docker-entrypoint.sh
+++ b/docker/nexus-webapp/standalone/docker-entrypoint.sh
@@ -35,5 +35,4 @@ python setup.py install --force
cd ${NEXUS_SRC}/analysis
python setup.py install --force
-
-${MESOS_HOME}/build/bin/mesos-agent.sh
--master=${MESOS_MASTER_NAME}:${MESOS_MASTER_PORT} --port=${MESOS_AGENT_PORT}
--work_dir=${MESOS_WORKDIR} --no-systemd_enable_support --launcher=posix
--no-switch_user --executor_environment_variables='{ "PYTHON_EGG_CACHE": "/tmp"
}'
+python -m webservice.webapp
diff --git a/docker/spark-mesos-base/Dockerfile
b/docker/spark-mesos-base/Dockerfile
deleted file mode 100644
index 4c27bd6..0000000
--- a/docker/spark-mesos-base/Dockerfile
+++ /dev/null
@@ -1,153 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM centos:7
-
-MAINTAINER Apache SDAP "dev@sdap.apache.org"
-
-WORKDIR /tmp
-
-RUN yum -y update && \
- yum -y install wget \
- git \
- which \
- bzip2 \
- gcc \
- python-devel
-
-# Install Oracle JDK 1.8u121-b13
-RUN wget -q --no-cookies --no-check-certificate --header "Cookie:
gpw_e24=http%3A%2F%2Fwww.oracle.com%2F;
oraclelicense=accept-securebackup-cookie"
"http://download.oracle.com/otn-pub/java/jdk/8u121-b13/e9e7ea248e2c4826b92b3f075a80e441/jdk-8u121-linux-x64.rpm";
&& \
- yum -y install jdk-8u121-linux-x64.rpm && \
- rm jdk-8u121-linux-x64.rpm
-ENV JAVA_HOME /usr/java/default
-
-# ########################
-# # Apache Maven #
-# ########################
-ENV M2_HOME /usr/local/apache-maven
-ENV M2 $M2_HOME/bin
-ENV PATH $PATH:$M2
-
-RUN mkdir $M2_HOME && \
- wget -q
http://mirror.stjschools.org/public/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
&& \
- tar -xvzf apache-maven-3.3.9-bin.tar.gz -C $M2_HOME --strip-components=1
&& \
- rm -f apache-maven-3.3.9-bin.tar.gz
-
-# ########################
-# # Anaconda #
-# ########################
-RUN wget -q https://repo.continuum.io/archive/Anaconda2-4.3.0-Linux-x86_64.sh
-O install_anaconda.sh && \
- /bin/bash install_anaconda.sh -b -p /usr/local/anaconda2 && \
- rm install_anaconda.sh
-ENV PATH $PATH:/usr/local/anaconda2/bin
-
-# Set environment variables. For Mesos, I used MESOS_VER because MESOS_VERSION
-# is expected to be a logical TRUE/FALSE flag that tells Mesos whether or not
-# to simply print the version number and exit.
-
-ENV INSTALL_LOC=/usr/local \
- HADOOP_VERSION=2.7.3 \
- SPARK_VERSION=2.1.0 \
- MESOS_VER=1.5.0 \
- MESOS_MASTER_PORT=5050 \
- MESOS_AGENT_PORT=5051 \
- MESOS_WORKDIR=/var/lib/mesos \
- MESOS_IP=0.0.0.0 \
- MESOS_MASTER_NAME=mesos-master \
- PYTHON_EGG_CACHE=/tmp
-
-ENV CONDA_HOME=${INSTALL_LOC}/anaconda2 \
- MESOS_HOME=${INSTALL_LOC}/mesos-${MESOS_VER} \
- SPARK_DIR=spark-${SPARK_VERSION} \
- SPARK_PACKAGE=spark-${SPARK_VERSION}-bin-hadoop2.7 \
- MESOS_MASTER=mesos://${MESOS_IP}:${MESOS_PORT} \
- MESOS_PACKAGE=mesos-${MESOS_VER}.tar.gz
-
-ENV SPARK_HOME=${INSTALL_LOC}/${SPARK_DIR} \
- PYSPARK_DRIVER_PYTHON=${CONDA_HOME}/bin/python \
- PYSPARK_PYTHON=${CONDA_HOME}/bin/python \
- PYSPARK_SUBMIT_ARGS="--driver-memory=4g pyspark-shell"
-
-ENV
PYTHONPATH=${PYTHONPATH}:${SPARK_HOME}/python:${SPARK_HOME}/python/lib/py4j-0.10.4-src.zip:${SPARK_HOME}/python/lib/pyspark.zip
\
- MESOS_NATIVE_JAVA_LIBRARY=${INSTALL_LOC}/lib/libmesos.so \
- SPARK_EXECUTOR_URI=${INSTALL_LOC}/${SPARK_PACKAGE}.tgz
-
-WORKDIR ${INSTALL_LOC}
-
-# Set up Spark
-
-RUN wget --quiet http://d3kbcqa49mib13.cloudfront.net/${SPARK_PACKAGE}.tgz && \
- tar -xzf ${SPARK_PACKAGE}.tgz && \
- chown -R root.root ${SPARK_PACKAGE} && \
- ln -s ${SPARK_PACKAGE} ${SPARK_DIR}
-
-# Set up Mesos
-
-COPY install_mesos.sh .
-
-RUN source ./install_mesos.sh && \
- mkdir ${MESOS_WORKDIR}
-
-# Set up Anaconda environment
-
-ENV PATH=${CONDA_HOME}/bin:${PATH}:${HADOOP_HOME}/bin:${SPARK_HOME}/bin
-
-RUN conda install -c conda-forge -y netCDF4 && \
- conda install -y numpy cython mpld3 scipy basemap gdal matplotlib && \
- pip install shapely cassandra-driver==3.5.0 && \
- conda install -c conda-forge backports.functools_lru_cache=1.3
-
-# Workaround missing libcom_err.so (needed for gdal)
-
-RUN cd /usr/lib64 && ln -s libcom_err.so.2 libcom_err.so.3
-
-# Workaround missing conda libs needed for gdal
-
-RUN cd ${CONDA_HOME}/lib && \
- ln -s libnetcdf.so.11 libnetcdf.so.7 && \
- ln -s libkea.so.1.4.6 libkea.so.1.4.5 && \
- ln -s libhdf5_cpp.so.12 libhdf5_cpp.so.10 && \
- ln -s libjpeg.so.9 libjpeg.so.8
-
-RUN yum install -y mesa-libGL.x86_64
-
-# Install nexusproto
-ARG APACHE_NEXUSPROTO=https://github.com/apache/incubator-sdap-nexusproto.git
-ARG APACHE_NEXUSPROTO_BRANCH=master
-ARG REBUILD_CODE=1
-COPY install_nexusproto.sh ./install_nexusproto.sh
-RUN ./install_nexusproto.sh $APACHE_NEXUSPROTO $APACHE_NEXUSPROTO_BRANCH
-
-# Retrieve NEXUS code and build it.
-
-WORKDIR /
-
-RUN git clone https://github.com/apache/incubator-sdap-nexus.git
-
-RUN sed -i 's/,webservice.algorithms.doms//g'
/incubator-sdap-nexus/analysis/webservice/config/web.ini && \
- sed -i 's/127.0.0.1/nexus-webapp/g'
/incubator-sdap-nexus/analysis/webservice/config/web.ini && \
- sed -i
's/127.0.0.1/cassandra1,cassandra2,cassandra3,cassandra4,cassandra5,cassandra6/g'
/incubator-sdap-nexus/data-access/nexustiles/config/datastores.ini && \
- sed -i 's/localhost:8983/solr1:8983/g'
/incubator-sdap-nexus/data-access/nexustiles/config/datastores.ini
-
-WORKDIR /incubator-sdap-nexus/data-access
-
-RUN python setup.py install
-
-WORKDIR /incubator-sdap-nexus/analysis
-
-RUN python setup.py install
-
-WORKDIR /tmp
-
-CMD ["/bin/bash"]
diff --git a/docs/quickstart.rst b/docs/quickstart.rst
index 42a403d..c03dec1 100644
--- a/docs/quickstart.rst
+++ b/docs/quickstart.rst
@@ -45,7 +45,7 @@ Pull the necessary Docker images from the `SDAP repository
<https://hub.docker.c
docker pull sdap/ningester:${VERSION}
docker pull sdap/solr-singlenode:${VERSION}
docker pull sdap/cassandra:${VERSION}
- docker pull sdap/nexus-webapp:${VERSION}
+ docker pull sdap/nexus-webapp:standalone.${VERSION}
.. _quickstart-step2:
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
