This is an automated email from the ASF dual-hosted git repository. lanking pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push: new 30b552d Dockerfiles for Publish Testing (#13707) 30b552d is described below commit 30b552d5da9f32fa14f2b2c56f7ecafbebb4da41 Author: Zach Kimberg <zach...@kimberg.com> AuthorDate: Wed Jan 9 17:49:57 2019 -0800 Dockerfiles for Publish Testing (#13707) --- ci/build.py | 11 ++++--- ...cala.sh => Dockerfile.publish.test.centos7_cpu} | 31 ++++++++++++-------- ...cala.sh => Dockerfile.publish.test.centos7_gpu} | 31 ++++++++++++-------- ...a.sh => Dockerfile.publish.test.ubuntu1404_cpu} | 32 ++++++++++++-------- ...a.sh => Dockerfile.publish.test.ubuntu1404_gpu} | 33 +++++++++++++-------- ...a.sh => Dockerfile.publish.test.ubuntu1604_cpu} | 32 ++++++++++++-------- ...a.sh => Dockerfile.publish.test.ubuntu1604_gpu} | 32 ++++++++++++-------- ...a.sh => Dockerfile.publish.test.ubuntu1804_cpu} | 34 ++++++++++++++-------- ...a.sh => Dockerfile.publish.test.ubuntu1804_gpu} | 34 ++++++++++++++-------- ..._scala.sh => Dockerfile.publish.ubuntu1404_cpu} | 29 ++++++++++-------- ..._scala.sh => Dockerfile.publish.ubuntu1404_gpu} | 29 ++++++++++-------- .../install/{centos7_scala.sh => centos7_base.sh} | 16 +++++----- ci/docker/install/centos7_scala.sh | 10 ++++++- .../install/{ubuntu_core.sh => ubuntu_base.sh} | 9 +----- ci/docker/install/ubuntu_core.sh | 9 +++++- ci/docker/install/ubuntu_publish.sh | 25 +++++++++++++--- ci/docker/install/ubuntu_scala.sh | 27 +++++++++++++---- ci/docker/runtime_functions.sh | 6 ++-- ci/publish/Jenkinsfile | 22 +++++++------- {scala-package/dev => ci/publish/scala}/build.sh | 9 ++---- .../dev => ci/publish/scala}/buildkey.py | 0 {scala-package/dev => ci/publish/scala}/deploy.sh | 7 ++--- .../dev/test.sh => ci/publish/scala/fullDeploy.sh | 6 ++-- {scala-package/dev => ci/publish/scala}/test.sh | 7 ++++- ci/test_docker_cache.py | 4 +-- scala-package/.gitignore | 1 - scala-package/packageTest/pom.xml | 2 +- 27 files changed, 304 insertions(+), 184 deletions(-) diff --git a/ci/build.py b/ci/build.py index e5cf933..1c7a4f8 100755 --- a/ci/build.py +++ b/ci/build.py @@ -92,22 +92,24 @@ def get_dockerfiles_path(): def get_platforms(path: str = get_dockerfiles_path()) -> List[str]: """Get a list of architectures given our dockerfiles""" - dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*")) + dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*")) dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles)) - files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles)) + files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles)) platforms = list(map(lambda x: os.path.split(x)[1], sorted(files))) return platforms def get_docker_tag(platform: str, registry: str) -> str: """:return: docker tag to be used for the container""" + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) if not registry: registry = "mxnet_local" - return "{0}/build.{1}".format(registry, platform) + return "{0}/{1}".format(registry, platform) def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str: - return os.path.join(path, "Dockerfile.build.{0}".format(platform)) + platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform) + return os.path.join(path, "Dockerfile.{0}".format(platform)) def get_docker_binary(use_nvidia_docker: bool) -> str: @@ -523,6 +525,7 @@ def main() -> int: elif args.all: platforms = get_platforms() + platforms = [platform for platform in platforms if 'build.' in platform] logging.info("Building for all architectures: %s", platforms) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.centos7_cpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.centos7_cpu index ea46de9..7d28445 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.centos7_cpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,24 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM centos:7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh -set -ex +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +COPY runtime_functions.sh /work/ diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.centos7_gpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.centos7_gpu index ea46de9..e7f5846 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.centos7_gpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,24 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to build and run MXNet on CentOS 7 for CPU + +FROM nvidia/cuda:9.2-cudnn7-devel-centos7 + +WORKDIR /work/deps + +COPY install/centos7_base.sh /work/ +RUN /work/centos7_base.sh + +COPY install/centos7_scala.sh /work/ +RUN /work/centos7_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +COPY install/centos7_adduser.sh /work/ +RUN /work/centos7_adduser.sh -set -ex +ENV PYTHONPATH=./python/ +WORKDIR /work/mxnet -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +COPY runtime_functions.sh /work/ diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.ubuntu1404_cpu index ea46de9..0358376 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.ubuntu1404_gpu index ea46de9..854dd68 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,26 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +# Use CPU with setup_gpu script +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.ubuntu1604_cpu index ea46de9..bbb7b6a --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU + +FROM ubuntu:16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.ubuntu1604_gpu index ea46de9..660461d --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,25 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to run MXNet on Ubuntu 16.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04 + +WORKDIR /work/deps + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.ubuntu1804_cpu index ea46de9..e3a8c19 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU + +FROM ubuntu:18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu old mode 100755 new mode 100644 similarity index 60% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.test.ubuntu1804_gpu index ea46de9..99f7e0d --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,27 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to run MXNet on Ubuntu 18.04 for GPU + +FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04 + +WORKDIR /work/deps + +ENV DEBIAN_FRONTEND noninteractive + +COPY install/ubuntu_base.sh /work/ +RUN /work/ubuntu_base.sh + +COPY install/ubuntu_scala.sh /work/ +RUN /work/ubuntu_scala.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.ubuntu1404_cpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.ubuntu1404_cpu index ea46de9..04ce94f --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.ubuntu1404_cpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,22 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.ubuntu1404_gpu old mode 100755 new mode 100644 similarity index 61% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/Dockerfile.publish.ubuntu1404_gpu index ea46de9..9855986 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/Dockerfile.publish.ubuntu1404_gpu @@ -1,5 +1,4 @@ -#!/usr/bin/env bash - +# -*- mode: dockerfile -*- # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information @@ -16,16 +15,22 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# +# Dockerfile to run MXNet on Ubuntu 14.04 for GPU + +FROM ubuntu:14.04 + +WORKDIR /work/deps + +COPY install/ubuntu_publish.sh /work/ +RUN /work/ubuntu_publish.sh -# build and install are separated so changes to build don't invalidate -# the whole docker cache for the image +ARG USER_ID=0 +ARG GROUP_ID=0 +COPY install/ubuntu_adduser.sh /work/ +RUN /work/ubuntu_adduser.sh -set -ex +COPY runtime_functions.sh /work/ -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +WORKDIR /work/mxnet +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_base.sh similarity index 72% copy from ci/docker/install/centos7_scala.sh copy to ci/docker/install/centos7_base.sh index ea46de9..3b84aeb 100755 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/install/centos7_base.sh @@ -22,10 +22,12 @@ set -ex -yum install -y java-1.8.0-openjdk-devel -# Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ -alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 +# Multipackage installation does not fail in yum +yum -y install epel-release +yum -y install git +yum -y install wget +yum -y install make +yum -y install cmake +yum -y install unzip +yum -y install ninja-build +yum -y install gcc-gfortran diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh index ea46de9..5c43f01 100755 --- a/ci/docker/install/centos7_scala.sh +++ b/ci/docker/install/centos7_scala.sh @@ -23,9 +23,17 @@ set -ex yum install -y java-1.8.0-openjdk-devel +export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk +export PATH=$JAVA_HOME/bin:$PATH # Build from source with Maven -wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +wget -q http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz tar xzf apache-maven-3.3.9-bin.tar.gz mkdir /usr/local/maven mv apache-maven-3.3.9/ /usr/local/maven/ alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + +echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile.d/maven.sh +echo "export M3_HOME=/usr/local/src/apache-maven" >> /etc/profile.d/maven.sh +echo "export PATH=$M3_HOME/bin:$JAVA_HOME/bin:$PATH" >> /etc/profile.d/maven.sh +chmod +x /etc/profile.d/maven.sh +source /etc/profile.d/maven.sh diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_base.sh similarity index 86% copy from ci/docker/install/ubuntu_core.sh copy to ci/docker/install/ubuntu_base.sh index 64f8af3..b34c0b3 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_base.sh @@ -23,20 +23,13 @@ set -ex apt-get update || true apt-get install -y \ - apt-transport-https \ build-essential \ ca-certificates \ cmake \ curl \ git \ - libatlas-base-dev \ - libcurl4-openssl-dev \ - libjemalloc-dev \ - liblapack-dev \ - libopenblas-dev \ - libopencv-dev \ - libzmq3-dev \ ninja-build \ + libgfortran3 \ software-properties-common \ sudo \ unzip \ diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh index 64f8af3..4382aa6 100755 --- a/ci/docker/install/ubuntu_core.sh +++ b/ci/docker/install/ubuntu_core.sh @@ -26,7 +26,6 @@ apt-get install -y \ apt-transport-https \ build-essential \ ca-certificates \ - cmake \ curl \ git \ libatlas-base-dev \ @@ -41,3 +40,11 @@ apt-get install -y \ sudo \ unzip \ wget + + +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + apt-get install -y cmake3 +else + apt-get install -y cmake +fi diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh index bc3513d..1ad6ab9 100644 --- a/ci/docker/install/ubuntu_publish.sh +++ b/ci/docker/install/ubuntu_publish.sh @@ -35,14 +35,31 @@ apt-get install -y git \ nasm \ libtool \ curl \ + wget \ + sudo \ + gnupg \ + gnupg2 \ + gnupg-agent \ pandoc \ python3-pip \ automake \ pkg-config \ openjdk-8-jdk -curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz -tar xzf apache-maven-3.3.9-bin.tar.gz -mkdir /usr/local/maven -mv apache-maven-3.3.9/ /usr/local/maven/ +curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +tar xzf apache-maven-3.3.9-bin.tar.gz +mkdir /usr/local/maven +mv apache-maven-3.3.9/ /usr/local/maven/ update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 update-ca-certificates -f + +apt-get install -y python python3 + +# the version of the pip shipped with ubuntu may be too lower, install a recent version here +wget -nv https://bootstrap.pypa.io/get-pip.py +python3 get-pip.py +python2 get-pip.py + +apt-get remove -y python3-urllib3 + +pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 +pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3 diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh index 22be230..5bade47 100755 --- a/ci/docker/install/ubuntu_scala.sh +++ b/ci/docker/install/ubuntu_scala.sh @@ -24,16 +24,31 @@ set -ex cd "$(dirname "$0")" # install libraries for mxnet's scala package on ubuntu echo 'Installing Scala...' -apt-get update || true -apt-get install -y software-properties-common -apt-get update || true -apt-get install -y openjdk-8-jdk -apt-get install -y openjdk-8-jre +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + add-apt-repository -y ppa:openjdk-r/ppa +fi + +# All Ubuntu apt-get update || true apt-get install -y \ - maven \ + openjdk-8-jdk \ + openjdk-8-jre \ + software-properties-common \ gnupg \ gnupg2 \ gnupg-agent \ scala + +# Ubuntu 14.04 +if [[ $(lsb_release -r | grep 14.04) ]]; then + curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz + tar xzf apache-maven-3.3.9-bin.tar.gz + mkdir /usr/local/maven + mv apache-maven-3.3.9/ /usr/local/maven/ + update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1 + update-ca-certificates -f +else + apt-get install -y maven +fi diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 4ec6b76..94803ce 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -1274,7 +1274,7 @@ publish_scala_build() { set -ex pushd . scala_prepare - ./scala-package/dev/build.sh + ./ci/publish/scala/build.sh popd } @@ -1282,7 +1282,7 @@ publish_scala_test() { set -ex pushd . scala_prepare - ./scala-package/dev/test.sh + ./ci/publish/scala/test.sh popd } @@ -1290,7 +1290,7 @@ publish_scala_deploy() { set -ex pushd . scala_prepare - ./scala-package/dev/deploy.sh + ./ci/publish/scala/deploy.sh popd } diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile index 49e7d7a..9a360c6 100644 --- a/ci/publish/Jenkinsfile +++ b/ci/publish/Jenkinsfile @@ -21,7 +21,7 @@ // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/ //mxnet libraries -mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**' +mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**' // timeout in minutes max_time = 120 @@ -54,22 +54,23 @@ def labels = ['cpu'] // , 'gpu'] for (x in labels) { def label = x // Required due to language toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") { - env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] - utils.init_git() - utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') - utils.pack_lib("scala_${label}", mx_scala_pub, false) + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + utils.init_git() + utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE') + utils.pack_lib("scala_${label}", mx_scala_pub, false) + } } } def toTest = [:] -def systems = ['ubuntu'] // , 'centos7'] +def systems = ['ubuntu1604', 'ubuntu1804', 'centos7'] for (x in labels) { def label = x // Required due to language for (y in systems) { def system = y // Required due to language toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") { utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu') + utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu') } } } @@ -78,9 +79,10 @@ def toDeploy = [:] for (x in labels) { def label = x // Required due to language toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") { - env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label] - utils.unpack_and_init("scala_${label}", mx_scala_pub, false) - utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) { + utils.unpack_and_init("scala_${label}", mx_scala_pub, false) + utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION') + } } } diff --git a/scala-package/dev/build.sh b/ci/publish/scala/build.sh similarity index 88% rename from scala-package/dev/build.sh rename to ci/publish/scala/build.sh index c336fd8..17f969a 100755 --- a/scala-package/dev/build.sh +++ b/ci/publish/scala/build.sh @@ -24,11 +24,6 @@ set -ex bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./ -# Scala steps to deploy -make scalapkg CI=1 - # Compile tests for discovery later -export GPG_TTY=$(tty) -make scalatestcompile CI=1 -# make scalainstall CI=1 -make scaladeploylocal CI=1 +cd scala-package +mvn -B deploy diff --git a/scala-package/dev/buildkey.py b/ci/publish/scala/buildkey.py similarity index 100% rename from scala-package/dev/buildkey.py rename to ci/publish/scala/buildkey.py diff --git a/scala-package/dev/deploy.sh b/ci/publish/scala/deploy.sh similarity index 86% rename from scala-package/dev/deploy.sh rename to ci/publish/scala/deploy.sh index 6f845ba..4eb3390 100755 --- a/scala-package/dev/deploy.sh +++ b/ci/publish/scala/deploy.sh @@ -23,7 +23,7 @@ set -ex # export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu # Run python to configure keys -python3 $PWD/scala-package/dev/buildkey.py +python3 ci/publish/scala/buildkey.py # Updating cache mkdir -p ~/.gnupg @@ -34,11 +34,8 @@ echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf export GPG_TTY=$(tty) cd scala-package -VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec) -cd .. -# echo "\n\n$VERSION\n" | make scalarelease-dryrun -make scaladeploy CI=1 +mvn -B deploy -Pnightly # Clear all password .xml files, exp files, and gpg key files rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp diff --git a/scala-package/dev/test.sh b/ci/publish/scala/fullDeploy.sh old mode 100755 new mode 100644 similarity index 90% copy from scala-package/dev/test.sh copy to ci/publish/scala/fullDeploy.sh index 03810fb..69d674a --- a/scala-package/dev/test.sh +++ b/ci/publish/scala/fullDeploy.sh @@ -18,6 +18,6 @@ set -ex -# Test -cd scala-package/packageTest -make scalaintegrationtestlocal CI=1 +./ci/publish/scala/build.sh +./ci/publish/scala/test.sh +./ci/publish/scala/deploy.sh diff --git a/scala-package/dev/test.sh b/ci/publish/scala/test.sh similarity index 88% rename from scala-package/dev/test.sh rename to ci/publish/scala/test.sh index 03810fb..5cef35c 100755 --- a/scala-package/dev/test.sh +++ b/ci/publish/scala/test.sh @@ -18,6 +18,11 @@ set -ex +if [ -z "$JAVA_HOME" ]; then + source /etc/profile +fi + # Test cd scala-package/packageTest -make scalaintegrationtestlocal CI=1 +# make testlocal CI=1 +make testsnapshot UNIT=1 CI=1 diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py index 358d549..0a3bc46 100644 --- a/ci/test_docker_cache.py +++ b/ci/test_docker_cache.py @@ -135,7 +135,7 @@ class TestDockerCache(unittest.TestCase): """ platform = 'test_full_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: with open(dockerfile_path, 'w') as dockerfile_handle: dockerfile_handle.write(dockerfile_content) @@ -196,7 +196,7 @@ class TestDockerCache(unittest.TestCase): """ platform = 'test_partial_cache' docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH) - dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform) + dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform) try: # Write initial Dockerfile with open(dockerfile_path, 'w') as dockerfile_handle: diff --git a/scala-package/.gitignore b/scala-package/.gitignore index bf60e7b..22b12b3 100644 --- a/scala-package/.gitignore +++ b/scala-package/.gitignore @@ -8,4 +8,3 @@ core/src/main/scala/org/apache/mxnet/SymbolBase.scala core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala examples/scripts/infer/images/ examples/scripts/infer/models/ -local-snapshot diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml index 9c5c11c..f7d9e3b 100644 --- a/scala-package/packageTest/pom.xml +++ b/scala-package/packageTest/pom.xml @@ -42,7 +42,7 @@ <repositories> <repository> <id>local-snapshot</id> - <url>file://${basedir}/../local-snapshot</url> + <url>file://${basedir}/../deploy/target/repo</url> <snapshots> <enabled>true</enabled> </snapshots>