This is an automated email from the ASF dual-hosted git repository.
lanking pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 13f7c58 Jenkins nightly maven with static build script and gpu
(#13767)
13f7c58 is described below
commit 13f7c58ef77afdfd7ff30c9c35c76100f44ea158
Author: Zach Kimberg <zach...@kimberg.com>
AuthorDate: Tue Jan 15 18:12:23 2019 -0800
Jenkins nightly maven with static build script and gpu (#13767)
---
.gitignore | 1 +
.../docker/Dockerfile.publish.ubuntu1604_cpu | 33 ++--
.../docker/Dockerfile.publish.ubuntu1604_gpu | 33 ++--
ci/docker/install/ubuntu_base.sh | 4 +
ci/docker/install/ubuntu_scala.sh | 3 -
ci/publish/Jenkinsfile | 27 +--
ci/publish/scala/build.sh | 4 +-
ci/publish/scala/deploy.sh | 16 +-
ci/publish/scala/fullDeploy.sh | 0
ci/publish/scala/test.sh | 9 +-
make/maven/maven_linux_cu92.mk | 185 +++++++++++++++++++++
scala-package/deploy/pom.xml | 2 +-
.../apache/mxnetexamples/rnn/ExampleRNNSuite.scala | 11 +-
scala-package/init/pom.xml | 7 +
scala-package/packageTest/README.md | 10 +-
scala-package/packageTest/examples/pom.xml | 51 +++++-
scala-package/packageTest/pom.xml | 2 +-
scala-package/pom.xml | 32 +++-
tools/dependencies/cityhash.sh | 8 +-
tools/dependencies/curl.sh | 8 +-
tools/dependencies/eigen.sh | 6 +-
tools/dependencies/libpng.sh | 8 +-
tools/dependencies/libtiff.sh | 8 +-
tools/dependencies/libturbojpeg.sh | 8 +-
tools/dependencies/libz.sh | 8 +-
tools/dependencies/lz4.sh | 8 +-
tools/dependencies/make_shared_dependencies.sh | 2 +
tools/dependencies/openblas.sh | 9 +-
tools/dependencies/opencv.sh | 8 +-
tools/dependencies/openssl.sh | 8 +-
tools/dependencies/protobuf.sh | 9 +-
tools/dependencies/zmq.sh | 9 +-
tools/setup_gpu_build_tools.sh | 2 +
tools/staticbuild/build.sh | 68 ++++++++
tools/{build => staticbuild}/build_lib.sh | 24 +--
tools/{build => staticbuild}/build_wheel.sh | 0
36 files changed, 520 insertions(+), 111 deletions(-)
diff --git a/.gitignore b/.gitignore
index 7eb8e7d..fc79926 100644
--- a/.gitignore
+++ b/.gitignore
@@ -146,6 +146,7 @@ tools/pip_package/mxnet
# temporary path for building dependencies when building wheel
./deps/
+./staticdeps/
bld
./tmp/*
*.jar
diff --git a/tools/build/build_wheel.sh
b/ci/docker/Dockerfile.publish.ubuntu1604_cpu
old mode 100755
new mode 100644
similarity index 60%
copy from tools/build/build_wheel.sh
copy to ci/docker/Dockerfile.publish.ubuntu1604_cpu
index a796341..df284be
--- a/tools/build/build_wheel.sh
+++ b/ci/docker/Dockerfile.publish.ubuntu1604_cpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,28 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
-# This script builds the wheel for binary distribution and performs sanity
check.
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-cd mxnet-build
-echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH
-cd -
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-# Make wheel for testing
-python setup.py bdist_wheel
+COPY runtime_functions.sh /work/
-wheel_name=$(ls -t dist | head -n 1)
-pip install -U --user --force-reinstall dist/$wheel_name
-python sanity_test.py
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/tools/build/build_wheel.sh
b/ci/docker/Dockerfile.publish.ubuntu1604_gpu
old mode 100755
new mode 100644
similarity index 59%
copy from tools/build/build_wheel.sh
copy to ci/docker/Dockerfile.publish.ubuntu1604_gpu
index a796341..2a1f859
--- a/tools/build/build_wheel.sh
+++ b/ci/docker/Dockerfile.publish.ubuntu1604_gpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,28 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_python.sh /work/
+RUN /work/ubuntu_python.sh
-# This script builds the wheel for binary distribution and performs sanity
check.
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-cd mxnet-build
-echo $(git rev-parse HEAD) >> python/mxnet/COMMIT_HASH
-cd -
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-# Make wheel for testing
-python setup.py bdist_wheel
+COPY runtime_functions.sh /work/
-wheel_name=$(ls -t dist | head -n 1)
-pip install -U --user --force-reinstall dist/$wheel_name
-python sanity_test.py
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/ubuntu_base.sh b/ci/docker/install/ubuntu_base.sh
index b34c0b3..f36e532 100755
--- a/ci/docker/install/ubuntu_base.sh
+++ b/ci/docker/install/ubuntu_base.sh
@@ -33,4 +33,8 @@ apt-get install -y \
software-properties-common \
sudo \
unzip \
+ expect \
+ gnupg \
+ gnupg2 \
+ gnupg-agent \
wget
diff --git a/ci/docker/install/ubuntu_scala.sh
b/ci/docker/install/ubuntu_scala.sh
index 5bade47..8b23d61 100755
--- a/ci/docker/install/ubuntu_scala.sh
+++ b/ci/docker/install/ubuntu_scala.sh
@@ -36,9 +36,6 @@ apt-get install -y \
openjdk-8-jdk \
openjdk-8-jre \
software-properties-common \
- gnupg \
- gnupg2 \
- gnupg-agent \
scala
# Ubuntu 14.04
diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile
index 9a360c6..c14ab25 100644
--- a/ci/publish/Jenkinsfile
+++ b/ci/publish/Jenkinsfile
@@ -21,7 +21,7 @@
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
//mxnet libraries
-mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a,
deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk,
scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**,
scala-package/local-snapshot/**'
+mx_scala_pub = 'lib/**, 3rdparty/dmlc-core/libdmlc.a,
3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a,
deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk,
scala-package/pom.xml, scala-package/**/pom.xml, scala-package/**/target/**,
scala-package/*/target/repo/**'
// timeout in minutes
max_time = 120
@@ -34,8 +34,9 @@ node('restricted-utility') {
utils.assign_node_labels(utility: 'restricted-utility', linux_cpu:
'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu',
linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu:
'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
// CPU and GPU. OSX nodes are not currently supported by Jenkins
-def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU]
+def nodeMap = ['cpu': NODE_LINUX_CPU, 'gpu': NODE_LINUX_GPU_P3]
def scalaOSMap = ['cpu': 'linux-x86_64-cpu', 'gpu': 'linux-x86_64-gpu']
+def scalaVariantMap = ['cpu': 'cpu', 'gpu': 'cu92']
def wrapStep(nodeToRun, workspaceName, step) {
return {
@@ -50,13 +51,13 @@ def wrapStep(nodeToRun, workspaceName, step) {
}
def toBuild = [:]
-def labels = ['cpu'] // , 'gpu']
+def labels = ['cpu', 'gpu']
for (x in labels) {
def label = x // Required due to language
- toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label],
"build-scala-${label}") {
- withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+ toBuild["Scala Build ${label}"] = wrapStep(nodeMap['cpu'],
"build-scala-${label}") {
+ withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}",
"mxnet_variant=${scalaVariantMap[label]}"]) {
utils.init_git()
- utils.docker_run("ubuntu_${label}", 'publish_scala_build', label ==
'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
+ utils.docker_run("publish.ubuntu1404_cpu", 'publish_scala_build', false,
'500m', 'MAVEN_PUBLISH_OS_TYPE mxnet_variant')
utils.pack_lib("scala_${label}", mx_scala_pub, false)
}
}
@@ -69,8 +70,10 @@ for (x in labels) {
for (y in systems) {
def system = y // Required due to language
toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label],
"test-scala-${system}-${label}") {
- utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
- utils.docker_run("publish.test.${system}_${label}",
'publish_scala_test', label == 'gpu')
+ withEnv(["mxnet_variant=${scalaVariantMap[label]}"]) {
+ utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+ utils.docker_run("publish.test.${system}_${label}",
'publish_scala_test', label == 'gpu', '500m', 'mxnet_variant')
+ }
}
}
}
@@ -78,10 +81,10 @@ for (x in labels) {
def toDeploy = [:]
for (x in labels) {
def label = x // Required due to language
- toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label],
"deploy-scala-${label}") {
- withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+ toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap['cpu'],
"deploy-scala-${label}") {
+ withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}",
"mxnet_variant=${scalaVariantMap[label]}"]) {
utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
- utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label ==
'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL
MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG
DOCKERHUB_SECRET_ENDPOINT_REGION')
+ utils.docker_run("publish.ubuntu1604_cpu", 'publish_scala_deploy',
false, '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL
MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG
DOCKERHUB_SECRET_ENDPOINT_REGION mxnet_variant')
}
}
}
@@ -101,7 +104,7 @@ core_logic: {
,
failure_handler: {
if (currentBuild.result == "FAILURE") {
- // emailext body: 'Generating the nightly maven has failed. Please view
the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN
FAILED] Build ${BUILD_NUMBER}', to: '${EMAIL}'
+ emailext body: 'Generating the nightly maven has failed. Please view the
build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[NIGHTLY MAVEN FAILED]
Build ${BUILD_NUMBER}', to: '${EMAIL}'
}
}
)
diff --git a/ci/publish/scala/build.sh b/ci/publish/scala/build.sh
index 17f969a..11386ce 100755
--- a/ci/publish/scala/build.sh
+++ b/ci/publish/scala/build.sh
@@ -22,7 +22,9 @@ set -ex
# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
-bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./
+source tools/staticbuild/build.sh $mxnet_variant maven
+
+set -ex
# Compile tests for discovery later
cd scala-package
diff --git a/ci/publish/scala/deploy.sh b/ci/publish/scala/deploy.sh
index 4eb3390..00c6721 100755
--- a/ci/publish/scala/deploy.sh
+++ b/ci/publish/scala/deploy.sh
@@ -18,12 +18,10 @@
set -ex
-# Setup Environment Variables
-# MAVEN_PUBLISH_OS_TYPE: linux-x86_64-cpu|linux-x86_64-gpu|osx-x86_64-cpu
-# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
-
-# Run python to configure keys
-python3 ci/publish/scala/buildkey.py
+# On Jenkins, run python script to configure keys
+if [[ $BUILD_ID ]]; then
+ python3 ci/publish/scala/buildkey.py
+fi
# Updating cache
mkdir -p ~/.gnupg
@@ -37,5 +35,7 @@ cd scala-package
mvn -B deploy -Pnightly
-# Clear all password .xml files, exp files, and gpg key files
-rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
+# On Jenkins, clear all password .xml files, exp files, and gpg key files
+if [[ $BUILD_ID ]]; then
+ rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
+fi
diff --git a/ci/publish/scala/fullDeploy.sh b/ci/publish/scala/fullDeploy.sh
old mode 100644
new mode 100755
diff --git a/ci/publish/scala/test.sh b/ci/publish/scala/test.sh
index 5cef35c..32e3629 100755
--- a/ci/publish/scala/test.sh
+++ b/ci/publish/scala/test.sh
@@ -24,5 +24,10 @@ fi
# Test
cd scala-package/packageTest
-# make testlocal CI=1
-make testsnapshot UNIT=1 CI=1
+
+if [[ $mxnet_variant == cu* ]]; then
+ export SCALA_TEST_ON_GPU=1
+ make testlocal USE_CUDA=1 CI=1
+else
+ make testlocal CI=1
+fi
diff --git a/make/maven/maven_linux_cu92.mk b/make/maven/maven_linux_cu92.mk
new file mode 100644
index 0000000..0ed18a7
--- /dev/null
+++ b/make/maven/maven_linux_cu92.mk
@@ -0,0 +1,185 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------------------------
+# Template configuration for compiling mxnet for making maven package
+#-------------------------------------------------------------------------------
+
+#---------------------
+# choice of compiler
+#--------------------
+
+export CC = gcc
+export CXX = g++
+export NVCC = nvcc
+
+# whether compile with options for MXNet developer
+DEV = 0
+
+# whether compile with debug
+DEBUG = 0
+
+# whether compiler with profiler
+USE_PROFILER = 1
+
+# whether to turn on signal handler (e.g. segfault logger)
+USE_SIGNAL_HANDLER = 0
+
+# the additional link flags you want to add
+ADD_LDFLAGS += -L$(DEPS_PATH)/lib $(DEPS_PATH)/lib/libculibos.a -lpng -ltiff
-ljpeg -lz -ldl -lgfortran
-Wl,--version-script=$(CURDIR)/make/config/libmxnet.ver,-rpath,'$${ORIGIN}',--gc-sections
+
+# the additional compile flags you want to add
+ADD_CFLAGS += -I$(DEPS_PATH)/include -ffunction-sections -fdata-sections
+
+#---------------------------------------------
+# matrix computation libraries for CPU/GPU
+#---------------------------------------------
+
+# choose the version of blas you want to use
+# can be: mkl, blas, atlas, openblas
+# in default use atlas for linux while apple for osx
+USE_BLAS=openblas
+
+# whether use opencv during compilation
+# you can disable it, however, you will not able to use
+# imbin iterator
+USE_OPENCV = 1
+
+# whether use CUDA during compile
+USE_CUDA = 1
+
+# add the path to CUDA library to link and compile flag
+# if you have already add them to environment variable, leave it as NONE
+# USE_CUDA_PATH = /usr/local/cuda
+USE_CUDA_PATH = $(DEPS_PATH)/usr/local/cuda-9.2
+
+# whether to use CuDNN library
+USE_CUDNN = 1
+
+# whether to use NCCL library
+USE_NCCL = 1
+
+# CUDA architecture setting: going with all of them.
+# For CUDA < 6.0, comment the *_50 lines for compatibility.
+# CUDA_ARCH :=
+
+# whether use cuda runtime compiling for writing kernels in native language
(i.e. Python)
+USE_NVRTC = 1
+
+# use openmp for parallelization
+USE_OPENMP = 1
+USE_OPERATOR_TUNING = 1
+USE_LIBJPEG_TURBO = 1
+
+# whether use MKL-DNN library
+USE_MKLDNN = 0
+
+
+# MKL ML Library for Intel CPU/Xeon Phi
+# Please refer to MKL_README.md for details
+
+# MKL ML Library folder, need to be root for /usr/local
+# Change to User Home directory for standard user
+# For USE_BLAS!=mkl only
+MKLML_ROOT=/usr/local
+
+# whether use MKL2017 library
+USE_MKL2017 = 0
+
+# whether use MKL2017 experimental feature for high performance
+# Prerequisite USE_MKL2017=1
+USE_MKL2017_EXPERIMENTAL = 0
+
+# whether use NNPACK library
+USE_NNPACK = 0
+
+# whether use lapack during compilation
+# only effective when compiled with blas versions openblas/apple/atlas/mkl
+USE_LAPACK = 1
+
+# path to lapack library in case of a non-standard installation
+USE_LAPACK_PATH = $(DEPS_PATH)/lib
+
+# add path to intel library, you may need it for MKL, if you did not add the
path
+# to environment variable
+USE_INTEL_PATH = NONE
+
+# If use MKL, choose static link automatically to allow python wrapper
+ifeq ($(USE_BLAS), mkl)
+USE_STATIC_MKL = 1
+else
+USE_STATIC_MKL = NONE
+endif
+
+#----------------------------
+# Settings for power and arm arch
+#----------------------------
+ARCH := $(shell uname -a)
+ifneq (,$(filter $(ARCH), armv6l armv7l powerpc64le ppc64le aarch64))
+ USE_SSE=0
+else
+ USE_SSE=1
+endif
+
+#----------------------------
+# distributed computing
+#----------------------------
+
+# whether or not to enable multi-machine supporting
+USE_DIST_KVSTORE = 1
+
+# whether or not allow to read and write HDFS directly. If yes, then hadoop is
+# required
+USE_HDFS = 0
+
+# path to libjvm.so. required if USE_HDFS=1
+LIBJVM=$(JAVA_HOME)/jre/lib/amd64/server
+
+# whether or not allow to read and write AWS S3 directly. If yes, then
+# libcurl4-openssl-dev is required, it can be installed on Ubuntu by
+# sudo apt-get install -y libcurl4-openssl-dev
+USE_S3 = 1
+
+#----------------------------
+# additional operators
+#----------------------------
+
+# path to folders containing projects specific operators that you don't want
to put in src/operators
+EXTRA_OPERATORS =
+
+
+#----------------------------
+# plugins
+#----------------------------
+
+# whether to use caffe integration. This requires installing caffe.
+# You also need to add CAFFE_PATH/build/lib to your LD_LIBRARY_PATH
+# CAFFE_PATH = $(HOME)/caffe
+# MXNET_PLUGINS += plugin/caffe/caffe.mk
+
+# whether to use torch integration. This requires installing torch.
+# You also need to add TORCH_PATH/install/lib to your LD_LIBRARY_PATH
+# TORCH_PATH = $(HOME)/torch
+# MXNET_PLUGINS += plugin/torch/torch.mk
+
+# WARPCTC_PATH = $(HOME)/warp-ctc
+# MXNET_PLUGINS += plugin/warpctc/warpctc.mk
+
+# whether to use sframe integration. This requires build sframe
+# g...@github.com:dato-code/SFrame.git
+# SFRAME_PATH = $(HOME)/SFrame
+# MXNET_PLUGINS += plugin/sframe/plugin.mk
diff --git a/scala-package/deploy/pom.xml b/scala-package/deploy/pom.xml
index 542bf6c..4e9da89 100644
--- a/scala-package/deploy/pom.xml
+++ b/scala-package/deploy/pom.xml
@@ -23,7 +23,6 @@
<ARTIFACT_ID>mxnet-full_2.11-${platform}-${flavor}</ARTIFACT_ID>
<revision>1.5.0-SNAPSHOT</revision>
<repositoryId>apache.snapshots.https</repositoryId>
- <repo_url>file://${project.build.directory}/repo</repo_url>
<deploy_asc_types/>
<deploy_asc_classifers/>
<deploy_asc_files/>
@@ -118,6 +117,7 @@
</goals>
<configuration>
<description>${project.description}</description>
+ <skip>false</skip>
<repositoryId>${repositoryId}</repositoryId>
<url>${repo_url}</url>
<groupId>${project.groupId}</groupId>
diff --git
a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala
b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala
index 14fb7b8..2ccd38f 100644
---
a/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala
+++
b/scala-package/examples/src/test/scala/org/apache/mxnetexamples/rnn/ExampleRNNSuite.scala
@@ -49,14 +49,19 @@ class ExampleRNNSuite extends FunSuite with
BeforeAndAfterAll {
System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
ctx = Context.gpu()
}
- LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt",
- tempDirPath + "/RNN/sherlockholmes.valid.txt", Array(ctx), 1)
+ if (!System.getenv().containsKey("CI")) {
+ LstmBucketing.runTraining(tempDirPath + "/RNN/sherlockholmes.train.txt",
+ tempDirPath + "/RNN/sherlockholmes.valid.txt",
Array(ctx), 1)
+ } else {
+ logger.info("Skipping test on CI...")
+ }
}
test("Example CI: Test TrainCharRNN") {
val tempDirPath = System.getProperty("java.io.tmpdir")
if (System.getenv().containsKey("SCALA_TEST_ON_GPU") &&
- System.getenv("SCALA_TEST_ON_GPU").toInt == 1) {
+ System.getenv("SCALA_TEST_ON_GPU").toInt == 1 &&
+ !System.getenv().containsKey("CI")) {
val ctx = Context.gpu()
TrainCharRnn.runTrainCharRnn(tempDirPath + "/RNN/obama.txt",
tempDirPath, ctx, 1)
diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml
index a0bb6be..c514177 100644
--- a/scala-package/init/pom.xml
+++ b/scala-package/init/pom.xml
@@ -60,6 +60,13 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
</plugins>
</build>
diff --git a/scala-package/packageTest/README.md
b/scala-package/packageTest/README.md
index 3f1eeb8..e9980f3 100644
--- a/scala-package/packageTest/README.md
+++ b/scala-package/packageTest/README.md
@@ -7,7 +7,7 @@ This is an project created to run the test suite on a fully
packaged mxnet jar.
To setup the packageTest, you must first build your tests. To build the tests,
follow these steps from the mxnet main directory:
1. Build MXNet and the scala package from source following the directions
[here](https://mxnet.incubator.apache.org/install/scala_setup.html#source)
-2. Build the tests by running `make scalatestcompile`.
+2. Build the tests by running `mvn test-compile`.
3. Follow setup instructions below for your testing goal
## Running
@@ -18,13 +18,13 @@ There are three different modes of operation for testing
based on the location o
If you have a jar file, you can install it to your maven cache
repository(`~/.m2/repository`). This might be useful if you acquire the .jar
file from elsewhere. To install, it is easiest to use `mvn install:install-file
-Dfile=<path-to-file> -DpomFile=<path-to-pomfile>`. If the pom file is not
available, you can also run `mvn install:install-file -Dfile=<path-to-file>
-DgroupId=<group-id> -DartifactId=<artifact-id> -Dversion=<version>
-Dpackaging=<packaging>`. With the full mxnet jar, th [...]
-You can also run `make scalainstall` to install from a local build.
+You can also run `mvn install` to install from a local build.
After installing, run `make testinstall` in the package test directory to run
the tests. Note that unless you also install an additional mxnetexamples jar,
you can only run the unit tests.
### Test Local Deployment
-To test the jars that would be produced by a deployment, you can run `make
scaladeploylocal` from the main mxnet directory. This produces a local snapshot
located at `scala-package/local-snapshot`. To test this local snapshot, run
`make testlocal`.
+To test the jars that would be produced by a deployment, you can run `mvn
deploy` from the main mxnet directory. This produces a local snapshot located
at `scala-package/deploy/target/repo`. To test this local snapshot, run `make
testlocal`. It also installs the component packages needed for testing the
examples in `scala-package/*/target/repo`.
### Remote Repository Snapshot
@@ -36,11 +36,13 @@ Test the snapshot repo using `make testsnapshot` or a
different repo using `make
You are able to run unit tests, integration tests, or both using this utility.
To run the unit tests, add the flag `UNIT=1` to make (e.g. `make testsnapshot
UNIT=1`). Use `INTEGRATION=1` for integration tests. The default behavior is to
run both the unit and integration tests. However, the integration tests require
that the mxnet examples be installed in addition to the full mxnet package (see
test mode instructions above).
+For running on GPU, add the flag `USE_CUDA=1`.
+
An additional option, you can specify the mxnet version with
`MXNET_VERSION=1.3.1-SNAPSHOT`.
## Cleaning Up
-You can clean temporary files and target artifacts by running `make
scalaclean`.
+You can clean temporary files and target artifacts by running `make clean`.
## Troubleshooting
diff --git a/scala-package/packageTest/examples/pom.xml
b/scala-package/packageTest/examples/pom.xml
index e11be65..070b78d 100644
--- a/scala-package/packageTest/examples/pom.xml
+++ b/scala-package/packageTest/examples/pom.xml
@@ -21,6 +21,53 @@
<skipTests>false</skipTests>
</properties>
</profile>
+ <profile>
+ <id>fromLocal</id>
+ <repositories>
+ <repository>
+ <id>parent</id>
+ <url>file://${basedir}/../../target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>init</id>
+ <url>file://${basedir}/../../init/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>macros</id>
+ <url>file://${basedir}/../../macros/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>core</id>
+ <url>file://${basedir}/../../core/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>infer</id>
+ <url>file://${basedir}/../../infer/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ <repository>
+ <id>examples</id>
+ <url>file://${basedir}/../../examples/target/repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ </profile>
</profiles>
<build>
@@ -39,8 +86,8 @@
<dependencies>
<dependency>
<groupId>org.apache.mxnet</groupId>
- <artifactId>mxnet-examples_${mxnet.scalaprofile}</artifactId>
- <version>${mxnet.version}</version>
+ <artifactId>mxnet-examples</artifactId>
+ <version>INTERNAL</version>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/scala-package/packageTest/pom.xml
b/scala-package/packageTest/pom.xml
index f7d9e3b..f6a16dd 100644
--- a/scala-package/packageTest/pom.xml
+++ b/scala-package/packageTest/pom.xml
@@ -41,7 +41,7 @@
<id>fromLocal</id>
<repositories>
<repository>
- <id>local-snapshot</id>
+ <id>full</id>
<url>file://${basedir}/../deploy/target/repo</url>
<snapshots>
<enabled>true</enabled>
diff --git a/scala-package/pom.xml b/scala-package/pom.xml
index d39075f..07baeab 100644
--- a/scala-package/pom.xml
+++ b/scala-package/pom.xml
@@ -43,6 +43,7 @@
<dollar>$</dollar>
<MXNET_DIR>${project.basedir}/..</MXNET_DIR>
<skipJavaTests>true</skipJavaTests>
+ <repo_url>file://${project.build.directory}/repo</repo_url>
</properties>
<packaging>pom</packaging>
@@ -158,6 +159,34 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>staging</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>nightly</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
</profiles>
<build>
@@ -337,7 +366,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<configuration>
- <skip>true</skip>
+ <skip>false</skip>
+
<altDeploymentRepository>deployrepo::default::${repo_url}</altDeploymentRepository>
</configuration>
</plugin>
</plugins>
diff --git a/tools/dependencies/cityhash.sh b/tools/dependencies/cityhash.sh
index 81cc9cb..6bc663e 100755
--- a/tools/dependencies/cityhash.sh
+++ b/tools/dependencies/cityhash.sh
@@ -18,15 +18,17 @@
# under the License.
# This script builds the static library of cityhash that can be used as
dependency of mxnet.
+set -ex
CITYHASH_VERSION=1.1.1
if [[ ! -f $DEPS_PATH/lib/libcityhash.a ]]; then
# Download and build cityhash
>&2 echo "Building cityhash..."
git clone https://github.com/google/cityhash
$DEPS_PATH/cityhash-$CITYHASH_VERSION
+ pushd .
cd $DEPS_PATH/cityhash-$CITYHASH_VERSION
git reset --hard 8af9b8c2b889d80c22d6bc26ba0df1afb79a30db
./configure -prefix=$DEPS_PATH --enable-sse4.2
- make CXXFLAGS="-g -O3 -msse4.2"
- make install
- cd -
+ $MAKE CXXFLAGS="-g -O3 -msse4.2"
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/curl.sh b/tools/dependencies/curl.sh
index 9633edb..bb94771 100755
--- a/tools/dependencies/curl.sh
+++ b/tools/dependencies/curl.sh
@@ -18,12 +18,14 @@
# under the License.
# This script builds the static library of libcurl that can be used as
dependency of mxnet.
+set -ex
LIBCURL_VERSION=7.61.0
if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
# download and build libcurl
>&2 echo "Building libcurl..."
curl -s -L https://curl.haxx.se/download/curl-$LIBCURL_VERSION.zip -o
$DEPS_PATH/libcurl.zip
unzip -q $DEPS_PATH/libcurl.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/curl-$LIBCURL_VERSION
if [[ $PLATFORM == 'linux' ]]; then
CONFIG_FLAG=""
@@ -58,7 +60,7 @@ if [[ ! -f $DEPS_PATH/lib/libcurl.a ]]; then
--disable-gopher \
--disable-manual \
--prefix=$DEPS_PATH
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/eigen.sh b/tools/dependencies/eigen.sh
index ac2f75a..a0cd8fc 100755
--- a/tools/dependencies/eigen.sh
+++ b/tools/dependencies/eigen.sh
@@ -18,6 +18,7 @@
# under the License.
# This script imports the headers from eigen3 that can be used to in opencv.
+set -ex
EIGEN_VERSION=3.3.4
if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
# download eigen
@@ -25,10 +26,11 @@ if [[ ! -d $DEPS_PATH/include/eigen3 ]]; then
curl -s -L
https://github.com/eigenteam/eigen-git-mirror/archive/$EIGEN_VERSION.zip -o
$DEPS_PATH/eigen.zip
unzip -q $DEPS_PATH/eigen.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
+ pushd .
cd $DEPS_PATH/eigen-git-mirror-$EIGEN_VERSION/build
cmake \
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH ..
- make install
- cd -
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/libpng.sh b/tools/dependencies/libpng.sh
index d1523c6..3faa9f0 100755
--- a/tools/dependencies/libpng.sh
+++ b/tools/dependencies/libpng.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of libpng that can be used as
dependency of mxnet/opencv.
+set -ex
PNG_VERSION=1.6.34
if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
# download and build libpng
@@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
curl -s -L https://github.com/glennrp/libpng/archive/v$PNG_VERSION.zip -o
$DEPS_PATH/libpng.zip
unzip -q $DEPS_PATH/libpng.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/libpng-$PNG_VERSION/build
+ pushd .
cd $DEPS_PATH/libpng-$PNG_VERSION/build
cmake \
-D PNG_SHARED=OFF \
@@ -32,9 +34,9 @@ if [[ ! -f $DEPS_PATH/lib/libpng.a ]]; then
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
-D CMAKE_C_FLAGS=-fPIC ..
- make
- make install
+ $MAKE
+ $MAKE install
mkdir -p $DEPS_PATH/include/libpng
ln -s $DEPS_PATH/include/png.h $DEPS_PATH/include/libpng/png.h
- cd -
+ popd
fi
diff --git a/tools/dependencies/libtiff.sh b/tools/dependencies/libtiff.sh
index 14dcb2d..2a402bb 100755
--- a/tools/dependencies/libtiff.sh
+++ b/tools/dependencies/libtiff.sh
@@ -18,15 +18,17 @@
# under the License.
# This script builds the static library of libtiff that can be used as
dependency of mxnet/opencv.
+set -ex
TIFF_VERSION="4-0-9"
if [[ ! -f $DEPS_PATH/lib/libtiff.a ]]; then
# download and build libtiff
>&2 echo "Building libtiff..."
curl -s -L
https://gitlab.com/libtiff/libtiff/-/archive/Release-v$TIFF_VERSION/libtiff-Release-v$TIFF_VERSION.zip
-o $DEPS_PATH/libtiff.zip
unzip -q $DEPS_PATH/libtiff.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/libtiff-Release-v$TIFF_VERSION
./configure --quiet --disable-shared --disable-jpeg --disable-zlib
--disable-jbig --disable-lzma --prefix=$DEPS_PATH
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/libturbojpeg.sh
b/tools/dependencies/libturbojpeg.sh
index 4991906..ac813eb 100755
--- a/tools/dependencies/libturbojpeg.sh
+++ b/tools/dependencies/libturbojpeg.sh
@@ -19,6 +19,7 @@
# This script builds the static library of libturbojpeg that can be used as
dependency of
# mxnet/opencv.
+set -ex
TURBO_JPEG_VERSION=1.5.90
if [[ $PLATFORM == 'darwin' ]]; then
JPEG_NASM_OPTION="-D CMAKE_ASM_NASM_COMPILER=/usr/local/bin/nasm"
@@ -30,6 +31,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f
$DEPS_PATH/lib/libturbojpeg.a
curl -s -L
https://github.com/libjpeg-turbo/libjpeg-turbo/archive/$TURBO_JPEG_VERSION.zip
-o $DEPS_PATH/libjpeg.zip
unzip -q $DEPS_PATH/libjpeg.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
+ pushd .
cd $DEPS_PATH/libjpeg-turbo-$TURBO_JPEG_VERSION/build
cmake \
-G"Unix Makefiles" \
@@ -41,7 +43,7 @@ if [[ ! -f $DEPS_PATH/lib/libjpeg.a ]] || [[ ! -f
$DEPS_PATH/lib/libturbojpeg.a
-D WITH_JPEG8=TRUE \
$JPEG_NASM_OPTION \
-D ENABLE_SHARED=FALSE ..
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/libz.sh b/tools/dependencies/libz.sh
index 927f1de..c5f9953 100755
--- a/tools/dependencies/libz.sh
+++ b/tools/dependencies/libz.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of libz that can be used as dependency
of mxnet.
+set -ex
ZLIB_VERSION=1.2.6
if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
# Download and build zlib
@@ -25,12 +26,13 @@ if [[ ! -f $DEPS_PATH/lib/libz.a ]]; then
curl -s -L https://github.com/LuaDist/zlib/archive/$ZLIB_VERSION.zip -o
$DEPS_PATH/zlib.zip
unzip -q $DEPS_PATH/zlib.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/zlib-$ZLIB_VERSION/build
+ pushd .
cd $DEPS_PATH/zlib-$ZLIB_VERSION/build
cmake \
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
-D BUILD_SHARED_LIBS=OFF ..
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/lz4.sh b/tools/dependencies/lz4.sh
index a4269bf..478c992 100755
--- a/tools/dependencies/lz4.sh
+++ b/tools/dependencies/lz4.sh
@@ -18,14 +18,16 @@
# under the License.
# This script builds the static library of lz4 that can be used as dependency
of mxnet.
+set -ex
LZ4_VERSION=r130
if [[ ! -f $DEPS_PATH/lib/liblz4.a ]]; then
# Download and build lz4
>&2 echo "Building lz4..."
curl -s -L https://github.com/lz4/lz4/archive/$LZ4_VERSION.zip -o
$DEPS_PATH/lz4.zip
unzip -q $DEPS_PATH/lz4.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/lz4-$LZ4_VERSION
- make
- make PREFIX=$DEPS_PATH install
- cd -
+ $MAKE
+ $MAKE PREFIX=$DEPS_PATH install
+ popd
fi
diff --git a/tools/dependencies/make_shared_dependencies.sh
b/tools/dependencies/make_shared_dependencies.sh
index d678fdd..1650858 100755
--- a/tools/dependencies/make_shared_dependencies.sh
+++ b/tools/dependencies/make_shared_dependencies.sh
@@ -38,3 +38,5 @@ source $DIR/protobuf.sh
source $DIR/cityhash.sh
source $DIR/zmq.sh
source $DIR/lz4.sh
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f
-name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib
diff --git a/tools/dependencies/openblas.sh b/tools/dependencies/openblas.sh
index 9463e33..27f4739 100755
--- a/tools/dependencies/openblas.sh
+++ b/tools/dependencies/openblas.sh
@@ -18,6 +18,8 @@
# under the License.
# This script builds the static library of openblas that can be used as
dependency of mxnet.
+set +e # This script throws an error but otherwise works
+set -x
OPENBLAS_VERSION=0.3.3
if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
# download and build openblas
@@ -25,11 +27,12 @@ if [[ ! -e $DEPS_PATH/lib/libopenblas.a ]]; then
curl -s -L
https://github.com/xianyi/OpenBLAS/archive/v$OPENBLAS_VERSION.zip -o
$DEPS_PATH/openblas.zip
unzip -q $DEPS_PATH/openblas.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/OpenBLAS-$OPENBLAS_VERSION
- make DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1
- make PREFIX=$DEPS_PATH install
- cd -
+ $MAKE DYNAMIC_ARCH=1 NO_SHARED=1 USE_OPENMP=1
+ $MAKE PREFIX=$DEPS_PATH install
+ popd
ln -s libopenblas.a $DEPS_PATH/lib/libcblas.a
ln -s libopenblas.a $DEPS_PATH/lib/liblapack.a
fi
diff --git a/tools/dependencies/opencv.sh b/tools/dependencies/opencv.sh
index 99d0ecb..11c9c21 100755
--- a/tools/dependencies/opencv.sh
+++ b/tools/dependencies/opencv.sh
@@ -19,6 +19,7 @@
# This script builds the static library of opencv that can be used as
dependency of mxnet.
# It expects openblas, libjpeg, libpng, libtiff, eigen, etc., to be in
$DEPS_PATH.
+set -ex
OPENCV_VERSION=3.4.2
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
if [[ $PLATFORM == 'linux' ]]; then
@@ -41,6 +42,7 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f
$DEPS_PATH/lib/libopenc
curl -s -L https://github.com/opencv/opencv/archive/$OPENCV_VERSION.zip -o
$DEPS_PATH/opencv.zip
unzip -q $DEPS_PATH/opencv.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/opencv-$OPENCV_VERSION/build
+ pushd .
cd $DEPS_PATH/opencv-$OPENCV_VERSION/build
cmake \
-D OPENCV_ENABLE_NONFREE=OFF \
@@ -184,9 +186,9 @@ if [[ ! -f $DEPS_PATH/lib/libopencv_core.a ]] || [[ ! -f
$DEPS_PATH/lib/libopenc
if [[ $PLATFORM == 'linux' ]]; then
cp $DIR/patch/opencv_lapack.h ./
fi
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
# @szha: compatibility header
cat $DEPS_PATH/include/opencv2/imgcodecs/imgcodecs_c.h >>
$DEPS_PATH/include/opencv2/imgcodecs.hpp
fi
diff --git a/tools/dependencies/openssl.sh b/tools/dependencies/openssl.sh
index b7e4317..93284db 100755
--- a/tools/dependencies/openssl.sh
+++ b/tools/dependencies/openssl.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of openssl that can be used as
dependency of mxnet.
+set -ex
OPENSSL_VERSION=1.0.2l
if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f $DEPS_PATH/lib/libcrypto.a
]]; then
# download and build openssl
@@ -25,6 +26,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f
$DEPS_PATH/lib/libcrypto.a ]];
OPENSSL_VERSION=$(echo $OPENSSL_VERSION | sed 's/\./_/g')
curl -s -L
https://github.com/openssl/openssl/archive/OpenSSL_$OPENSSL_VERSION.zip -o
$DEPS_PATH/openssl.zip
unzip -q $DEPS_PATH/openssl.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/openssl-OpenSSL_$OPENSSL_VERSION
if [[ $PLATFORM == 'linux' ]]; then
TARGET=linux-x86_64
@@ -32,7 +34,7 @@ if [[ ! -f $DEPS_PATH/lib/libssl.a ]] || [[ ! -f
$DEPS_PATH/lib/libcrypto.a ]];
TARGET=darwin64-x86_64-cc
fi
./Configure no-shared no-zlib --prefix=$DEPS_PATH
--openssldir=$DEPS_PATH/ssl $TARGET
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/dependencies/protobuf.sh b/tools/dependencies/protobuf.sh
index 1564701..76ce1de 100755
--- a/tools/dependencies/protobuf.sh
+++ b/tools/dependencies/protobuf.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of protobuf along with protoc, that
can be used as dependency of mxnet.
+set -ex
PROTOBUF_VERSION=3.5.1
if [[ $PLATFORM == 'darwin' ]]; then
DY_EXT="dylib"
@@ -32,12 +33,12 @@ if [[ ! -e $LIBPROTOBUF ]] || [[ ! -e $LIBPROTOC ]]; then
>&2 echo "Building protobuf..."
curl -s -L
https://github.com/google/protobuf/archive/v$PROTOBUF_VERSION.zip -o
$DEPS_PATH/protobuf.zip
unzip -q $DEPS_PATH/protobuf.zip -d $DEPS_PATH
+ pushd .
cd $DEPS_PATH/protobuf-$PROTOBUF_VERSION
./autogen.sh
./configure -prefix=$DEPS_PATH
- make
- make install
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
-export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$(dirname $(find $DEPS_PATH -type f
-name 'libprotoc*' | grep protobuf | head -n 1)):$DEPS_PATH/lib
diff --git a/tools/dependencies/zmq.sh b/tools/dependencies/zmq.sh
index 55e1779..0042d6b 100755
--- a/tools/dependencies/zmq.sh
+++ b/tools/dependencies/zmq.sh
@@ -18,6 +18,7 @@
# under the License.
# This script builds the static library of zeroMQ that can be used as
dependency of mxnet.
+set -ex
ZEROMQ_VERSION=4.2.2
if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
# Download and build zmq
@@ -25,14 +26,14 @@ if [[ ! -f $DEPS_PATH/lib/libzmq.a ]]; then
curl -s -L https://github.com/zeromq/libzmq/archive/v$ZEROMQ_VERSION.zip
-o $DEPS_PATH/zeromq.zip
unzip -q $DEPS_PATH/zeromq.zip -d $DEPS_PATH
mkdir -p $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build
+ pushd .
cd $DEPS_PATH/libzmq-$ZEROMQ_VERSION/build
cmake \
-D CMAKE_BUILD_TYPE=RELEASE \
-D CMAKE_INSTALL_PREFIX=$DEPS_PATH \
-D WITH_LIBSODIUM=OFF \
-D BUILD_SHARED_LIBS=OFF ..
- make
- make install
- cp $DEPS_PATH/lib/x86_64-linux-gnu/libzmq.a $DEPS_PATH/lib/libzmq.a
- cd -
+ $MAKE
+ $MAKE install
+ popd
fi
diff --git a/tools/setup_gpu_build_tools.sh b/tools/setup_gpu_build_tools.sh
index 44b44c5..167d4c6 100755
--- a/tools/setup_gpu_build_tools.sh
+++ b/tools/setup_gpu_build_tools.sh
@@ -246,6 +246,8 @@ if [[ ! -d $DEPS_PATH/usr/local/cuda-${CUDA_MAJOR_VERSION}
]]; then
rm package.deb
done
+ mkdir -p ${prefix}/include
+ mkdir -p ${prefix}/lib
cp ${prefix}/usr/include/x86_64-linux-gnu/cudnn_v${LIBCUDNN_MAJOR}.h
${prefix}/include/cudnn.h
ln -s libcudnn_static_v${LIBCUDNN_MAJOR}.a
${prefix}/usr/lib/x86_64-linux-gnu/libcudnn.a
cp ${prefix}/usr/local/cuda-${CUDA_MAJOR_VERSION}/lib64/*.a ${prefix}/lib/
diff --git a/tools/staticbuild/build.sh b/tools/staticbuild/build.sh
new file mode 100755
index 0000000..4f8a78d
--- /dev/null
+++ b/tools/staticbuild/build.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ $# -lt 1 ]; then
+ >&2 echo "Usage: build.sh <VARIANT> <TARGET>"
+fi
+
+export CURDIR=$PWD
+export DEPS_PATH=$PWD/staticdeps
+export VARIANT=$(echo $1 | tr '[:upper:]' '[:lower:]')
+export STATIC_BUILD_TARGET=$(echo $2 | tr '[:upper:]' '[:lower:]')
+export PLATFORM=$(uname | tr '[:upper:]' '[:lower:]')
+
+if [[ $VARIANT == darwin* ]]; then
+ export VARIANT="darwin"
+fi
+
+NUM_PROC=1
+if [[ ! -z $(command -v nproc) ]]; then
+ NUM_PROC=$(nproc)
+elif [[ ! -z $(command -v sysctl) ]]; then
+ NUM_PROC=$(sysctl -n hw.ncpu)
+else
+ >&2 echo "Can't discover number of cores."
+fi
+export NUM_PROC
+>&2 echo "Using $NUM_PROC parallel jobs in building."
+
+if [[ $DEBUG -eq 1 ]]; then
+ export ADD_MAKE_FLAG="-j $NUM_PROC"
+else
+ export ADD_MAKE_FLAG="--quiet -j $NUM_PROC"
+fi
+export MAKE="make $ADD_MAKE_FLAG"
+
+export CC="gcc -fPIC"
+export CXX="g++ -fPIC"
+export FC="gfortran"
+export
PKG_CONFIG_PATH=$DEPS_PATH/lib/pkgconfig:$DEPS_PATH/lib64/pkgconfig:$DEPS_PATH/lib/x86_64-linux-gnu/pkgconfig:$PKG_CONFIG_PATH
+export CPATH=$DEPS_PATH/include:$CPATH
+
+if [[ $PLATFORM == 'linux' && $VARIANT == cu* ]]; then
+ source tools/setup_gpu_build_tools.sh $VARIANT $DEPS_PATH
+fi
+
+mkdir -p $DEPS_PATH
+
+# Build Dependencies
+source tools/dependencies/make_shared_dependencies.sh
+
+# Build mxnet
+source tools/staticbuild/build_lib.sh
diff --git a/tools/build/build_lib.sh b/tools/staticbuild/build_lib.sh
similarity index 77%
rename from tools/build/build_lib.sh
rename to tools/staticbuild/build_lib.sh
index 032fcb9..dc6bd4e 100755
--- a/tools/build/build_lib.sh
+++ b/tools/staticbuild/build_lib.sh
@@ -18,22 +18,20 @@
# under the License.
# This script builds the libraries of mxnet.
-make_config=config/pip_${PLATFORM}_${VARIANT}.mk
+make_config=make/${STATIC_BUILD_TARGET}/${STATIC_BUILD_TARGET}_${PLATFORM}_${VARIANT}.mk
if [[ ! -f $make_config ]]; then
>&2 echo "Couldn't find make config $make_config for the current settings."
exit 1
fi
-git clone --recursive https://github.com/apache/incubator-mxnet mxnet-build
-
>&2 echo "Now building mxnet modules..."
-cp $make_config mxnet-build/config.mk
+cp $make_config config.mk
-cd mxnet-build
+git submodule update --init --recursive || true
-make DEPS_PATH=$DEPS_PATH DMLCCORE
-make DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a
-make DEPS_PATH=$DEPS_PATH PSLITE
+$MAKE DEPS_PATH=$DEPS_PATH DMLCCORE
+$MAKE DEPS_PATH=$DEPS_PATH $PWD/3rdparty/tvm/nnvm/lib/libnnvm.a
+$MAKE DEPS_PATH=$DEPS_PATH PSLITE
if [[ $VARIANT == *mkl ]]; then
MKLDNN_LICENSE='license.txt'
@@ -46,7 +44,11 @@ if [[ $VARIANT == *mkl ]]; then
MKLML_LIBFILE='libmklml.dylib'
MKLDNN_LIBFILE='libmkldnn.0.dylib'
fi
- make DEPS_PATH=$DEPS_PATH mkldnn
+ $MAKE DEPS_PATH=$DEPS_PATH mkldnn
+ cp 3rdparty/mkldnn/build/install/lib/$IOMP_LIBFILE lib
+ cp 3rdparty/mkldnn/build/install/lib/$MKLML_LIBFILE lib
+ cp 3rdparty/mkldnn/build/install/lib/$MKLDNN_LIBFILE lib
+ cp 3rdparty/mkldnn/build/install/$MKLDNN_LICENSE lib
cp 3rdparty/mkldnn/LICENSE ./MKLML_LICENSE
fi
@@ -58,7 +60,7 @@ if [[ $VARIANT == *mkl ]]; then
fi
>&2 echo "Now building mxnet..."
-make DEPS_PATH=$DEPS_PATH || exit 1;
+$MAKE DEPS_PATH=$DEPS_PATH
if [[ $PLATFORM == 'linux' ]]; then
cp -L /usr/lib/gcc/x86_64-linux-gnu/4.8/libgfortran.so lib/libgfortran.so.3
@@ -77,4 +79,4 @@ else
>&2 echo "Not available"
fi
-cd ../
+ln -s staticdeps/ deps
diff --git a/tools/build/build_wheel.sh b/tools/staticbuild/build_wheel.sh
similarity index 100%
rename from tools/build/build_wheel.sh
rename to tools/staticbuild/build_wheel.sh
