marcoabreu closed pull request #10917: [MXNET-416] Add docker cache
URL: https://github.com/apache/incubator-mxnet/pull/10917
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Jenkinsfile b/Jenkinsfile
index f18d7a993ad..4d8dfb713eb 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -92,24 +92,34 @@ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
"""
}
+def docker_run(platform, function_name, use_nvidia, shared_mem = '500m') {
+ def command = "ci/build.py --download-docker-cache --docker-cache-bucket
${env.DOCKER_CACHE_BUCKET} %USE_NVIDIA% --platform %PLATFORM% --shm-size
%SHARED_MEM% /work/runtime_functions.sh %FUNCTION_NAME%"
+ command = command.replaceAll('%USE_NVIDIA%', use_nvidia ? '--nvidiadocker' :
'')
+ command = command.replaceAll('%PLATFORM%', platform)
+ command = command.replaceAll('%FUNCTION_NAME%', function_name)
+ command = command.replaceAll('%SHARED_MEM%', shared_mem)
+
+ sh command
+}
+
// Python unittest for CPU
// Python 2
def python2_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --platform ${docker_container_name}
/work/runtime_functions.sh unittest_ubuntu_python2_cpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python2_cpu', false)
}
}
// Python 3
def python3_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --platform ${docker_container_name}
/work/runtime_functions.sh unittest_ubuntu_python3_cpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu', false)
}
}
def python3_ut_mkldnn(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --platform ${docker_container_name}
/work/runtime_functions.sh unittest_ubuntu_python3_cpu_mkldnn"
+ docker_run(docker_container_name, 'unittest_ubuntu_python3_cpu_mkldnn',
false)
}
}
@@ -118,14 +128,14 @@ def python3_ut_mkldnn(docker_container_name) {
// Python 2
def python2_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --nvidiadocker --platform ${docker_container_name}
/work/runtime_functions.sh unittest_ubuntu_python2_gpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python2_gpu', true)
}
}
// Python 3
def python3_gpu_ut(docker_container_name) {
timeout(time: max_time, unit: 'MINUTES') {
- sh "ci/build.py --nvidiadocker --platform ${docker_container_name}
/work/runtime_functions.sh unittest_ubuntu_python3_gpu"
+ docker_run(docker_container_name, 'unittest_ubuntu_python3_gpu', true)
}
}
@@ -134,7 +144,7 @@ try {
node('mxnetlinux-cpu') {
ws('workspace/sanity') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
sanity_check"
+ docker_run('ubuntu_cpu', 'sanity_check', false)
}
}
}
@@ -145,7 +155,7 @@ try {
ws('workspace/build-centos7-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh
build_centos7_cpu"
+ docker_run('centos7_cpu', 'build_centos7_cpu', false)
pack_lib('centos7_cpu')
}
}
@@ -156,7 +166,7 @@ try {
ws('workspace/build-centos7-mkldnn') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh
build_centos7_mkldnn"
+ docker_run('centos7_cpu', 'build_centos7_mkldnn', false)
pack_lib('centos7_mkldnn')
}
}
@@ -167,7 +177,7 @@ try {
ws('workspace/build-centos7-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform centos7_gpu /work/runtime_functions.sh
build_centos7_gpu"
+ docker_run('centos7_gpu', 'build_centos7_gpu', false)
pack_lib('centos7_gpu')
}
}
@@ -178,7 +188,7 @@ try {
ws('workspace/build-cpu-openblas') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_cpu_openblas"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_openblas', false)
pack_lib('cpu', mx_dist_lib)
}
}
@@ -189,7 +199,7 @@ try {
ws('workspace/build-cpu-clang39') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_cpu_clang39"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39', false)
}
}
}
@@ -199,7 +209,7 @@ try {
ws('workspace/build-cpu-clang50') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_cpu_clang50"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50', false)
}
}
}
@@ -209,7 +219,7 @@ try {
ws('workspace/build-cpu-mkldnn-clang39') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_cpu_clang39_mkldnn"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang39_mkldnn', false)
pack_lib('mkldnn_cpu_clang3', mx_mkldnn_lib)
}
}
@@ -220,7 +230,7 @@ try {
ws('workspace/build-cpu-mkldnn-clang50') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_cpu_clang50_mkldnn"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_clang50_mkldnn', false)
pack_lib('mkldnn_cpu_clang5', mx_mkldnn_lib)
}
}
@@ -231,7 +241,7 @@ try {
ws('workspace/build-mkldnn-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_cpu_mkldnn"
+ docker_run('ubuntu_cpu', 'build_ubuntu_cpu_mkldnn', false)
pack_lib('mkldnn_cpu', mx_mkldnn_lib)
}
}
@@ -242,7 +252,7 @@ try {
ws('workspace/build-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_build_cuda
/work/runtime_functions.sh build_ubuntu_gpu_mkldnn"
+ docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_mkldnn', false)
pack_lib('mkldnn_gpu', mx_mkldnn_lib)
}
}
@@ -253,7 +263,7 @@ try {
ws('workspace/build-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_build_cuda
/work/runtime_functions.sh build_ubuntu_gpu_cuda91_cudnn7"
+ docker_run('ubuntu_build_cuda', 'build_ubuntu_gpu_cuda91_cudnn7',
false)
pack_lib('gpu', mx_dist_lib)
stash includes: 'build/cpp-package/example/lenet', name:
'cpp_lenet'
stash includes: 'build/cpp-package/example/alexnet', name:
'cpp_alexnet'
@@ -274,7 +284,7 @@ try {
ws('workspace/amalgamationmin') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_amalgamation_min"
+ docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation_min', false)
}
}
}
@@ -284,7 +294,7 @@ try {
ws('workspace/amalgamation') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
build_ubuntu_amalgamation"
+ docker_run('ubuntu_cpu', 'build_ubuntu_amalgamation', false)
}
}
}
@@ -295,7 +305,7 @@ try {
ws('workspace/build-cmake-mkldnn-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh
build_ubuntu_gpu_cmake_mkldnn" //build_cuda
+ docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake_mkldnn', false)
pack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
}
}
@@ -306,7 +316,7 @@ try {
ws('workspace/build-cmake-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_gpu /work/runtime_functions.sh
build_ubuntu_gpu_cmake" //build_cuda
+ docker_run('ubuntu_gpu', 'build_ubuntu_gpu_cmake', false)
pack_lib('cmake_gpu', mx_cmake_lib)
}
}
@@ -424,7 +434,7 @@ try {
ws('workspace/build-jetson-armv8') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform jetson /work/runtime_functions.sh
build_jetson"
+ docker_run('jetson', 'build_jetson', false)
}
}
}
@@ -434,7 +444,7 @@ try {
ws('workspace/build-raspberry-armv7') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform armv7 /work/runtime_functions.sh
build_armv7"
+ docker_run('armv7', 'build_armv7', false)
}
}
}
@@ -444,7 +454,7 @@ try {
ws('workspace/build-raspberry-armv6') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform armv6 /work/runtime_functions.sh
build_armv6"
+ docker_run('armv6', 'build_armv6', false)
}
}
}
@@ -494,7 +504,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu', mx_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh unittest_ubuntu_python2_quantization_gpu"
+ docker_run('ubuntu_gpu',
'unittest_ubuntu_python2_quantization_gpu', true)
}
}
}
@@ -505,7 +515,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu', mx_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh unittest_ubuntu_python3_quantization_gpu"
+ docker_run('ubuntu_gpu',
'unittest_ubuntu_python3_quantization_gpu', true)
}
}
}
@@ -552,7 +562,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('centos7_cpu')
- sh "ci/build.py --platform centos7_cpu /work/runtime_functions.sh
unittest_centos7_cpu"
+ docker_run('centos7_cpu', 'unittest_centos7_cpu', false)
}
}
}
@@ -563,7 +573,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('centos7_gpu')
- sh "ci/build.py --nvidiadocker --platform centos7_gpu
/work/runtime_functions.sh unittest_centos7_gpu"
+ docker_run('centos7_gpu', 'unittest_centos7_gpu', true)
}
}
}
@@ -574,7 +584,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu', mx_dist_lib)
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
unittest_ubuntu_cpu_scala"
+ docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_scala', false)
}
}
}
@@ -585,7 +595,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu', mx_dist_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh unittest_ubuntu_gpu_scala"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_scala', true)
}
}
}
@@ -596,7 +606,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu')
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
unittest_ubuntu_cpugpu_perl"
+ docker_run('ubuntu_cpu', 'unittest_ubuntu_cpugpu_perl', false)
}
}
}
@@ -607,7 +617,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh unittest_ubuntu_cpugpu_perl"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_cpugpu_perl', true)
}
}
}
@@ -618,7 +628,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cmake_gpu', mx_cmake_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh unittest_ubuntu_gpu_cpp"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
}
}
}
@@ -629,7 +639,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cmake_mkldnn_gpu', mx_cmake_mkldnn_lib)
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh unittest_ubuntu_gpu_cpp"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_cpp', true)
}
}
}
@@ -640,7 +650,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu')
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
unittest_ubuntu_cpu_R"
+ docker_run('ubuntu_cpu', 'unittest_ubuntu_cpu_R', false)
}
}
}
@@ -651,7 +661,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh unittest_ubuntu_gpu_R"
+ docker_run('ubuntu_gpu', 'unittest_ubuntu_gpu_R', true)
}
}
}
@@ -756,7 +766,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('cpu')
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
integrationtest_ubuntu_cpu_onnx"
+ docker_run('ubuntu_cpu', 'integrationtest_ubuntu_cpu_onnx', false)
}
}
}
@@ -767,7 +777,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh integrationtest_ubuntu_gpu_python"
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_python', true)
}
}
}
@@ -778,7 +788,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh integrationtest_ubuntu_gpu_caffe"
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_caffe', true)
}
}
}
@@ -799,7 +809,7 @@ try {
unstash 'cpp_mlp_gpu'
unstash 'cpp_test_score'
unstash 'cpp_test_optimizer'
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh integrationtest_ubuntu_gpu_cpp_package"
+ docker_run('ubuntu_gpu', 'integrationtest_ubuntu_gpu_cpp_package',
true)
}
}
}
@@ -810,7 +820,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh integrationtest_ubuntu_gpu_dist_kvstore"
+ docker_run('ubuntu_gpu',
'integrationtest_ubuntu_gpu_dist_kvstore', true)
}
}
}
@@ -821,7 +831,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh tutorialtest_ubuntu_python2_gpu"
+ docker_run('ubuntu_gpu', 'tutorialtest_ubuntu_python2_gpu', true,
'3g')
}
}
}
@@ -832,7 +842,7 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
unpack_lib('gpu')
- sh "ci/build.py --shm-size=3g --nvidiadocker --platform ubuntu_gpu
/work/runtime_functions.sh tutorialtest_ubuntu_python3_gpu"
+ docker_run('ubuntu_gpu', 'tutorialtest_ubuntu_python3_gpu', true,
'3g')
}
}
}
@@ -844,7 +854,7 @@ try {
ws('workspace/docs') {
timeout(time: max_time, unit: 'MINUTES') {
init_git()
- sh "ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh
deploy_docs"
+ docker_run('ubuntu_cpu', 'deploy_docs', false)
sh "tests/ci_build/deploy/ci_deploy_doc.sh ${env.BRANCH_NAME}
${env.BUILD_NUMBER}"
}
}
diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache
new file mode 100644
index 00000000000..8a0428b58c3
--- /dev/null
+++ b/ci/Jenkinsfile_docker_cache
@@ -0,0 +1,81 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Jenkins pipeline to generate the centralized docker cache
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+total_timeout = 120
+git_timeout = 15
+// assign any caught errors here
+err = null
+
+// initialize source codes
+def init_git() {
+ deleteDir()
+ retry(5) {
+ try {
+ // Make sure wait long enough for api.github.com request quota.
Important: Don't increase the amount of
+ // retries as this will increase the amount of requests and worsen the
throttling
+ timeout(time: git_timeout, unit: 'MINUTES') {
+ checkout scm
+ sh 'git submodule update --init --recursive'
+ sh 'git clean -x -d -f'
+ }
+ } catch (exc) {
+ deleteDir()
+ error "Failed to fetch source codes with ${exc}"
+ sleep 2
+ }
+ }
+}
+
+
+try {
+ stage("Docker cache build & publish") {
+ node('mxnetlinux-cpu') {
+ ws('workspace/docker_cache') {
+ timeout(time: total_timeout, unit: 'MINUTES') {
+ init_git()
+ sh "ci/docker_cache.py --docker-cache-bucket
${env.DOCKER_CACHE_BUCKET}"
+ }
+ }
+ }
+ }
+
+ // set build status to success at the end
+ currentBuild.result = "SUCCESS"
+} catch (caughtError) {
+ node("mxnetlinux-cpu") {
+ sh "echo caught ${caughtError}"
+ err = caughtError
+ currentBuild.result = "FAILURE"
+ }
+} finally {
+ node("mxnetlinux-cpu") {
+ // Only send email if master failed
+ if (currentBuild.result == "FAILURE" && env.BRANCH_NAME == "master") {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please
view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED]
Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+ // Remember to rethrow so the build is marked as failing
+ if (err) {
+ throw err
+ }
+ }
+}
diff --git a/ci/__init__.py b/ci/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/ci/build.py b/ci/build.py
index 6d8d0141170..6b1d23e0391 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -33,6 +33,7 @@
import shutil
import subprocess
import sys
+import docker_cache
from copy import deepcopy
from itertools import chain
from subprocess import call, check_call
@@ -61,17 +62,44 @@ def get_docker_binary(use_nvidia_docker: bool) -> str:
def build_docker(platform: str, docker_binary: str) -> None:
- """Build a container for the given platform"""
+ """
+ Build a container for the given platform
+ :param platform: Platform
+ :param docker_binary: docker binary to use (docker/nvidia-docker)
+ :return: Id of the top level image
+ """
+
tag = get_docker_tag(platform)
logging.info("Building container tagged '%s' with %s", tag, docker_binary)
cmd = [docker_binary, "build",
"-f", get_dockerfile(platform),
+ "--rm=false", # Keep intermediary layers to prime the build cache
"--build-arg", "USER_ID={}".format(os.getuid()),
+ "--cache-from", tag,
"-t", tag,
"docker"]
logging.info("Running command: '%s'", ' '.join(cmd))
check_call(cmd)
+ # Get image id by reading the tag. It's guaranteed (except race condition)
that the tag exists. Otherwise, the
+ # check_call would have failed
+ image_id = _get_local_image_id(docker_binary=docker_binary, docker_tag=tag)
+ if not image_id:
+ raise FileNotFoundError('Unable to find docker image id matching with
{}'.format(tag))
+ return image_id
+
+
+def _get_local_image_id(docker_binary, docker_tag):
+ """
+ Get the image id of the local docker layer with the passed tag
+ :param docker_tag: docker tag
+ :return: Image id as string or None if tag does not exist
+ """
+ cmd = [docker_binary, "images", "-q", docker_tag]
+ image_id_b = subprocess.check_output(cmd)
+ image_id = image_id_b.decode('utf-8').strip()
+ return image_id
+
def get_mxnet_root() -> str:
curpath = os.path.abspath(os.path.dirname(__file__))
@@ -123,6 +151,7 @@ def container_run(platform: str,
if not dry_run and ret != 0:
logging.error("Running of command in container failed (%s): %s", ret,
cmd)
logging.error("You can try to get into the container by using the
following command: %s", docker_run_cmd)
+
raise subprocess.CalledProcessError(ret, cmd)
return docker_run_cmd
@@ -131,7 +160,6 @@ def container_run(platform: str,
def list_platforms() -> str:
print("\nSupported platforms:\n{}".format('\n'.join(get_platforms())))
-
def main() -> int:
# We need to be in the same directory than the script so the commands in
the dockerfiles work as
# expected. But the script can be invoked from a different path
@@ -180,6 +208,14 @@ def script_name() -> str:
help="go in a shell inside the container",
action='store_true')
+ parser.add_argument("--download-docker-cache",
+ help="Download the docker cache from our central
repository instead of rebuilding locally",
+ action='store_true')
+
+ parser.add_argument("--docker-cache-bucket",
+ help="S3 docker cache bucket, e.g.
mxnet-ci-docker-cache",
+ type=str)
+
parser.add_argument("command",
help="command to run in the container",
nargs='*', action='append', type=str)
@@ -194,12 +230,15 @@ def script_name() -> str:
list_platforms()
elif args.platform:
platform = args.platform
+ tag = get_docker_tag(platform)
+ if args.download_docker_cache:
+ logging.info('Docker cache download is enabled')
+
docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket,
docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
- logging.warn("Container was just built. Exiting due to
build-only.")
+ logging.warning("Container was just built. Exiting due to
build-only.")
return 0
- tag = get_docker_tag(platform)
if command:
container_run(platform, docker_binary, shared_memory_size, command)
elif args.print_docker_run:
@@ -216,6 +255,10 @@ def script_name() -> str:
logging.info("Building for all architectures: {}".format(platforms))
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
+ if args.download_docker_cache:
+ tag = get_docker_tag(platform)
+ logging.info('Docker cache download is enabled')
+
docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket,
docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
continue
diff --git a/ci/docker/Dockerfile.build.amzn_linux_cpu
b/ci/docker/Dockerfile.build.amzn_linux_cpu
deleted file mode 100755
index 7d6f2236af3..00000000000
--- a/ci/docker/Dockerfile.build.amzn_linux_cpu
+++ /dev/null
@@ -1,44 +0,0 @@
-# -*- mode: dockerfile -*-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-# Dockerfile to build and run MXNet for Amazon Linux on CPU
-
-FROM amazonlinux
-
-WORKDIR /work/deps
-COPY install/amzn_linux_core.sh /work/
-RUN /work/amzn_linux_core.sh
-COPY install/amzn_linux_opencv.sh /work/
-RUN /work/amzn_linux_opencv.sh
-COPY install/amzn_linux_openblas.sh /work/
-RUN /work/amzn_linux_openblas.sh
-COPY install/amzn_linux_python2.sh /work/
-RUN /work/amzn_linux_python2.sh
-COPY install/amzn_linux_python3.sh /work/
-RUN /work/amzn_linux_python3.sh
-COPY install/amzn_linux_testdeps.sh /work/
-RUN /work/amzn_linux_testdeps.sh
-COPY install/amzn_linux_julia.sh /work/
-RUN /work/amzn_linux_julia.sh
-COPY install/amzn_linux_maven.sh /work/
-RUN /work/amzn_linux_maven.sh
-COPY install/amzn_linux_library.sh /work/
-RUN /work/amzn_linux_library.sh
-WORKDIR /work/mxnet
-
-COPY runtime_functions.sh /work/
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.android_armv7
b/ci/docker/Dockerfile.build.android_armv7
index 0074c1f9330..c22e000cad1 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -84,13 +84,6 @@ ENV CC
/usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang
ENV CXX /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang++
ENV BUILD_OPTS "USE_BLAS=openblas USE_SSE=0 DMLC_LOG_STACK_TRACE=0
USE_OPENCV=0 USE_LAPACK=0"
-# Build MXNet
-ADD mxnet mxnet
-ADD arm.crosscompile.android.mk /work/mxnet/make/config.mk
-RUN cd mxnet && \
- make -j$(nproc) $BUILD_OPTS
+WORKDIR /work/mxnet
-WORKDIR /work/build/
-RUN cp /work/mxnet/lib/* .
-
-# TODO: Bring this into the new format
\ No newline at end of file
+COPY runtime_functions.sh /work/
diff --git a/ci/docker/Dockerfile.build.centos7_cpu
b/ci/docker/Dockerfile.build.centos7_cpu
index a44d6464ee3..92314faf121 100755
--- a/ci/docker/Dockerfile.build.centos7_cpu
+++ b/ci/docker/Dockerfile.build.centos7_cpu
@@ -20,8 +20,6 @@
FROM centos:7
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/centos7_core.sh /work/
@@ -30,6 +28,8 @@ COPY install/centos7_python.sh /work/
RUN /work/centos7_python.sh
COPY install/ubuntu_mklml.sh /work/
RUN /work/ubuntu_mklml.sh
+
+ARG USER_ID=0
COPY install/centos7_adduser.sh /work/
RUN /work/centos7_adduser.sh
diff --git a/ci/docker/Dockerfile.build.centos7_gpu
b/ci/docker/Dockerfile.build.centos7_gpu
index 4dcf5bf08ca..2d28170f11b 100755
--- a/ci/docker/Dockerfile.build.centos7_gpu
+++ b/ci/docker/Dockerfile.build.centos7_gpu
@@ -20,14 +20,14 @@
FROM nvidia/cuda:9.1-cudnn7-devel-centos7
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/centos7_core.sh /work/
RUN /work/centos7_core.sh
COPY install/centos7_python.sh /work/
RUN /work/centos7_python.sh
+
+ARG USER_ID=0
COPY install/centos7_adduser.sh /work/
RUN /work/centos7_adduser.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda
b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index 9156d6f7b69..4d3c4664363 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -23,8 +23,6 @@
FROM nvidia/cuda:9.1-cudnn7-devel
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
@@ -48,6 +46,7 @@ COPY install/ubuntu_nvidia.sh /work/
RUN /work/ubuntu_nvidia.sh
# Keep this at the end since this command is not cachable
+ARG USER_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu
b/ci/docker/Dockerfile.build.ubuntu_cpu
index f706f88461f..2dc7ef13f21 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -20,8 +20,6 @@
FROM ubuntu:16.04
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
@@ -44,6 +42,8 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
RUN /work/ubuntu_docs.sh
+
+ARG USER_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu
b/ci/docker/Dockerfile.build.ubuntu_gpu
index 547f9843d34..10971724aaa 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -20,8 +20,6 @@
FROM nvidia/cuda:9.1-cudnn7-devel
-ARG USER_ID=0
-
WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
@@ -50,6 +48,8 @@ COPY install/ubuntu_docs.sh /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_tutorials.sh /work/
RUN /work/ubuntu_tutorials.sh
+
+ARG USER_ID=0
COPY install/ubuntu_adduser.sh /work/
RUN /work/ubuntu_adduser.sh
diff --git a/ci/docker/install/amzn_linux_core.sh
b/ci/docker/install/amzn_linux_core.sh
deleted file mode 100755
index c13c9695197..00000000000
--- a/ci/docker/install/amzn_linux_core.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-yum install -y git
-yum install -y wget
-yum install -y sudo
-yum install -y re2c
-yum groupinstall -y 'Development Tools'
-
-# Ninja
-git clone --recursive https://github.com/ninja-build/ninja.git
-cd ninja
-./configure.py --bootstrap
-cp ninja /usr/local/bin
-popd
-
-# CMake
-pushd .
-git clone --recursive https://github.com/Kitware/CMake.git --branch v3.10.2
-cd CMake
-./bootstrap
-make -j$(nproc)
-make install
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_julia.sh
b/ci/docker/install/amzn_linux_julia.sh
deleted file mode 100755
index bfaf3c4924b..00000000000
--- a/ci/docker/install/amzn_linux_julia.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-wget -nv
https://julialang.s3.amazonaws.com/bin/linux/x64/0.5/julia-0.5.0-linux-x86_64.tar.gz
-mv julia-0.5.0-linux-x86_64.tar.gz /tmp/
-tar xfvz /tmp/julia-0.5.0-linux-x86_64.tar.gz
-rm -f /tmp/julia-0.5.0-linux-x86_64.tar.gz
-# tar extracted in current directory
-ln -s -f ${PWD}/julia-3c9d75391c/bin/julia /usr/bin/julia
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_library.sh
b/ci/docker/install/amzn_linux_library.sh
deleted file mode 100755
index 04708957033..00000000000
--- a/ci/docker/install/amzn_linux_library.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-yum -y install graphviz
-pip install graphviz
-pip install opencv-python
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_maven.sh
b/ci/docker/install/amzn_linux_maven.sh
deleted file mode 100755
index 22875d0ec86..00000000000
--- a/ci/docker/install/amzn_linux_maven.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-wget -nv
http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-mv apache-maven-3.3.9-bin.tar.gz /tmp/
-tar xfvz /tmp/apache-maven-3.3.9-bin.tar.gz
-yum install -y java-1.8.0-openjdk-devel
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_openblas.sh
b/ci/docker/install/amzn_linux_openblas.sh
deleted file mode 100755
index 94088d6ccf1..00000000000
--- a/ci/docker/install/amzn_linux_openblas.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-git clone https://github.com/xianyi/OpenBLAS
-cd OpenBLAS
-make FC=gfortran -j $(($(nproc) + 1))
-make PREFIX=/usr/local install
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_opencv.sh
b/ci/docker/install/amzn_linux_opencv.sh
deleted file mode 100755
index 956407e8362..00000000000
--- a/ci/docker/install/amzn_linux_opencv.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-yum install -y python27 python27-setuptools
-git clone https://github.com/opencv/opencv
-cd opencv
-mkdir -p build
-cd build
-cmake -DBUILD_opencv_gpu=OFF -DWITH_EIGEN=ON -DWITH_TBB=ON -DWITH_CUDA=OFF
-DWITH_1394=OFF \
--DCMAKE_BUILD_TYPE=RELEASE -DCMAKE_INSTALL_PREFIX=/usr/local -GNinja ..
-ninja install
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_python2.sh
b/ci/docker/install/amzn_linux_python2.sh
deleted file mode 100755
index e099ad6d6c4..00000000000
--- a/ci/docker/install/amzn_linux_python2.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-yum groupinstall -y "Development Tools"
-yum install -y mlocate python27 python27-setuptools python27-tools
python27-numpy python27-scipy python27-nose python27-matplotlib unzip
-ln -s -f /usr/bin/python2.7 /usr/bin/python2
-wget -nv https://bootstrap.pypa.io/get-pip.py
-python2 get-pip.py
-$(which easy_install-2.7) --upgrade pip
-if [ -f /usr/local/bin/pip ] && [ -f /usr/bin/pip ]; then
- mv /usr/bin/pip /usr/bin/pip.bak
- ln /usr/local/bin/pip /usr/bin/pip
-fi
-
-ln -s -f /usr/local/bin/pip /usr/bin/pip
-for i in ipython[all] jupyter pandas scikit-image h5py pandas sklearn sympy;
do echo "${i}..."; pip install -U $i >/dev/null; done
diff --git a/ci/docker/install/amzn_linux_python3.sh
b/ci/docker/install/amzn_linux_python3.sh
deleted file mode 100755
index 3f80d7d98d8..00000000000
--- a/ci/docker/install/amzn_linux_python3.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pushd .
-wget -nv https://bootstrap.pypa.io/get-pip.py
-mkdir py3
-cd py3
-wget -nv https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz
-tar -xvzf Python-3.5.2.tgz
-cd Python-3.5.2
-yum install -y zlib-devel openssl-devel sqlite-devel bzip2-devel gdbm-devel
ncurses-devel xz-devel readline-devel
-./configure --prefix=/opt/ --with-zlib-dir=/usr/lib64
-make -j$(nproc)
-mkdir /opt/bin
-mkdir /opt/lib
-make install
-ln -s -f /opt/bin/python3 /usr/bin/python3
-cd ../..
-python3 get-pip.py
-ln -s -f /opt/bin/pip /usr/bin/pip3
-
-mkdir -p ~/.local/lib/python3.5/site-packages/
-pip3 install numpy
-popd
\ No newline at end of file
diff --git a/ci/docker/install/amzn_linux_testdeps.sh
b/ci/docker/install/amzn_linux_testdeps.sh
deleted file mode 100755
index f5c49d9e37b..00000000000
--- a/ci/docker/install/amzn_linux_testdeps.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
-
-set -ex
-pip install cpplint 'pylint==1.4.4' 'astroid==1.3.6'
-pip3 install nose
-ln -s -f /opt/bin/nosetests /usr/local/bin/nosetests3
-ln -s -f /opt/bin/nosetests-3.4 /usr/local/bin/nosetests-3.4
\ No newline at end of file
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
new file mode 100755
index 00000000000..7fdfbcfe80c
--- /dev/null
+++ b/ci/docker_cache.py
@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Utility to handle distributed docker cache. This is done by keeping the entire
image chain of a docker container
+on an S3 bucket. This utility allows cache creation and download. After
execution, the cache will be in an identical
+state as if the container would have been built locally already.
+"""
+
+import os
+import logging
+import argparse
+import sys
+import boto3
+import tempfile
+import pprint
+import threading
+import build as build_util
+import botocore
+import subprocess
+from botocore.handlers import disable_signing
+from subprocess import call, check_call, CalledProcessError
+from joblib import Parallel, delayed
+
+S3_METADATA_IMAGE_ID_KEY = 'docker-image-id'
+LOG_PROGRESS_PERCENTAGE_THRESHOLD = 10
+
+cached_aws_session = None
+
+
+class ProgressPercentage(object):
+ def __init__(self, object_name, size):
+ self._object_name = object_name
+ self._size = size
+ self._seen_so_far = 0
+ self._last_percentage = 0
+ self._lock = threading.Lock()
+
+ def __call__(self, bytes_amount) -> None:
+ # To simplify we'll assume this is hooked up
+ # to a single filename.
+ with self._lock:
+ self._seen_so_far += bytes_amount
+ percentage = int((self._seen_so_far / self._size) * 100)
+ if (percentage - self._last_percentage) >=
LOG_PROGRESS_PERCENTAGE_THRESHOLD:
+ self._last_percentage = percentage
+ logging.info('{}% of {}'.format(percentage, self._object_name))
+
+
+def build_save_containers(platforms, bucket) -> int:
+ """
+ Entry point to build and upload all built dockerimages in parallel
+ :param platforms: List of platforms
+ :param bucket: S3 bucket name
+ :return: 1 if error occurred, 0 otherwise
+ """
+ if len(platforms) == 0:
+ return 0
+
+ platform_results = Parallel(n_jobs=len(platforms),
backend="multiprocessing")(
+ delayed(_build_save_container)(platform, bucket)
+ for platform in platforms)
+
+ is_error = False
+ for platform_result in platform_results:
+ if platform_result is not None:
+ logging.error('Failed to generate {}'.format(platform_result))
+ is_error = True
+
+ return 1 if is_error else 0
+
+
+def _build_save_container(platform, bucket) -> str:
+ """
+ Build image for passed platform and upload the cache to the specified S3
bucket
+ :param platform: Platform
+ :param bucket: Target s3 bucket
+ :return: Platform if failed, None otherwise
+ """
+ docker_tag = build_util.get_docker_tag(platform)
+
+ # Preload cache
+ # TODO: Allow to disable this in order to allow clean rebuilds
+ load_docker_cache(bucket_name=bucket, docker_tag=docker_tag)
+
+ # Start building
+ logging.debug('Building {} as {}'.format(platform, docker_tag))
+ try:
+ image_id = build_util.build_docker(docker_binary='docker',
platform=platform)
+ logging.info('Built {} as {}'.format(docker_tag, image_id))
+
+ # Compile and upload tarfile
+ _compile_upload_cache_file(bucket_name=bucket, docker_tag=docker_tag,
image_id=image_id)
+ return None
+ except Exception:
+ logging.exception('Unexpected exception during build of
{}'.format(docker_tag))
+ return platform
+ # Error handling is done by returning the errorous platform name. This
is necessary due to
+ # Parallel being unable to handle exceptions
+
+
+def _compile_upload_cache_file(bucket_name, docker_tag, image_id) -> None:
+ """
+ Upload the passed image by id, tag it with docker tag and upload to S3
bucket
+ :param bucket_name: S3 bucket name
+ :param docker_tag: Docker tag
+ :param image_id: Image id
+ :return: None
+ """
+ session = _get_aws_session()
+ s3_object = session.resource('s3').Object(bucket_name, docker_tag)
+
+ remote_image_id = _get_remote_image_id(s3_object)
+ if remote_image_id == image_id:
+ logging.info('{} ({}) for {} has not been updated -
skipping'.format(docker_tag, image_id, docker_tag))
+ return
+ else:
+ logging.debug('Cached image {} differs from local {} for
{}'.format(remote_image_id, image_id, docker_tag))
+
+ # Compile layers into tarfile
+ with tempfile.TemporaryDirectory() as temp_dir:
+ tar_file_path = _format_docker_cache_filepath(output_dir=temp_dir,
docker_tag=docker_tag)
+ logging.debug('Writing layers of {} to {}'.format(docker_tag,
tar_file_path))
+ history_cmd = ['docker', 'history', '-q', docker_tag]
+
+ image_ids_b = subprocess.check_output(history_cmd)
+ image_ids_str = image_ids_b.decode('utf-8').strip()
+ layer_ids = [id.strip() for id in image_ids_str.split('\n') if id !=
'<missing>']
+
+ # docker_tag is important to preserve the image name. Otherwise, the
--cache-from feature will not be able to
+ # reference the loaded cache later on. The other layer ids are added
to ensure all intermediary layers
+ # are preserved to allow resuming the cache at any point
+ cmd = ['docker', 'save', '-o', tar_file_path, docker_tag]
+ cmd.extend(layer_ids)
+ try:
+ check_call(cmd)
+ except CalledProcessError as e:
+ logging.error('Error during save of {} at {}. Command: {}'.
+ format(docker_tag, tar_file_path,
pprint.pprint(cmd)))
+ return
+
+ # Upload file
+ logging.info('Uploading {} to S3'.format(docker_tag))
+ with open(tar_file_path, 'rb') as data:
+ s3_object.upload_fileobj(
+ Fileobj=data,
+ Callback=ProgressPercentage(object_name=docker_tag,
size=os.path.getsize(tar_file_path)),
+ ExtraArgs={"Metadata": {S3_METADATA_IMAGE_ID_KEY: image_id}})
+ logging.info('Uploaded {} to S3'.format(docker_tag))
+
+
+def _get_remote_image_id(s3_object) -> str:
+ """
+ Get the image id of the docker cache which is represented by the S3 object
+ :param s3_object: S3 object
+ :return: Image id as string or None if object does not exist
+ """
+ try:
+ if S3_METADATA_IMAGE_ID_KEY in s3_object.metadata:
+ cached_image_id = s3_object.metadata[S3_METADATA_IMAGE_ID_KEY]
+ return cached_image_id
+ else:
+ logging.debug('No cached image available for
{}'.format(s3_object.key))
+ except botocore.exceptions.ClientError as e:
+ if e.response['Error']['Code'] == "404":
+ logging.debug('{} does not exist in S3 yet'.format(s3_object.key))
+ else:
+ raise
+
+ return None
+
+
+def load_docker_cache(bucket_name, docker_tag) -> None:
+ """
+ Load the precompiled docker cache from the passed S3 bucket
+ :param bucket_name: S3 bucket name
+ :param docker_tag: Docker tag to load
+ :return: None
+ """
+ # Allow anonymous access
+ s3_resource = boto3.resource('s3')
+ s3_resource.meta.client.meta.events.register('choose-signer.s3.*',
disable_signing)
+ s3_object = s3_resource.Object(bucket_name, docker_tag)
+
+ # Check if cache is still valid and exists
+ remote_image_id = _get_remote_image_id(s3_object)
+ if remote_image_id:
+ if _docker_layer_exists(remote_image_id):
+ logging.info('Local docker cache already present for
{}'.format(docker_tag))
+ return
+ else:
+ logging.info('Local docker cache not present for
{}'.format(docker_tag))
+
+ # Download using public S3 endpoint (without requiring credentials)
+ with tempfile.TemporaryDirectory() as temp_dir:
+ tar_file_path = os.path.join(temp_dir, 'layers.tar')
+ s3_object.download_file(
+ Filename=tar_file_path,
+ Callback=ProgressPercentage(object_name=docker_tag,
size=s3_object.content_length))
+
+ # Load layers
+ cmd = ['docker', 'load', '-i', tar_file_path]
+ try:
+ check_call(cmd)
+ logging.info('Docker cache for {} loaded
successfully'.format(docker_tag))
+ except CalledProcessError as e:
+ logging.error('Error during load of docker cache for {} at
{}'.format(docker_tag, tar_file_path))
+ logging.exception(e)
+ return
+ else:
+ logging.info('No cached remote image of {} present'.format(docker_tag))
+
+
+def _docker_layer_exists(layer_id) -> bool:
+ """
+ Check if the docker cache contains the layer with the passed id
+ :param layer_id: layer id
+ :return: True if exists, False otherwise
+ """
+ cmd = ['docker', 'images', '-q']
+ image_ids_b = subprocess.check_output(cmd)
+ image_ids_str = image_ids_b.decode('utf-8').strip()
+ return layer_id in [id.strip() for id in image_ids_str.split('\n')]
+
+
+def _get_aws_session() -> boto3.Session: # pragma: no cover
+ """
+ Get the boto3 AWS session
+ :return: Session object
+ """
+ global cached_aws_session
+ if cached_aws_session:
+ return cached_aws_session
+
+ session = boto3.Session() # Uses IAM user credentials
+ cached_aws_session = session
+ return session
+
+
+def _format_docker_cache_filepath(output_dir, docker_tag) -> str:
+ return os.path.join(output_dir, docker_tag.replace('/', '_') + '.tar')
+
+
+def main() -> int:
+ # We need to be in the same directory than the script so the commands in
the dockerfiles work as
+ # expected. But the script can be invoked from a different path
+ base = os.path.split(os.path.realpath(__file__))[0]
+ os.chdir(base)
+
+ logging.getLogger().setLevel(logging.DEBUG)
+ logging.getLogger('botocore').setLevel(logging.INFO)
+ logging.getLogger('boto3').setLevel(logging.INFO)
+ logging.getLogger('urllib3').setLevel(logging.INFO)
+ logging.getLogger('s3transfer').setLevel(logging.INFO)
+
+ def script_name() -> str:
+ return os.path.split(sys.argv[0])[1]
+
+ logging.basicConfig(format='{}: %(asctime)-15s
%(message)s'.format(script_name()))
+
+ parser = argparse.ArgumentParser(description="Utility for preserving and
loading Docker cache",epilog="")
+ parser.add_argument("--docker-cache-bucket",
+ help="S3 docker cache bucket, e.g.
mxnet-ci-docker-cache",
+ type=str,
+ required=True)
+
+ args = parser.parse_args()
+
+ platforms = build_util.get_platforms()
+ _get_aws_session() # Init AWS credentials
+ return build_save_containers(platforms=platforms,
bucket=args.docker_cache_bucket)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/ci/docker_cache_requirements b/ci/docker_cache_requirements
new file mode 100644
index 00000000000..47c16ff3b4a
--- /dev/null
+++ b/ci/docker_cache_requirements
@@ -0,0 +1,8 @@
+boto3==1.7.13
+botocore==1.10.13
+docutils==0.14
+jmespath==0.9.3
+joblib==0.11
+python-dateutil==2.7.2
+s3transfer==0.1.13
+six==1.11.0
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
