marcoabreu closed pull request #11302: [MXNET-553] Restructure dockcross
dockerfiles to fix caching
URL: https://github.com/apache/incubator-mxnet/pull/11302
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/build.py b/ci/build.py
index b6821b328a1..680f4574fd8 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -162,6 +162,7 @@ def container_run(platform: str,
'-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
'-e', 'CCACHE_TEMPDIR=/tmp/ccache', # temp dir should be local
and not shared
'-e', "CCACHE_DIR=/work/ccache", # this path is inside the
container as /work/ccache is mounted
+ '-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped
log, useful for ccache verification.
tag]
runlist.extend(command)
cmd = ' '.join(runlist)
diff --git a/ci/docker/Dockerfile.build.android_arm64
b/ci/docker/Dockerfile.build.android_arm64
index 4dbc49d163a..427cc0368db 100755
--- a/ci/docker/Dockerfile.build.android_arm64
+++ b/ci/docker/Dockerfile.build.android_arm64
@@ -18,22 +18,19 @@
#
# Dockerfile to build MXNet for Android ARM64/ARMv8
-FROM ubuntu:16.04 as ccachebuilder
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
-
FROM dockcross/base:latest
MAINTAINER Pedro Larroy "pllar...@amazon.com"
-# extract ccache binary into latest context
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-
RUN apt-get update && apt-get install -y \
unzip
+WORKDIR /work/deps
+
+# Build x86 dependencies.
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
+
+# Setup Android cross-compilation environment.
ENV CROSS_TRIPLE=aarch64-linux-android
ENV CROSS_ROOT=/usr/${CROSS_TRIPLE}
ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \
@@ -63,7 +60,7 @@ ENV ANDROID_NDK_REVISION 15c
ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang
ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
-WORKDIR /work/deps
+# Build ARM dependencies.
COPY install/android_arm64_ndk.sh /work/
RUN /work/android_arm64_ndk.sh
COPY install/android_arm64_openblas.sh /work/
diff --git a/ci/docker/Dockerfile.build.android_armv7
b/ci/docker/Dockerfile.build.android_armv7
index 70bb9fb0eae..859ea02fb36 100755
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -18,19 +18,9 @@
#
# Dockerfile to build MXNet for Android ARMv7
-FROM ubuntu:16.04 as ccachebuilder
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
-
FROM dockcross/base:latest
MAINTAINER Pedro Larroy "pllar...@amazon.com"
-# extract ccache binary into latest context
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-
# The cross-compiling emulator
RUN apt-get update && apt-get install -y \
qemu-user \
@@ -88,6 +78,12 @@ RUN git clone https://github.com/xianyi/OpenBLAS.git && \
ENV OPENBLAS_ROOT /work/OpenBLAS
ENV LIBRARY_PATH /work/OpenBLAS/lib/:/work/OpenBLAS/:$LIBRARY_PATH
ENV CPLUS_INCLUDE_PATH
/work/OpenBLAS/include/:/work/OpenBLAS/:$CPLUS_INCLUDE_PATH
+
+WORKDIR /work/deps
+
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
+
WORKDIR /work
ENV CC /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang
diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64
index 2a950078b42..fd87bf0fa6c 100755
--- a/ci/docker/Dockerfile.build.arm64
+++ b/ci/docker/Dockerfile.build.arm64
@@ -18,20 +18,10 @@
#
# Dockerfile to build MXNet for ARM64/ARMv8
-FROM ubuntu:16.04 as ccachebuilder
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
-
# Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
#FROM dockcross/linux-arm64
FROM mxnetci/dockcross-linux-arm64:05082018
-# extract ccache binary into latest context
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-
ENV ARCH aarch64
ENV HOSTCC gcc
ENV TARGET ARMV8
@@ -47,5 +37,8 @@ RUN /work/arm_openblas.sh
ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/build
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 93be54025f4..156c57da3e1 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -18,18 +18,8 @@
#
# Dockerfile to build MXNet for ARMv6
-FROM ubuntu:16.04 as ccachebuilder
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
-
FROM dockcross/linux-armv6
-# extract ccache binary into latest context
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-
ENV ARCH armv6l
ENV HOSTCC gcc
ENV TARGET ARMV6
@@ -45,5 +35,8 @@ RUN /work/arm_openblas.sh
ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index 95e05e7cb51..6316270f9cf 100755
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -18,18 +18,8 @@
#
# Dockerfile to build MXNet for Android ARMv7
-FROM ubuntu:16.04 as ccachebuilder
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
-
FROM dockcross/linux-armv7
-# extract ccache binary into latest context
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-
ENV ARCH armv7l
ENV HOSTCC gcc
ENV TARGET ARMV7
@@ -45,5 +35,8 @@ RUN /work/arm_openblas.sh
ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.jetson
b/ci/docker/Dockerfile.build.jetson
index 098225e5af5..9cacdc34b72 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -22,20 +22,11 @@
FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder
-FROM ubuntu:16.04 as ccachebuilder
-
-COPY install/ubuntu_core.sh /work/
-RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
# Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837
# FROM dockcross/linux-arm64
FROM mxnetci/dockcross-linux-arm64:05082018
-# extract ccache binary into latest context
-COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache
-
ENV ARCH aarch64
ENV HOSTCC gcc
ENV TARGET ARMV8
@@ -51,6 +42,9 @@ RUN /work/arm_openblas.sh
ENV OpenBLAS_HOME=${CROSS_ROOT}
ENV OpenBLAS_DIR=${CROSS_ROOT}
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
+
# Setup CUDA build env (including configuring and copying nvcc)
COPY --from=cudabuilder /usr/local/cuda /usr/local/cuda
ENV TARGET_ARCH aarch64
diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda
b/ci/docker/Dockerfile.build.ubuntu_build_cuda
index a87651a4f5a..51b398c832c 100755
--- a/ci/docker/Dockerfile.build.ubuntu_build_cuda
+++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda
@@ -27,8 +27,8 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
COPY install/ubuntu_python.sh /work/
RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu
b/ci/docker/Dockerfile.build.ubuntu_cpu
index f82ee75c482..57cf1e93e54 100755
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -24,8 +24,8 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
COPY install/ubuntu_python.sh /work/
RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu
b/ci/docker/Dockerfile.build.ubuntu_gpu
index 5f4bfc5a87d..1696c183e15 100755
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -24,8 +24,8 @@ WORKDIR /work/deps
COPY install/ubuntu_core.sh /work/
RUN /work/ubuntu_core.sh
-COPY install/ubuntu_ccache.sh /work/
-RUN /work/ubuntu_ccache.sh
+COPY install/deb_ubuntu_ccache.sh /work/
+RUN /work/deb_ubuntu_ccache.sh
COPY install/ubuntu_python.sh /work/
RUN /work/ubuntu_python.sh
COPY install/ubuntu_scala.sh /work/
diff --git a/ci/docker/install/ubuntu_ccache.sh
b/ci/docker/install/deb_ubuntu_ccache.sh
similarity index 85%
rename from ci/docker/install/ubuntu_ccache.sh
rename to ci/docker/install/deb_ubuntu_ccache.sh
index 4a6b63f0f72..31cd5c8b25a 100755
--- a/ci/docker/install/ubuntu_ccache.sh
+++ b/ci/docker/install/deb_ubuntu_ccache.sh
@@ -17,7 +17,7 @@
# specific language governing permissions and limitations
# under the License.
-# Script to build ccache for ubuntu based images
+# Script to build ccache for debian and ubuntu based images.
set -ex
@@ -43,7 +43,9 @@ git clone --recursive -b v3.4.2
https://github.com/ccache/ccache.git
cd ccache
./autogen.sh
-./configure
+# Manually specify x86 gcc versions so that this script remains compatible
with dockcross (which uses an ARM based gcc
+# by default).
+CC=/usr/bin/gcc CXX=/usr/bin/g++ ./configure
# Don't build documentation #11214
#perl -pi -e 's!\s+\Q$(installcmd) -d $(DESTDIR)$(mandir)/man1\E!!g' Makefile
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 7170b8b22cc..6cefeea9fbc 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -108,6 +108,23 @@ build_jetson() {
popd
}
+report_ccache_usage() {
+ set -ex
+ pushd .
+
+ # Show global ccache summary at the end of each run.
+ ccache -s
+ if [ -e $CCACHE_LOGFILE ]
+ then
+ # Display local ccache log, excluding some overly verbose output.
+ cat $CCACHE_LOGFILE | grep -v "Config:" | grep -v "stats.lock"
+ else
+ echo "No ccache log found."
+ fi
+
+ popd
+}
+
build_armv6() {
set -ex
pushd .
@@ -136,6 +153,7 @@ build_armv6() {
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
build_wheel
popd
}
@@ -167,6 +185,7 @@ build_armv7() {
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
build_wheel
popd
}
@@ -186,6 +205,7 @@ build_amzn_linux_cpu() {
-DUSE_DIST_KVSTORE=ON\
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
}
@@ -202,6 +222,7 @@ build_arm64() {
-DUSE_MKL_IF_AVAILABLE=OFF\
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
cd /work/mxnet/python
python setup.py bdist_wheel --universal
@@ -225,6 +246,7 @@ build_android_armv7() {
-DUSE_MKL_IF_AVAILABLE=OFF\
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
cd /work/mxnet/python
python setup.py bdist_wheel --universal
@@ -260,6 +282,7 @@ build_centos7_cpu() {
cd /work/mxnet
export CC="ccache gcc"
export CXX="ccache g++"
+
make \
DEV=1 \
USE_LAPACK=1 \
@@ -267,6 +290,8 @@ build_centos7_cpu() {
USE_BLAS=openblas \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_centos7_mkldnn() {
@@ -274,6 +299,7 @@ build_centos7_mkldnn() {
cd /work/mxnet
export CC="ccache gcc"
export CXX="ccache g++"
+
make \
DEV=1 \
USE_LAPACK=1 \
@@ -281,6 +307,8 @@ build_centos7_mkldnn() {
USE_MKLDNN=1 \
USE_BLAS=openblas \
-j$(nproc)
+
+ report_ccache_usage
}
build_centos7_gpu() {
@@ -314,6 +342,8 @@ build_ubuntu_cpu_openblas() {
USE_BLAS=openblas \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang39() {
@@ -330,6 +360,8 @@ build_ubuntu_cpu_clang39() {
USE_OPENMP=0 \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang50() {
@@ -346,6 +378,8 @@ build_ubuntu_cpu_clang50() {
USE_OPENMP=1 \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang39_mkldnn() {
@@ -362,6 +396,8 @@ build_ubuntu_cpu_clang39_mkldnn() {
USE_MKLDNN=1 \
USE_OPENMP=0 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang50_mkldnn() {
@@ -378,17 +414,23 @@ build_ubuntu_cpu_clang50_mkldnn() {
USE_MKLDNN=1 \
USE_OPENMP=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_mkldnn() {
set -ex
+
build_ccache_wrappers
+
make \
DEV=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_gpu() {
@@ -397,7 +439,9 @@ build_ubuntu_gpu() {
build_ubuntu_gpu_mkldnn() {
set -ex
+
build_ccache_wrappers
+
make \
DEV=1 \
USE_CPP_PACKAGE=1 \
@@ -407,6 +451,8 @@ build_ubuntu_gpu_mkldnn() {
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_gpu_cuda91_cudnn7() {
@@ -453,6 +499,7 @@ build_ubuntu_gpu_cmake_mkldnn() {
/work/mxnet
ninja -v
+ report_ccache_usage
# libmkldnn.so.0 is a link file. We need an actual binary file named
libmkldnn.so.0.
cp 3rdparty/mkldnn/src/libmkldnn.so.0
3rdparty/mkldnn/src/libmkldnn.so.0.tmp
mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
3rdparty/mkldnn/src/libmkldnn.so.0
@@ -474,6 +521,7 @@ build_ubuntu_gpu_cmake() {
/work/mxnet
ninja -v
+ report_ccache_usage
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
