marcoabreu closed pull request #11302: [MXNET-553] Restructure dockcross dockerfiles to fix caching URL: https://github.com/apache/incubator-mxnet/pull/11302
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/ci/build.py b/ci/build.py index b6821b328a1..680f4574fd8 100755 --- a/ci/build.py +++ b/ci/build.py @@ -162,6 +162,7 @@ def container_run(platform: str, '-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE), '-e', 'CCACHE_TEMPDIR=/tmp/ccache', # temp dir should be local and not shared '-e', "CCACHE_DIR=/work/ccache", # this path is inside the container as /work/ccache is mounted + '-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification. tag] runlist.extend(command) cmd = ' '.join(runlist) diff --git a/ci/docker/Dockerfile.build.android_arm64 b/ci/docker/Dockerfile.build.android_arm64 index 4dbc49d163a..427cc0368db 100755 --- a/ci/docker/Dockerfile.build.android_arm64 +++ b/ci/docker/Dockerfile.build.android_arm64 @@ -18,22 +18,19 @@ # # Dockerfile to build MXNet for Android ARM64/ARMv8 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/base:latest MAINTAINER Pedro Larroy "pllar...@amazon.com" -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - RUN apt-get update && apt-get install -y \ unzip +WORKDIR /work/deps + +# Build x86 dependencies. +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + +# Setup Android cross-compilation environment. ENV CROSS_TRIPLE=aarch64-linux-android ENV CROSS_ROOT=/usr/${CROSS_TRIPLE} ENV AS=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-as \ @@ -63,7 +60,7 @@ ENV ANDROID_NDK_REVISION 15c ENV CC=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++ -WORKDIR /work/deps +# Build ARM dependencies. COPY install/android_arm64_ndk.sh /work/ RUN /work/android_arm64_ndk.sh COPY install/android_arm64_openblas.sh /work/ diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7 index 70bb9fb0eae..859ea02fb36 100755 --- a/ci/docker/Dockerfile.build.android_armv7 +++ b/ci/docker/Dockerfile.build.android_armv7 @@ -18,19 +18,9 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/base:latest MAINTAINER Pedro Larroy "pllar...@amazon.com" -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - # The cross-compiling emulator RUN apt-get update && apt-get install -y \ qemu-user \ @@ -88,6 +78,12 @@ RUN git clone https://github.com/xianyi/OpenBLAS.git && \ ENV OPENBLAS_ROOT /work/OpenBLAS ENV LIBRARY_PATH /work/OpenBLAS/lib/:/work/OpenBLAS/:$LIBRARY_PATH ENV CPLUS_INCLUDE_PATH /work/OpenBLAS/include/:/work/OpenBLAS/:$CPLUS_INCLUDE_PATH + +WORKDIR /work/deps + +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + WORKDIR /work ENV CC /usr/arm-linux-androideabi/bin/arm-linux-androideabi-clang diff --git a/ci/docker/Dockerfile.build.arm64 b/ci/docker/Dockerfile.build.arm64 index 2a950078b42..fd87bf0fa6c 100755 --- a/ci/docker/Dockerfile.build.arm64 +++ b/ci/docker/Dockerfile.build.arm64 @@ -18,20 +18,10 @@ # # Dockerfile to build MXNet for ARM64/ARMv8 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837 #FROM dockcross/linux-arm64 FROM mxnetci/dockcross-linux-arm64:05082018 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH aarch64 ENV HOSTCC gcc ENV TARGET ARMV8 @@ -47,5 +37,8 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + COPY runtime_functions.sh /work/ WORKDIR /work/build diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6 index 93be54025f4..156c57da3e1 100755 --- a/ci/docker/Dockerfile.build.armv6 +++ b/ci/docker/Dockerfile.build.armv6 @@ -18,18 +18,8 @@ # # Dockerfile to build MXNet for ARMv6 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/linux-armv6 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH armv6l ENV HOSTCC gcc ENV TARGET ARMV6 @@ -45,5 +35,8 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + COPY runtime_functions.sh /work/ WORKDIR /work/mxnet diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7 index 95e05e7cb51..6316270f9cf 100755 --- a/ci/docker/Dockerfile.build.armv7 +++ b/ci/docker/Dockerfile.build.armv7 @@ -18,18 +18,8 @@ # # Dockerfile to build MXNet for Android ARMv7 -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh - FROM dockcross/linux-armv7 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH armv7l ENV HOSTCC gcc ENV TARGET ARMV7 @@ -45,5 +35,8 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + COPY runtime_functions.sh /work/ WORKDIR /work/mxnet diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson index 098225e5af5..9cacdc34b72 100755 --- a/ci/docker/Dockerfile.build.jetson +++ b/ci/docker/Dockerfile.build.jetson @@ -22,20 +22,11 @@ FROM nvidia/cuda:9.0-cudnn7-devel as cudabuilder -FROM ubuntu:16.04 as ccachebuilder - -COPY install/ubuntu_core.sh /work/ -RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh # Temporary fix due to https://github.com/apache/incubator-mxnet/issues/10837 # FROM dockcross/linux-arm64 FROM mxnetci/dockcross-linux-arm64:05082018 -# extract ccache binary into latest context -COPY --from=ccachebuilder /usr/local/bin/ccache /usr/local/bin/ccache - ENV ARCH aarch64 ENV HOSTCC gcc ENV TARGET ARMV8 @@ -51,6 +42,9 @@ RUN /work/arm_openblas.sh ENV OpenBLAS_HOME=${CROSS_ROOT} ENV OpenBLAS_DIR=${CROSS_ROOT} +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh + # Setup CUDA build env (including configuring and copying nvcc) COPY --from=cudabuilder /usr/local/cuda /usr/local/cuda ENV TARGET_ARCH aarch64 diff --git a/ci/docker/Dockerfile.build.ubuntu_build_cuda b/ci/docker/Dockerfile.build.ubuntu_build_cuda index a87651a4f5a..51b398c832c 100755 --- a/ci/docker/Dockerfile.build.ubuntu_build_cuda +++ b/ci/docker/Dockerfile.build.ubuntu_build_cuda @@ -27,8 +27,8 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh COPY install/ubuntu_python.sh /work/ RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu index f82ee75c482..57cf1e93e54 100755 --- a/ci/docker/Dockerfile.build.ubuntu_cpu +++ b/ci/docker/Dockerfile.build.ubuntu_cpu @@ -24,8 +24,8 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh COPY install/ubuntu_python.sh /work/ RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu index 5f4bfc5a87d..1696c183e15 100755 --- a/ci/docker/Dockerfile.build.ubuntu_gpu +++ b/ci/docker/Dockerfile.build.ubuntu_gpu @@ -24,8 +24,8 @@ WORKDIR /work/deps COPY install/ubuntu_core.sh /work/ RUN /work/ubuntu_core.sh -COPY install/ubuntu_ccache.sh /work/ -RUN /work/ubuntu_ccache.sh +COPY install/deb_ubuntu_ccache.sh /work/ +RUN /work/deb_ubuntu_ccache.sh COPY install/ubuntu_python.sh /work/ RUN /work/ubuntu_python.sh COPY install/ubuntu_scala.sh /work/ diff --git a/ci/docker/install/ubuntu_ccache.sh b/ci/docker/install/deb_ubuntu_ccache.sh similarity index 85% rename from ci/docker/install/ubuntu_ccache.sh rename to ci/docker/install/deb_ubuntu_ccache.sh index 4a6b63f0f72..31cd5c8b25a 100755 --- a/ci/docker/install/ubuntu_ccache.sh +++ b/ci/docker/install/deb_ubuntu_ccache.sh @@ -17,7 +17,7 @@ # specific language governing permissions and limitations # under the License. -# Script to build ccache for ubuntu based images +# Script to build ccache for debian and ubuntu based images. set -ex @@ -43,7 +43,9 @@ git clone --recursive -b v3.4.2 https://github.com/ccache/ccache.git cd ccache ./autogen.sh -./configure +# Manually specify x86 gcc versions so that this script remains compatible with dockcross (which uses an ARM based gcc +# by default). +CC=/usr/bin/gcc CXX=/usr/bin/g++ ./configure # Don't build documentation #11214 #perl -pi -e 's!\s+\Q$(installcmd) -d $(DESTDIR)$(mandir)/man1\E!!g' Makefile diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh index 7170b8b22cc..6cefeea9fbc 100755 --- a/ci/docker/runtime_functions.sh +++ b/ci/docker/runtime_functions.sh @@ -108,6 +108,23 @@ build_jetson() { popd } +report_ccache_usage() { + set -ex + pushd . + + # Show global ccache summary at the end of each run. + ccache -s + if [ -e $CCACHE_LOGFILE ] + then + # Display local ccache log, excluding some overly verbose output. + cat $CCACHE_LOGFILE | grep -v "Config:" | grep -v "stats.lock" + else + echo "No ccache log found." + fi + + popd +} + build_armv6() { set -ex pushd . @@ -136,6 +153,7 @@ build_armv6() { -G Ninja /work/mxnet ninja -v + report_ccache_usage build_wheel popd } @@ -167,6 +185,7 @@ build_armv7() { -G Ninja /work/mxnet ninja -v + report_ccache_usage build_wheel popd } @@ -186,6 +205,7 @@ build_amzn_linux_cpu() { -DUSE_DIST_KVSTORE=ON\ -G Ninja /work/mxnet ninja -v + report_ccache_usage export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so } @@ -202,6 +222,7 @@ build_arm64() { -DUSE_MKL_IF_AVAILABLE=OFF\ -G Ninja /work/mxnet ninja -v + report_ccache_usage export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so cd /work/mxnet/python python setup.py bdist_wheel --universal @@ -225,6 +246,7 @@ build_android_armv7() { -DUSE_MKL_IF_AVAILABLE=OFF\ -G Ninja /work/mxnet ninja -v + report_ccache_usage export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so cd /work/mxnet/python python setup.py bdist_wheel --universal @@ -260,6 +282,7 @@ build_centos7_cpu() { cd /work/mxnet export CC="ccache gcc" export CXX="ccache g++" + make \ DEV=1 \ USE_LAPACK=1 \ @@ -267,6 +290,8 @@ build_centos7_cpu() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) + + report_ccache_usage } build_centos7_mkldnn() { @@ -274,6 +299,7 @@ build_centos7_mkldnn() { cd /work/mxnet export CC="ccache gcc" export CXX="ccache g++" + make \ DEV=1 \ USE_LAPACK=1 \ @@ -281,6 +307,8 @@ build_centos7_mkldnn() { USE_MKLDNN=1 \ USE_BLAS=openblas \ -j$(nproc) + + report_ccache_usage } build_centos7_gpu() { @@ -314,6 +342,8 @@ build_ubuntu_cpu_openblas() { USE_BLAS=openblas \ USE_DIST_KVSTORE=1 \ -j$(nproc) + + report_ccache_usage } build_ubuntu_cpu_clang39() { @@ -330,6 +360,8 @@ build_ubuntu_cpu_clang39() { USE_OPENMP=0 \ USE_DIST_KVSTORE=1 \ -j$(nproc) + + report_ccache_usage } build_ubuntu_cpu_clang50() { @@ -346,6 +378,8 @@ build_ubuntu_cpu_clang50() { USE_OPENMP=1 \ USE_DIST_KVSTORE=1 \ -j$(nproc) + + report_ccache_usage } build_ubuntu_cpu_clang39_mkldnn() { @@ -362,6 +396,8 @@ build_ubuntu_cpu_clang39_mkldnn() { USE_MKLDNN=1 \ USE_OPENMP=0 \ -j$(nproc) + + report_ccache_usage } build_ubuntu_cpu_clang50_mkldnn() { @@ -378,17 +414,23 @@ build_ubuntu_cpu_clang50_mkldnn() { USE_MKLDNN=1 \ USE_OPENMP=1 \ -j$(nproc) + + report_ccache_usage } build_ubuntu_cpu_mkldnn() { set -ex + build_ccache_wrappers + make \ DEV=1 \ USE_CPP_PACKAGE=1 \ USE_BLAS=openblas \ USE_MKLDNN=1 \ -j$(nproc) + + report_ccache_usage } build_ubuntu_gpu() { @@ -397,7 +439,9 @@ build_ubuntu_gpu() { build_ubuntu_gpu_mkldnn() { set -ex + build_ccache_wrappers + make \ DEV=1 \ USE_CPP_PACKAGE=1 \ @@ -407,6 +451,8 @@ build_ubuntu_gpu_mkldnn() { USE_CUDA_PATH=/usr/local/cuda \ USE_CUDNN=1 \ -j$(nproc) + + report_ccache_usage } build_ubuntu_gpu_cuda91_cudnn7() { @@ -453,6 +499,7 @@ build_ubuntu_gpu_cmake_mkldnn() { /work/mxnet ninja -v + report_ccache_usage # libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0. cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0 @@ -474,6 +521,7 @@ build_ubuntu_gpu_cmake() { /work/mxnet ninja -v + report_ccache_usage } ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services