This is an automated email from the ASF dual-hosted git repository. wesm pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push: new d0d7245 ARROW-5650: [Python] Update manylinux dependency versions d0d7245 is described below commit d0d7245ac199c0a2c0117e995aa7ca470e54ad6d Author: Antoine Pitrou <anto...@python.org> AuthorDate: Thu Jun 20 18:22:36 2019 -0500 ARROW-5650: [Python] Update manylinux dependency versions Author: Antoine Pitrou <anto...@python.org> Closes #4634 from pitrou/ARROW-5650-update-manylinux-deps and squashes the following commits: 5c36c6c2b <Antoine Pitrou> ARROW-5650: Update manylinux dependency versions --- python/manylinux1/scripts/build_bison.sh | 13 ++++++++----- python/manylinux1/scripts/build_boost.sh | 4 ++-- python/manylinux1/scripts/build_brotli.sh | 2 +- python/manylinux1/scripts/build_ccache.sh | 13 ++++++++----- python/manylinux1/scripts/build_double_conversion.sh | 6 +++--- python/manylinux1/scripts/build_glog.sh | 2 +- python/manylinux1/scripts/build_gtest.sh | 2 +- python/manylinux1/scripts/build_llvm.sh | 2 +- python/manylinux1/scripts/build_lz4.sh | 5 +++-- python/manylinux1/scripts/build_openssl.sh | 3 ++- python/manylinux1/scripts/build_protobuf.sh | 3 ++- python/manylinux1/scripts/build_re2.sh | 5 +++-- python/manylinux1/scripts/build_virtualenvs.sh | 6 ++++-- python/manylinux1/scripts/build_zlib.sh | 2 +- python/manylinux1/scripts/build_zstd.sh | 11 +++++++++-- python/manylinux1/scripts/install_cmake.sh | 8 ++++---- 16 files changed, 53 insertions(+), 34 deletions(-) diff --git a/python/manylinux1/scripts/build_bison.sh b/python/manylinux1/scripts/build_bison.sh index 29cc0be..e6f82fa 100755 --- a/python/manylinux1/scripts/build_bison.sh +++ b/python/manylinux1/scripts/build_bison.sh @@ -16,11 +16,14 @@ # specific language governing permissions and limitations # under the License. -wget http://ftp.gnu.org/gnu/bison/bison-3.0.4.tar.gz -tar xf bison-3.0.4.tar.gz -pushd bison-3.0.4 +BISON_VERSION=3.3.2 +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) + +wget http://ftp.gnu.org/gnu/bison/bison-${BISON_VERSION}.tar.gz +tar xf bison-${BISON_VERSION}.tar.gz +pushd bison-${BISON_VERSION} ./configure --prefix=/usr -make -j4 +make -j$NCORES make install popd -rm -rf bison-3.0.4 bison-3.0.4.tar.gz +rm -rf bison-${BISON_VERSION} bison-${BISON_VERSION}.tar.gz diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh index 666968d..ef3aef2 100755 --- a/python/manylinux1/scripts/build_boost.sh +++ b/python/manylinux1/scripts/build_boost.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -BOOST_VERSION=1.66.0 +BOOST_VERSION=1.68.0 BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_} NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) @@ -26,7 +26,7 @@ mkdir /arrow_boost pushd /boost_${BOOST_VERSION_UNDERSCORE} ./bootstrap.sh ./b2 -j${NCORES} tools/bcp -./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build algorithm locale format variant multiprecision/cpp_int /arrow_boost +./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build predef algorithm locale format variant multiprecision/cpp_int /arrow_boost popd pushd /arrow_boost diff --git a/python/manylinux1/scripts/build_brotli.sh b/python/manylinux1/scripts/build_brotli.sh index f26bf59..41774ba 100755 --- a/python/manylinux1/scripts/build_brotli.sh +++ b/python/manylinux1/scripts/build_brotli.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -export BROTLI_VERSION="0.6.0" +export BROTLI_VERSION="1.0.7" curl -sL "https://github.com/google/brotli/archive/v${BROTLI_VERSION}.tar.gz" -o brotli-${BROTLI_VERSION}.tar.gz tar xf brotli-${BROTLI_VERSION}.tar.gz pushd brotli-${BROTLI_VERSION} diff --git a/python/manylinux1/scripts/build_ccache.sh b/python/manylinux1/scripts/build_ccache.sh index c27f508..9dc0336 100755 --- a/python/manylinux1/scripts/build_ccache.sh +++ b/python/manylinux1/scripts/build_ccache.sh @@ -16,14 +16,17 @@ # specific language governing permissions and limitations # under the License. -curl -sL https://www.samba.org/ftp/ccache/ccache-3.3.4.tar.bz2 -o ccache-3.3.4.tar.bz2 -tar xf ccache-3.3.4.tar.bz2 -pushd ccache-3.3.4 +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) +CCACHE_VERSION=3.6 + +curl -sLO https://www.samba.org/ftp/ccache/ccache-${CCACHE_VERSION}.tar.bz2 +tar xf ccache-${CCACHE_VERSION}.tar.bz2 +pushd ccache-${CCACHE_VERSION} ./configure --prefix=/usr -make -j5 +make -j$NCORES make install popd -rm -rf ccache-3.3.4.tar.bz2 ccache-3.3.4 +rm -rf ccache-${CCACHE_VERSION}.tar.bz2 ccache-${CCACHE_VERSION} # Initialize the config directory, otherwise the build sometimes fails. mkdir /root/.ccache diff --git a/python/manylinux1/scripts/build_double_conversion.sh b/python/manylinux1/scripts/build_double_conversion.sh index 0e73c70..27dd829 100755 --- a/python/manylinux1/scripts/build_double_conversion.sh +++ b/python/manylinux1/scripts/build_double_conversion.sh @@ -16,14 +16,14 @@ # specific language governing permissions and limitations # under the License. -DC_VERSION=3.1.1 +DC_VERSION=3.1.4 curl -sL https://github.com/google/double-conversion/archive/v${DC_VERSION}.tar.gz -o double-conversion-${DC_VERSION}.tar.gz tar xf double-conversion-${DC_VERSION}.tar.gz -pushd double-conversion-3.1.1 +pushd double-conversion-${DC_VERSION} cmake . -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF -GNinja -DCMAKE_POSITION_INDEPENDENT_CODE=ON ninja install popd -rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-3.1.1 +rm -rf double-conversion-${DC_VERSION}.tar.gz double-conversion-${DC_VERSION} diff --git a/python/manylinux1/scripts/build_glog.sh b/python/manylinux1/scripts/build_glog.sh index 5964e35..46a23f6 100755 --- a/python/manylinux1/scripts/build_glog.sh +++ b/python/manylinux1/scripts/build_glog.sh @@ -32,5 +32,5 @@ cmake -DCMAKE_BUILD_TYPE=Release \ -GNinja . ninja install popd -rm -rf glog-${GLOG_VERSION}.tar.gz.tar.gz glog-${GLOG_VERSION} +rm -rf glog-${GLOG_VERSION}.tar.gz glog-${GLOG_VERSION} diff --git a/python/manylinux1/scripts/build_gtest.sh b/python/manylinux1/scripts/build_gtest.sh index 723b59b..6fd1a5b 100755 --- a/python/manylinux1/scripts/build_gtest.sh +++ b/python/manylinux1/scripts/build_gtest.sh @@ -36,4 +36,4 @@ ninja install popd popd -rm -rf googletest-release-${GTEST_VERSION}.tar.gz +rm -rf googletest-release-${GTEST_VERSION}.tar.gz googletest-release-${GTEST_VERSION} diff --git a/python/manylinux1/scripts/build_llvm.sh b/python/manylinux1/scripts/build_llvm.sh index 299d867..b975961 100755 --- a/python/manylinux1/scripts/build_llvm.sh +++ b/python/manylinux1/scripts/build_llvm.sh @@ -43,4 +43,4 @@ cmake -DCMAKE_INSTALL_PREFIX=$PREFIX \ ninja install popd popd -rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION} +rm -rf llvm-${LLVM_VERSION}.src.tar.xz llvm-${LLVM_VERSION}.src.tar llvm-${LLVM_VERSION}.src diff --git a/python/manylinux1/scripts/build_lz4.sh b/python/manylinux1/scripts/build_lz4.sh index 75f791c..cffb029 100755 --- a/python/manylinux1/scripts/build_lz4.sh +++ b/python/manylinux1/scripts/build_lz4.sh @@ -16,6 +16,7 @@ # specific language governing permissions and limitations # under the License. +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) export LZ4_VERSION="1.8.3" export PREFIX="/usr" export CFLAGS="${CFLAGS} -O3 -fPIC" @@ -24,8 +25,8 @@ curl -sL "https://github.com/lz4/lz4/archive/v${LZ4_VERSION}.tar.gz" -o lz4-${LZ tar xf lz4-${LZ4_VERSION}.tar.gz pushd lz4-${LZ4_VERSION} -make -j5 PREFIX=${PREFIX} -make install PREFIX=$PREFIX +make -j$NCORES PREFIX=${PREFIX} +make install PREFIX=${PREFIX} popd rm -rf lz4-${LZ4_VERSION}.tar.gz lz4-${LZ4_VERSION} # We don't want to link against shared libs diff --git a/python/manylinux1/scripts/build_openssl.sh b/python/manylinux1/scripts/build_openssl.sh index b07ebac..c0ee2e5 100755 --- a/python/manylinux1/scripts/build_openssl.sh +++ b/python/manylinux1/scripts/build_openssl.sh @@ -16,7 +16,8 @@ # specific language governing permissions and limitations # under the License. -OPENSSL_VERSION="1.0.2q" +# XXX OpenSSL 1.1.1 needs Perl 5.10 to compile. +OPENSSL_VERSION="1.0.2s" NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) wget --no-check-certificate https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -O openssl-${OPENSSL_VERSION}.tar.gz diff --git a/python/manylinux1/scripts/build_protobuf.sh b/python/manylinux1/scripts/build_protobuf.sh index 27d0d9a..7261973 100755 --- a/python/manylinux1/scripts/build_protobuf.sh +++ b/python/manylinux1/scripts/build_protobuf.sh @@ -16,13 +16,14 @@ # specific language governing permissions and limitations # under the License. +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) PROTOBUF_VERSION="3.7.1" curl -sL "https://github.com/google/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz" -o protobuf-${PROTOBUF_VERSION}.tar.gz tar xf protobuf-${PROTOBUF_VERSION}.tar.gz pushd protobuf-${PROTOBUF_VERSION} ./configure --disable-shared --prefix=/usr "CXXFLAGS=-O2 -fPIC" -make -j10 +make -j$NCORES make install popd rm -rf protobuf-${PROTOBUF_VERSION}.tar.gz protobuf-${PROTOBUF_VERSION} diff --git a/python/manylinux1/scripts/build_re2.sh b/python/manylinux1/scripts/build_re2.sh index f600de9..67902c1 100755 --- a/python/manylinux1/scripts/build_re2.sh +++ b/python/manylinux1/scripts/build_re2.sh @@ -16,7 +16,8 @@ # specific language governing permissions and limitations # under the License. -export RE2_VERSION="2019-01-01" +export RE2_VERSION="2019-04-01" +NCORES=$(($(grep -c ^processor /proc/cpuinfo) + 1)) curl -sL "http://github.com/google/re2/archive/${RE2_VERSION}.tar.gz" -o re2-${RE2_VERSION}.tar.gz tar xf re2-${RE2_VERSION}.tar.gz @@ -26,7 +27,7 @@ export CXXFLAGS="-fPIC ${CXXFLAGS}" export CFLAGS="-fPIC ${CFLAGS}" # Build shared libraries -make prefix=/usr -j8 install +make prefix=/usr -j${NCORES} install popd rm -rf re2-${RE2_VERSION}.tar.gz re2-${RE2_VERSION} /usr/lib/libre2.so* diff --git a/python/manylinux1/scripts/build_virtualenvs.sh b/python/manylinux1/scripts/build_virtualenvs.sh index a737e2f..beaf133 100755 --- a/python/manylinux1/scripts/build_virtualenvs.sh +++ b/python/manylinux1/scripts/build_virtualenvs.sh @@ -33,7 +33,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do PATH="$PATH:$(cpython_path $PYTHON ${U_WIDTH})" echo "=== (${PYTHON}, ${U_WIDTH}) Installing build dependencies ===" - $PIP install "numpy==1.14.5" "cython==0.29.3" "virtualenv==16.3.0" + $PIP install "numpy==1.14.5" "cython==0.29.8" "virtualenv==16.3.0" # Pandas requires numpy and cython $PIP install "pandas==0.24.0" @@ -44,7 +44,6 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do fi fi - echo "=== (${PYTHON}, ${U_WIDTH}) Preparing virtualenv for tests ===" "$(cpython_path $PYTHON ${U_WIDTH})/bin/virtualenv" -p ${PYTHON_INTERPRETER} --no-download /venv-test-${PYTHON}-${U_WIDTH} source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate @@ -56,6 +55,7 @@ done find /venv-test-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' find /venv-test-*/lib/*/site-packages/numpy -name '*.so' -exec strip '{}' ';' find /opt/_internal/cpython-*/lib/*/site-packages/pandas -name '*.so' -exec strip '{}' ';' +find /opt/_internal/cpython-*/lib/*/site-packages/tensorflow -name '*.so' -exec strip '{}' ';' # Only Python 3.6+ packages are stripable as they are built inside of the image find /opt/_internal/cpython-3.6.*/lib/python3.6/site-packages/numpy -name '*.so' -exec strip '{}' ';' find /opt/_internal/cpython-3.7.*/lib/python3.7/site-packages/numpy -name '*.so' -exec strip '{}' ';' @@ -68,3 +68,5 @@ rm -rf /root/.cache # venv, i.e. 216MiB in total rm -rf /opt/_internal/*/lib/*/site-packages/pandas/tests rm -rf /venv-test-*/lib/*/site-packages/pandas/tests +# Remove unused Python versions +rm -rf /opt/_internal/cpython-3.4* diff --git a/python/manylinux1/scripts/build_zlib.sh b/python/manylinux1/scripts/build_zlib.sh index 182a42f..272b6c4 100755 --- a/python/manylinux1/scripts/build_zlib.sh +++ b/python/manylinux1/scripts/build_zlib.sh @@ -20,7 +20,7 @@ curl -sL https://zlib.net/zlib-1.2.11.tar.gz -o /zlib-1.2.11.tar.gz tar xf zlib-1.2.11.tar.gz pushd zlib-1.2.11 ./configure -make -j4 +make -j8 make install popd rm -rf zlib-1.2.11.tar.gz zlib-1.2.11 diff --git a/python/manylinux1/scripts/build_zstd.sh b/python/manylinux1/scripts/build_zstd.sh index 1add13f..696499a 100755 --- a/python/manylinux1/scripts/build_zstd.sh +++ b/python/manylinux1/scripts/build_zstd.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -export ZSTD_VERSION="1.3.8" +export ZSTD_VERSION="1.4.0" curl -sL "https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz" -o zstd-${ZSTD_VERSION}.tar.gz tar xf zstd-${ZSTD_VERSION}.tar.gz @@ -24,7 +24,14 @@ pushd zstd-${ZSTD_VERSION} mkdir build_cmake pushd build_cmake -cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr ../build/cmake -DZSTD_BUILD_STATIC=ON -DZSTD_BUILD_SHARED=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=1 +cmake -GNinja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr \ + -DZSTD_BUILD_PROGRAMS=off \ + -DZSTD_BUILD_SHARED=off \ + -DZSTD_BUILD_STATIC=on \ + -DZSTD_MULTITHREAD_SUPPORT=off \ + -DCMAKE_POSITION_INDEPENDENT_CODE=1 \ + ../build/cmake ninja install popd diff --git a/python/manylinux1/scripts/install_cmake.sh b/python/manylinux1/scripts/install_cmake.sh index cda2fce..95b8a66 100755 --- a/python/manylinux1/scripts/install_cmake.sh +++ b/python/manylinux1/scripts/install_cmake.sh @@ -16,7 +16,7 @@ # specific language governing permissions and limitations # under the License. -/opt/python/cp35-cp35m/bin/pip install cmake ninja -ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake -ln -s /opt/python/cp35-cp35m/bin/ninja /usr/bin/ninja -strip /opt/_internal/cpython-3.*/lib/python3.5/site-packages/cmake/data/bin/* +/opt/python/cp37-cp37m/bin/pip install cmake ninja +ln -s /opt/python/cp37-cp37m/bin/cmake /usr/bin/cmake +ln -s /opt/python/cp37-cp37m/bin/ninja /usr/bin/ninja +strip /opt/_internal/cpython-3.*/lib/python3.7/site-packages/cmake/data/bin/*