Repository: arrow Updated Branches: refs/heads/master 569426b91 -> 02a121f18
ARROW-927: C++/Python: Add manylinux1 builds to Travis matrix Also pre-builds flatbuffers and gtest so that they are only build once. Author: Uwe L. Korn <uw...@xhochy.com> Closes #624 from xhochy/ARROW-927 and squashes the following commits: c901b7a [Uwe L. Korn] Separate scripts folder 0dd4e08 [Uwe L. Korn] Add scripts folder e705813 [Uwe L. Korn] Move boost and openssl to scripts 1b44878 [Uwe L. Korn] Add base image for thirdparties f4ff321 [Uwe L. Korn] ARROW-927: C++/Python: Add manylinux1 builds to Travis matrix Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/02a121f1 Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/02a121f1 Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/02a121f1 Branch: refs/heads/master Commit: 02a121f18b6b5a34b63dd8d2bf7b1955ac7e11b2 Parents: 569426b Author: Uwe L. Korn <uw...@xhochy.com> Authored: Tue May 2 13:40:47 2017 -0400 Committer: Wes McKinney <wes.mckin...@twosigma.com> Committed: Tue May 2 13:40:47 2017 -0400 ---------------------------------------------------------------------- .travis.yml | 11 +++++ python/manylinux1/Dockerfile-x86_64 | 41 +------------------ python/manylinux1/Dockerfile-x86_64_base | 52 ++++++++++++++++++++++++ python/manylinux1/build_arrow.sh | 3 +- python/manylinux1/scripts/build_boost.sh | 21 ++++++++++ python/manylinux1/scripts/build_jemalloc.sh | 21 ++++++++++ python/manylinux1/scripts/build_openssl.sh | 21 ++++++++++ 7 files changed, 128 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/02a121f1/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index 6ebebd4..19e71ae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,6 +19,8 @@ addons: - gtk-doc-tools - autoconf-archive - libgirepository1.0-dev +services: + - docker cache: ccache: true @@ -54,6 +56,15 @@ matrix: script: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh + - language: cpp + before_script: + - docker pull quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927 + script: | + pushd python/manylinux1 + git clone ../../ arrow + docker build -t arrow-base-x86_64 -f Dockerfile-x86_64 . + docker run --rm -v $PWD:/io arrow-base-x86_64 /io/build_arrow.sh + ls -l dist/ - language: java os: linux jdk: oraclejdk7 http://git-wip-us.apache.org/repos/asf/arrow/blob/02a121f1/python/manylinux1/Dockerfile-x86_64 ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 index 56b27ad..8f55ba7 100644 --- a/python/manylinux1/Dockerfile-x86_64 +++ b/python/manylinux1/Dockerfile-x86_64 @@ -9,46 +9,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. See accompanying LICENSE file. - -FROM quay.io/pypa/manylinux1_x86_64:latest - -# Install dependencies -RUN yum install -y flex zlib-devel - -# Build a newer OpenSSL version to support Thrift 0.10.0, note that we don't trigger the SSL code in Arrow. -WORKDIR / -RUN wget --no-check-certificate https://www.openssl.org/source/openssl-1.0.2k.tar.gz -O openssl-1.0.2k.tar.gz -RUN tar xf openssl-1.0.2k.tar.gz -WORKDIR openssl-1.0.2k -RUN ./config -fpic shared --prefix=/usr -RUN make -j5 -RUN make install - -WORKDIR / -RUN wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz -RUN tar xf boost_1_60_0.tar.gz -WORKDIR /boost_1_60_0 -RUN ./bootstrap.sh -RUN ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install - -WORKDIR / -RUN wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2 -RUN tar xf jemalloc-4.4.0.tar.bz2 -WORKDIR /jemalloc-4.4.0 -RUN ./configure -RUN make -j5 -RUN make install - -WORKDIR / -# Install cmake manylinux1 package -RUN /opt/python/cp35-cp35m/bin/pip install cmake -RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake - -WORKDIR / -RUN git clone https://github.com/matthew-brett/multibuild.git -WORKDIR /multibuild -RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963 - +FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-927 ADD arrow /arrow WORKDIR /arrow/cpp http://git-wip-us.apache.org/repos/asf/arrow/blob/02a121f1/python/manylinux1/Dockerfile-x86_64_base ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-x86_64_base b/python/manylinux1/Dockerfile-x86_64_base new file mode 100644 index 0000000..e38296d --- /dev/null +++ b/python/manylinux1/Dockerfile-x86_64_base @@ -0,0 +1,52 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. +FROM quay.io/pypa/manylinux1_x86_64:latest + +# Install dependencies +RUN yum install -y flex zlib-devel + +ADD scripts/build_openssl.sh / +RUN /build_openssl.sh + +ADD scripts/build_boost.sh / +RUN /build_boost.sh + +ADD scripts/build_jemalloc.sh / +RUN /build_jemalloc.sh + +WORKDIR / +# Install cmake manylinux1 package +RUN /opt/python/cp35-cp35m/bin/pip install cmake +RUN ln -s /opt/python/cp35-cp35m/bin/cmake /usr/bin/cmake + +WORKDIR / +RUN git clone https://github.com/matthew-brett/multibuild.git +WORKDIR /multibuild +RUN git checkout ffe59955ad8690c2f8bb74766cb7e9b0d0ee3963 + +WORKDIR / +RUN wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz -O googletest-release-1.7.0.tar.gz +RUN tar xf googletest-release-1.7.0.tar.gz +WORKDIR /googletest-release-1.7.0 +RUN cmake -DCMAKE_CXX_FLAGS='-fPIC' -Dgtest_force_shared_crt=ON . +RUN make -j5 +ENV GTEST_HOME /googletest-release-1.7.0 + +WORKDIR / +RUN wget https://github.com/google/flatbuffers/archive/v1.6.0.tar.gz -O flatbuffers-1.6.0.tar.gz +RUN tar xf flatbuffers-1.6.0.tar.gz +WORKDIR /flatbuffers-1.6.0 +RUN cmake "-DCMAKE_CXX_FLAGS=-fPIC" "-DCMAKE_INSTALL_PREFIX:PATH=/usr" "-DFLATBUFFERS_BUILD_TESTS=OFF" +RUN make -j5 +RUN make install +ENV FLATBUFFERS_HOME /usr + http://git-wip-us.apache.org/repos/asf/arrow/blob/02a121f1/python/manylinux1/build_arrow.sh ---------------------------------------------------------------------- diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh index 8ef087c..a11d3d4 100755 --- a/python/manylinux1/build_arrow.sh +++ b/python/manylinux1/build_arrow.sh @@ -31,7 +31,6 @@ cd /arrow/python # PyArrow build configuration export PYARROW_BUILD_TYPE='release' -export PYARROW_CMAKE_OPTIONS='-DPYARROW_BUILD_TESTS=ON' export PYARROW_WITH_PARQUET=1 export PYARROW_WITH_JEMALLOC=1 export PYARROW_BUNDLE_ARROW_CPP=1 @@ -51,7 +50,7 @@ for PYTHON in ${PYTHON_VERSIONS}; do echo "=== (${PYTHON}) Installing build dependencies ===" $PIPI_IO "numpy==1.9.0" - $PIPI_IO "cython==0.24" + $PIPI_IO "cython==0.25.2" $PIPI_IO "pandas==0.19.2" echo "=== (${PYTHON}) Building Arrow C++ libraries ===" http://git-wip-us.apache.org/repos/asf/arrow/blob/02a121f1/python/manylinux1/scripts/build_boost.sh ---------------------------------------------------------------------- diff --git a/python/manylinux1/scripts/build_boost.sh b/python/manylinux1/scripts/build_boost.sh new file mode 100755 index 0000000..6a31336 --- /dev/null +++ b/python/manylinux1/scripts/build_boost.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + + +wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O /boost_1_60_0.tar.gz +tar xf boost_1_60_0.tar.gz +pushd /boost_1_60_0 +./bootstrap.sh +./bjam cxxflags=-fPIC cflags=-fPIC --prefix=/usr --with-filesystem --with-date_time --with-system --with-regex install +popd +rm -rf boost_1_60_0.tar.gz boost_1_60_0 http://git-wip-us.apache.org/repos/asf/arrow/blob/02a121f1/python/manylinux1/scripts/build_jemalloc.sh ---------------------------------------------------------------------- diff --git a/python/manylinux1/scripts/build_jemalloc.sh b/python/manylinux1/scripts/build_jemalloc.sh new file mode 100755 index 0000000..8153baa --- /dev/null +++ b/python/manylinux1/scripts/build_jemalloc.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2 -O jemalloc-4.4.0.tar.bz2 +tar xf jemalloc-4.4.0.tar.bz2 +pushd /jemalloc-4.4.0 +./configure +make -j5 +make install +popd +rm -rf jemalloc-4.4.0.tar.bz2 jemalloc-4.4.0 http://git-wip-us.apache.org/repos/asf/arrow/blob/02a121f1/python/manylinux1/scripts/build_openssl.sh ---------------------------------------------------------------------- diff --git a/python/manylinux1/scripts/build_openssl.sh b/python/manylinux1/scripts/build_openssl.sh new file mode 100755 index 0000000..3bcb2b9 --- /dev/null +++ b/python/manylinux1/scripts/build_openssl.sh @@ -0,0 +1,21 @@ +#!/bin/bash -ex +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. See accompanying LICENSE file. + +wget --no-check-certificate https://www.openssl.org/source/openssl-1.0.2k.tar.gz -O openssl-1.0.2k.tar.gz +tar xf openssl-1.0.2k.tar.gz +pushd openssl-1.0.2k +./config -fpic shared --prefix=/usr +make -j5 +make install +popd +rm -rf openssl-1.0.2k.tar.gz openssl-1.0.2k
