This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 8b1c811 ARROW-2245: ARROW-2246: [Python] Revert static linkage of
parquet-cpp in manylinux1 wheel
8b1c811 is described below
commit 8b1c8118b017a941f0102709d72df7e5a9783aa4
Author: Uwe L. Korn <[email protected]>
AuthorDate: Sun Mar 4 14:40:12 2018 -0500
ARROW-2245: ARROW-2246: [Python] Revert static linkage of parquet-cpp in
manylinux1 wheel
Author: Uwe L. Korn <[email protected]>
Closes #1692 from xhochy/ARROW-2245 and squashes the following commits:
fa33dd5e <Uwe L. Korn> ARROW-2245: ARROW-2246: Revert static linkage of
parquet-cpp in manylinux1 wheel
---
cpp/cmake_modules/ThirdpartyToolchain.cmake | 3 ++-
cpp/src/plasma/CMakeLists.txt | 10 ++++++++++
python/CMakeLists.txt | 29 ++++++++++++++++++++++++++++-
python/manylinux1/Dockerfile-x86_64 | 6 +++---
python/manylinux1/build_arrow.sh | 11 +++++------
python/manylinux1/scripts/build_boost.sh | 20 ++++++++++++--------
python/manylinux1/scripts/build_thrift.sh | 8 +++++---
python/setup.py | 6 +++++-
8 files changed, 70 insertions(+), 23 deletions(-)
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 4103af4..c330e2a 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -501,10 +501,11 @@ if (ARROW_JEMALLOC)
set(JEMALLOC_SHARED_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(JEMALLOC_STATIC_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(JEMALLOC_VENDORED 1)
+ # We need to disable TLS or otherwise C++ exceptions won't work anymore.
ExternalProject_Add(jemalloc_ep
URL
${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
- CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+ CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
"--disable-tls"
${EP_LOG_OPTIONS}
BUILD_IN_SOURCE 1
BUILD_COMMAND ${MAKE}
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 3448d00..bc00f98 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -124,6 +124,16 @@ endif()
add_executable(plasma_store store.cc)
target_link_libraries(plasma_store plasma_static ${PLASMA_LINK_LIBS})
+if (ARROW_RPATH_ORIGIN)
+ if (APPLE)
+ set(_lib_install_rpath "@loader_path")
+ else()
+ set(_lib_install_rpath "\$ORIGIN")
+ endif()
+ set_target_properties(plasma_store PROPERTIES
+ INSTALL_RPATH ${_lib_install_rpath})
+endif()
+
# Headers: top level
install(FILES
common.h
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index e9de08b..72294d4 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -66,7 +66,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL
"${CMAKE_CURRENT_SOURCE_DIR}")
ON)
option(PYARROW_BOOST_USE_SHARED
"Rely on boost shared libraries on linking static parquet"
- OFF)
+ ON)
option(PYARROW_BUILD_PLASMA
"Build the PyArrow Plasma integration"
OFF)
@@ -235,6 +235,24 @@ function(bundle_arrow_implib library_path)
COPYONLY)
endfunction(bundle_arrow_implib)
+function(bundle_boost_lib library_path)
+ get_filename_component(LIBRARY_NAME ${${library_path}} NAME)
+ get_filename_component(LIBRARY_NAME_WE ${${library_path}} NAME_WE)
+ configure_file(${${library_path}}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}
+ COPYONLY)
+ set(Boost_SO_VERSION
"${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
+ if (APPLE)
+ configure_file(${${library_path}}
+
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}.${Boost_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ COPYONLY)
+ else()
+ configure_file(${${library_path}}
+
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}${CMAKE_SHARED_LIBRARY_SUFFIX}.${Boost_SO_VERSION}
+ COPYONLY)
+ endif()
+endfunction()
+
# Always bundle includes
file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION
${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
@@ -247,6 +265,15 @@ if (PYARROW_BUNDLE_ARROW_CPP)
ABI_VERSION ${ARROW_ABI_VERSION}
SO_VERSION ${ARROW_SO_VERSION})
+ # boost
+ if (PYARROW_BOOST_USE_SHARED)
+ set(Boost_USE_STATIC_LIBS OFF)
+ find_package(Boost COMPONENTS system filesystem regex REQUIRED)
+ bundle_boost_lib(Boost_REGEX_LIBRARY)
+ bundle_boost_lib(Boost_FILESYSTEM_LIBRARY)
+ bundle_boost_lib(Boost_SYSTEM_LIBRARY)
+ endif()
+
if (MSVC)
bundle_arrow_implib(ARROW_SHARED_IMP_LIB)
bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB)
diff --git a/python/manylinux1/Dockerfile-x86_64
b/python/manylinux1/Dockerfile-x86_64
index 62a0893..d48bd0d 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,13 +14,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2212
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2245
ADD arrow /arrow
WORKDIR /arrow/cpp
RUN mkdir build-plain
WORKDIR /arrow/cpp/build-plain
-RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON
-DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
+RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=ON -DARROW_JEMALLOC=ON
-DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF
-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist ..
RUN ninja install
ADD scripts/check_arrow_visibility.sh /
@@ -29,5 +29,5 @@ RUN /check_arrow_visibility.sh
WORKDIR /
RUN git clone https://github.com/apache/parquet-cpp.git
WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF
-DPARQUET_BUILD_SHARED=OFF -DPARQUET_BUILD_STATIC=ON
-DPARQUET_BOOST_USE_SHARED=OFF -GNinja .
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF
-DPARQUET_BUILD_SHARED=ON -DPARQUET_BUILD_STATIC=OFF
-DPARQUET_BOOST_USE_SHARED=ON -DBoost_NAMESPACE=arrow_boost
-DBOOST_ROOT=/arrow_boost_dist -GNinja .
RUN ninja install
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 5fd27c8..f83c759 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -38,11 +38,10 @@ cd /arrow/python
# PyArrow build configuration
export PYARROW_BUILD_TYPE='release'
export PYARROW_WITH_PARQUET=1
-export PYARROW_WITH_STATIC_PARQUET=1
export PYARROW_WITH_PLASMA=1
export PYARROW_BUNDLE_ARROW_CPP=1
export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
-export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr'
+export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost
-DBOOST_ROOT=/arrow_boost_dist'
# Ensure the target directory exists
mkdir -p /io/dist
@@ -60,14 +59,14 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}-${U_WIDTH}
mkdir -p "${ARROW_BUILD_DIR}"
pushd "${ARROW_BUILD_DIR}"
- PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off
-DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON
-DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON ..
- make -j5 install
+ PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=ON -DARROW_JEMALLOC=ON
-DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON}
-DARROW_PLASMA=ON -DARROW_ORC=ON -DBoost_NAMESPACE=arrow_boost
-DBOOST_ROOT=/arrow_boost_dist -GNinja ..
+ ninja install
popd
# Clear output directory
rm -rf dist/
echo "=== (${PYTHON}) Building wheel ==="
- PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext
--inplace --with-parquet --with-static-parquet --bundle-arrow-cpp
+ PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext
--inplace --with-parquet --bundle-arrow-cpp
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
echo "=== (${PYTHON}) Test the existence of optional modules ==="
@@ -83,7 +82,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
pip install repaired_wheels/*.whl
- py.test -v -r sxX --durations=15 --parquet
/venv-test-${PYTHON}-${U_WIDTH}/lib/*/site-packages/pyarrow
+ py.test -v -r sxX --durations=15 --parquet
${VIRTUAL_ENV}/lib/*/site-packages/pyarrow
deactivate
mv repaired_wheels/*.whl /io/dist
diff --git a/python/manylinux1/scripts/build_boost.sh
b/python/manylinux1/scripts/build_boost.sh
index 1a6ffd7..47d8262 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -21,18 +21,22 @@ BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
wget --no-check-certificate
https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
-O /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
+mkdir /arrow_boost
pushd /boost_${BOOST_VERSION_UNDERSCORE}
./bootstrap.sh
-./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=/usr
--with-filesystem --with-date_time --with-system --with-regex install
+./b2 tools/bcp
+./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time
system regex build algorithm locale format /arrow_boost
popd
-rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
boost_${BOOST_VERSION_UNDERSCORE}
+
+pushd /arrow_boost
+ls -l
+./bootstrap.sh
+./bjam cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11"
variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem
--with-date_time --with-system --with-regex install
+popd
+rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
boost_${BOOST_VERSION_UNDERSCORE} arrow_boost
# Boost always install header-only parts but they also take up quite some
space.
# We don't need them in array, so don't persist them in the docker layer.
-# phoenix 18.1 MiB
-rm -r /usr/include/boost/phoenix
# fusion 16.7 MiB
-rm -r /usr/include/boost/fusion
+rm -r /arrow_boost_dist/include/boost/fusion
# spirit 8.2 MiB
-rm -r /usr/include/boost/spirit
-# geometry 6.0 MiB
-rm -r /usr/include/boost/geometry
+rm -r /arrow_boost_dist/include/boost/spirit
diff --git a/python/manylinux1/scripts/build_thrift.sh
b/python/manylinux1/scripts/build_thrift.sh
index aaec4ad..451025c 100755
--- a/python/manylinux1/scripts/build_thrift.sh
+++ b/python/manylinux1/scripts/build_thrift.sh
@@ -34,9 +34,11 @@ cmake -DCMAKE_BUILD_TYPE=release \
"-DWITH_JAVA=OFF" \
"-DWITH_PYTHON=OFF" \
"-DWITH_CPP=ON" \
- "-DWITH_STATIC_LIB=ON" ..
-make -j5
-make install
+ "-DWITH_STATIC_LIB=ON" \
+ -DBoost_NAMESPACE=arrow_boost \
+ -DBOOST_ROOT=/arrow_boost_dist \
+ -GNinja ..
+ninja install
popd
popd
rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION}
diff --git a/python/setup.py b/python/setup.py
index 2a63264..34d2d90 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -120,7 +120,7 @@ class build_ext(_build_ext):
self.with_static_parquet = strtobool(
os.environ.get('PYARROW_WITH_STATIC_PARQUET', '0'))
self.with_static_boost = strtobool(
- os.environ.get('PYARROW_WITH_STATIC_BOOST', '1'))
+ os.environ.get('PYARROW_WITH_STATIC_BOOST', '0'))
self.with_plasma = strtobool(
os.environ.get('PYARROW_WITH_PLASMA', '0'))
self.with_orc = strtobool(
@@ -255,6 +255,10 @@ class build_ext(_build_ext):
move_shared_libs(build_prefix, build_lib, "plasma")
if self.with_parquet and not self.with_static_parquet:
move_shared_libs(build_prefix, build_lib, "parquet")
+ if not self.with_static_boost:
+ move_shared_libs(build_prefix, build_lib,
"arrow_boost_filesystem")
+ move_shared_libs(build_prefix, build_lib,
"arrow_boost_system")
+ move_shared_libs(build_prefix, build_lib,
"arrow_boost_regex")
print('Bundling includes: ' + pjoin(build_prefix, 'include'))
if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
--
To stop receiving notification emails like this one, please contact
[email protected].
