[
https://issues.apache.org/jira/browse/ARROW-2245?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16385315#comment-16385315
]
ASF GitHub Bot commented on ARROW-2245:
---------------------------------------
wesm closed pull request #1692: ARROW-2245: ARROW-2246: [Python] Revert static
linkage of parquet-cpp in manylinux1 wheel
URL: https://github.com/apache/arrow/pull/1692
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
index 4103af41b..c330e2ae3 100644
--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
@@ -501,10 +501,11 @@ if (ARROW_JEMALLOC)
set(JEMALLOC_SHARED_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(JEMALLOC_STATIC_LIB
"${JEMALLOC_PREFIX}/lib/libjemalloc_pic${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(JEMALLOC_VENDORED 1)
+ # We need to disable TLS or otherwise C++ exceptions won't work anymore.
ExternalProject_Add(jemalloc_ep
URL
${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/jemalloc/${JEMALLOC_VERSION}.tar.gz
PATCH_COMMAND touch doc/jemalloc.3 doc/jemalloc.html
- CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
+ CONFIGURE_COMMAND ./autogen.sh "--prefix=${JEMALLOC_PREFIX}"
"--with-jemalloc-prefix=je_arrow_" "--with-private-namespace=je_arrow_private_"
"--disable-tls"
${EP_LOG_OPTIONS}
BUILD_IN_SOURCE 1
BUILD_COMMAND ${MAKE}
diff --git a/cpp/src/plasma/CMakeLists.txt b/cpp/src/plasma/CMakeLists.txt
index 3448d009c..bc00f9806 100644
--- a/cpp/src/plasma/CMakeLists.txt
+++ b/cpp/src/plasma/CMakeLists.txt
@@ -124,6 +124,16 @@ endif()
add_executable(plasma_store store.cc)
target_link_libraries(plasma_store plasma_static ${PLASMA_LINK_LIBS})
+if (ARROW_RPATH_ORIGIN)
+ if (APPLE)
+ set(_lib_install_rpath "@loader_path")
+ else()
+ set(_lib_install_rpath "\$ORIGIN")
+ endif()
+ set_target_properties(plasma_store PROPERTIES
+ INSTALL_RPATH ${_lib_install_rpath})
+endif()
+
# Headers: top level
install(FILES
common.h
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index e9de08ba1..72294d494 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -66,7 +66,7 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL
"${CMAKE_CURRENT_SOURCE_DIR}")
ON)
option(PYARROW_BOOST_USE_SHARED
"Rely on boost shared libraries on linking static parquet"
- OFF)
+ ON)
option(PYARROW_BUILD_PLASMA
"Build the PyArrow Plasma integration"
OFF)
@@ -235,6 +235,24 @@ function(bundle_arrow_implib library_path)
COPYONLY)
endfunction(bundle_arrow_implib)
+function(bundle_boost_lib library_path)
+ get_filename_component(LIBRARY_NAME ${${library_path}} NAME)
+ get_filename_component(LIBRARY_NAME_WE ${${library_path}} NAME_WE)
+ configure_file(${${library_path}}
+ ${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME}
+ COPYONLY)
+ set(Boost_SO_VERSION
"${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
+ if (APPLE)
+ configure_file(${${library_path}}
+
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}.${Boost_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX}
+ COPYONLY)
+ else()
+ configure_file(${${library_path}}
+
${BUILD_OUTPUT_ROOT_DIRECTORY}/${LIBRARY_NAME_WE}${CMAKE_SHARED_LIBRARY_SUFFIX}.${Boost_SO_VERSION}
+ COPYONLY)
+ endif()
+endfunction()
+
# Always bundle includes
file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION
${BUILD_OUTPUT_ROOT_DIRECTORY}/include)
@@ -247,6 +265,15 @@ if (PYARROW_BUNDLE_ARROW_CPP)
ABI_VERSION ${ARROW_ABI_VERSION}
SO_VERSION ${ARROW_SO_VERSION})
+ # boost
+ if (PYARROW_BOOST_USE_SHARED)
+ set(Boost_USE_STATIC_LIBS OFF)
+ find_package(Boost COMPONENTS system filesystem regex REQUIRED)
+ bundle_boost_lib(Boost_REGEX_LIBRARY)
+ bundle_boost_lib(Boost_FILESYSTEM_LIBRARY)
+ bundle_boost_lib(Boost_SYSTEM_LIBRARY)
+ endif()
+
if (MSVC)
bundle_arrow_implib(ARROW_SHARED_IMP_LIB)
bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB)
diff --git a/python/manylinux1/Dockerfile-x86_64
b/python/manylinux1/Dockerfile-x86_64
index 62a089329..d48bd0d2c 100644
--- a/python/manylinux1/Dockerfile-x86_64
+++ b/python/manylinux1/Dockerfile-x86_64
@@ -14,13 +14,13 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2212
+FROM quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-2245
ADD arrow /arrow
WORKDIR /arrow/cpp
RUN mkdir build-plain
WORKDIR /arrow/cpp/build-plain
-RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=ON
-DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF ..
+RUN cmake -GNinja -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=ON -DARROW_JEMALLOC=ON
-DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF
-DBoost_NAMESPACE=arrow_boost -DBOOST_ROOT=/arrow_boost_dist ..
RUN ninja install
ADD scripts/check_arrow_visibility.sh /
@@ -29,5 +29,5 @@ RUN /check_arrow_visibility.sh
WORKDIR /
RUN git clone https://github.com/apache/parquet-cpp.git
WORKDIR /parquet-cpp
-RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF
-DPARQUET_BUILD_SHARED=OFF -DPARQUET_BUILD_STATIC=ON
-DPARQUET_BOOST_USE_SHARED=OFF -GNinja .
+RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF
-DPARQUET_BUILD_SHARED=ON -DPARQUET_BUILD_STATIC=OFF
-DPARQUET_BOOST_USE_SHARED=ON -DBoost_NAMESPACE=arrow_boost
-DBOOST_ROOT=/arrow_boost_dist -GNinja .
RUN ninja install
diff --git a/python/manylinux1/build_arrow.sh b/python/manylinux1/build_arrow.sh
index 5fd27c8d0..f83c75972 100755
--- a/python/manylinux1/build_arrow.sh
+++ b/python/manylinux1/build_arrow.sh
@@ -38,11 +38,10 @@ cd /arrow/python
# PyArrow build configuration
export PYARROW_BUILD_TYPE='release'
export PYARROW_WITH_PARQUET=1
-export PYARROW_WITH_STATIC_PARQUET=1
export PYARROW_WITH_PLASMA=1
export PYARROW_BUNDLE_ARROW_CPP=1
export PKG_CONFIG_PATH=/arrow-dist/lib64/pkgconfig
-export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr'
+export PYARROW_CMAKE_OPTIONS='-DTHRIFT_HOME=/usr -DBoost_NAMESPACE=arrow_boost
-DBOOST_ROOT=/arrow_boost_dist'
# Ensure the target directory exists
mkdir -p /io/dist
@@ -60,14 +59,14 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
ARROW_BUILD_DIR=/arrow/cpp/build-PY${PYTHON}-${U_WIDTH}
mkdir -p "${ARROW_BUILD_DIR}"
pushd "${ARROW_BUILD_DIR}"
- PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=OFF -DARROW_JEMALLOC=off
-DARROW_RPATH_ORIGIN=ON -DARROW_JEMALLOC_USE_SHARED=OFF -DARROW_PYTHON=ON
-DPythonInterp_FIND_VERSION=${PYTHON} -DARROW_PLASMA=ON -DARROW_ORC=ON ..
- make -j5 install
+ PATH="${CPYTHON_PATH}/bin:$PATH" cmake -DCMAKE_BUILD_TYPE=Release
-DCMAKE_INSTALL_PREFIX=/arrow-dist -DARROW_BUILD_TESTS=OFF
-DARROW_BUILD_SHARED=ON -DARROW_BOOST_USE_SHARED=ON -DARROW_JEMALLOC=ON
-DARROW_RPATH_ORIGIN=ON -DARROW_PYTHON=ON -DPythonInterp_FIND_VERSION=${PYTHON}
-DARROW_PLASMA=ON -DARROW_ORC=ON -DBoost_NAMESPACE=arrow_boost
-DBOOST_ROOT=/arrow_boost_dist -GNinja ..
+ ninja install
popd
# Clear output directory
rm -rf dist/
echo "=== (${PYTHON}) Building wheel ==="
- PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext
--inplace --with-parquet --with-static-parquet --bundle-arrow-cpp
+ PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py build_ext
--inplace --with-parquet --bundle-arrow-cpp
PATH="$PATH:${CPYTHON_PATH}/bin" $PYTHON_INTERPRETER setup.py bdist_wheel
echo "=== (${PYTHON}) Test the existence of optional modules ==="
@@ -83,7 +82,7 @@ for PYTHON_TUPLE in ${PYTHON_VERSIONS}; do
source /venv-test-${PYTHON}-${U_WIDTH}/bin/activate
pip install repaired_wheels/*.whl
- py.test -v -r sxX --durations=15 --parquet
/venv-test-${PYTHON}-${U_WIDTH}/lib/*/site-packages/pyarrow
+ py.test -v -r sxX --durations=15 --parquet
${VIRTUAL_ENV}/lib/*/site-packages/pyarrow
deactivate
mv repaired_wheels/*.whl /io/dist
diff --git a/python/manylinux1/scripts/build_boost.sh
b/python/manylinux1/scripts/build_boost.sh
index 1a6ffd7eb..47d826251 100755
--- a/python/manylinux1/scripts/build_boost.sh
+++ b/python/manylinux1/scripts/build_boost.sh
@@ -21,18 +21,22 @@ BOOST_VERSION_UNDERSCORE=${BOOST_VERSION//\./_}
wget --no-check-certificate
https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
-O /boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
tar xf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
+mkdir /arrow_boost
pushd /boost_${BOOST_VERSION_UNDERSCORE}
./bootstrap.sh
-./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=/usr
--with-filesystem --with-date_time --with-system --with-regex install
+./b2 tools/bcp
+./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time
system regex build algorithm locale format /arrow_boost
popd
-rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
boost_${BOOST_VERSION_UNDERSCORE}
+
+pushd /arrow_boost
+ls -l
+./bootstrap.sh
+./bjam cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11"
variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem
--with-date_time --with-system --with-regex install
+popd
+rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz
boost_${BOOST_VERSION_UNDERSCORE} arrow_boost
# Boost always install header-only parts but they also take up quite some
space.
# We don't need them in array, so don't persist them in the docker layer.
-# phoenix 18.1 MiB
-rm -r /usr/include/boost/phoenix
# fusion 16.7 MiB
-rm -r /usr/include/boost/fusion
+rm -r /arrow_boost_dist/include/boost/fusion
# spirit 8.2 MiB
-rm -r /usr/include/boost/spirit
-# geometry 6.0 MiB
-rm -r /usr/include/boost/geometry
+rm -r /arrow_boost_dist/include/boost/spirit
diff --git a/python/manylinux1/scripts/build_thrift.sh
b/python/manylinux1/scripts/build_thrift.sh
index aaec4ad6b..451025c66 100755
--- a/python/manylinux1/scripts/build_thrift.sh
+++ b/python/manylinux1/scripts/build_thrift.sh
@@ -34,9 +34,11 @@ cmake -DCMAKE_BUILD_TYPE=release \
"-DWITH_JAVA=OFF" \
"-DWITH_PYTHON=OFF" \
"-DWITH_CPP=ON" \
- "-DWITH_STATIC_LIB=ON" ..
-make -j5
-make install
+ "-DWITH_STATIC_LIB=ON" \
+ -DBoost_NAMESPACE=arrow_boost \
+ -DBOOST_ROOT=/arrow_boost_dist \
+ -GNinja ..
+ninja install
popd
popd
rm -rf thrift-${THRIFT_VERSION}.tar.gz thrift-${THRIFT_VERSION}
diff --git a/python/setup.py b/python/setup.py
index 2a6326489..34d2d9079 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -120,7 +120,7 @@ def initialize_options(self):
self.with_static_parquet = strtobool(
os.environ.get('PYARROW_WITH_STATIC_PARQUET', '0'))
self.with_static_boost = strtobool(
- os.environ.get('PYARROW_WITH_STATIC_BOOST', '1'))
+ os.environ.get('PYARROW_WITH_STATIC_BOOST', '0'))
self.with_plasma = strtobool(
os.environ.get('PYARROW_WITH_PLASMA', '0'))
self.with_orc = strtobool(
@@ -255,6 +255,10 @@ def _run_cmake(self):
move_shared_libs(build_prefix, build_lib, "plasma")
if self.with_parquet and not self.with_static_parquet:
move_shared_libs(build_prefix, build_lib, "parquet")
+ if not self.with_static_boost:
+ move_shared_libs(build_prefix, build_lib,
"arrow_boost_filesystem")
+ move_shared_libs(build_prefix, build_lib,
"arrow_boost_system")
+ move_shared_libs(build_prefix, build_lib,
"arrow_boost_regex")
print('Bundling includes: ' + pjoin(build_prefix, 'include'))
if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Python] Revert static linkage of parquet-cpp in manylinux1 wheel
> -----------------------------------------------------------------
>
> Key: ARROW-2245
> URL: https://issues.apache.org/jira/browse/ARROW-2245
> Project: Apache Arrow
> Issue Type: Bug
> Components: Python
> Reporter: Uwe L. Korn
> Assignee: Uwe L. Korn
> Priority: Major
> Labels: pull-request-available
> Fix For: 0.9.0
>
>
> Although we are not in a theoretical way the authoritative source of
> parquet-cpp with the pyarrow manylinux1 wheel, in practical way we actually
> are this and statically linking parquet-cpp can introduce some problems that
> dynamically linking it does not (e.g. duplicate unloading of the library if
> you include it in a Python wheel and in the process that creates the Python
> interpreter).
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
