Repository: arrow Updated Branches: refs/heads/master 530f0da34 -> 8229688a6
ARROW-1078: [Python] Account for Apache Parquet shared library consolidation Author: Wes McKinney <wes.mckin...@twosigma.com> Closes #720 from wesm/ARROW-1078 and squashes the following commits: 4bcf580e [Wes McKinney] Remove move_lib for parquet_arrow 9bd6c365 [Wes McKinney] Remove PARQUET_ARROW CMake flag in build fileS 47b38cd6 [Wes McKinney] Clean up after libparquet/libparquet_arrow consolidation Project: http://git-wip-us.apache.org/repos/asf/arrow/repo Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/8229688a Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/8229688a Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/8229688a Branch: refs/heads/master Commit: 8229688a6f6df668f1767f74f277c3454f0c6e6a Parents: 530f0da Author: Wes McKinney <wes.mckin...@twosigma.com> Authored: Wed May 31 18:42:46 2017 -0400 Committer: Wes McKinney <wes.mckin...@twosigma.com> Committed: Wed May 31 18:42:46 2017 -0400 ---------------------------------------------------------------------- ci/travis_script_python.sh | 1 - cpp/doc/Parquet.md | 2 +- python/CMakeLists.txt | 27 ++----------------- python/cmake_modules/FindParquet.cmake | 40 +---------------------------- python/manylinux1/Dockerfile-x86_64 | 2 +- python/setup.py | 1 - 6 files changed, 5 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/arrow/blob/8229688a/ci/travis_script_python.sh ---------------------------------------------------------------------- diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index ce5f7ec..c3735cc 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -49,7 +49,6 @@ build_parquet_cpp() { cmake \ -DCMAKE_BUILD_TYPE=debug \ -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ - -DPARQUET_ARROW=on \ -DPARQUET_BUILD_BENCHMARKS=off \ -DPARQUET_BUILD_EXECUTABLES=off \ -DPARQUET_ZLIB_VENDORED=off \ http://git-wip-us.apache.org/repos/asf/arrow/blob/8229688a/cpp/doc/Parquet.md ---------------------------------------------------------------------- diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md index 4985dd3..ce2961a 100644 --- a/cpp/doc/Parquet.md +++ b/cpp/doc/Parquet.md @@ -24,7 +24,7 @@ export ARROW_HOME=$HOME/local git clone https://github.com/apache/parquet-cpp.git cd parquet-cpp -cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_ARROW=on +cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME make -j4 make install ``` http://git-wip-us.apache.org/repos/asf/arrow/blob/8229688a/python/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b02f23b..f35326a 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -290,7 +290,7 @@ if (PYARROW_BUILD_PARQUET) ## Parquet find_package(Parquet) - if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND)) + if(NOT PARQUET_FOUND) message(FATAL_ERROR "Unable to locate Parquet libraries") endif() include_directories(SYSTEM ${PARQUET_INCLUDE_DIR}) @@ -298,14 +298,10 @@ if (PYARROW_BUILD_PARQUET) if (PYARROW_BUNDLE_ARROW_CPP) get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY) get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE) - get_filename_component(PARQUET_ARROW_LIBRARY_NAME ${PARQUET_ARROW_SHARED_LIB} NAME_WE) message(STATUS "Configure start") configure_file(${PARQUET_SHARED_LIB} ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - configure_file(${PARQUET_ARROW_SHARED_LIB} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_ARROW_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) if (APPLE) configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} @@ -313,12 +309,6 @@ if (PYARROW_BUILD_PARQUET) configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) elseif (NOT MSVC) configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} @@ -326,34 +316,21 @@ if (PYARROW_BUILD_PARQUET) configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} - COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} - COPYONLY) endif() message(STATUS "Configure end") #SET(PARQUET_SHARED_LIB # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}) - #SET(PARQUET_ARROW_SHARED_LIB - # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}) endif() if (MSVC) ADD_THIRDPARTY_LIB(parquet SHARED_LIB ${PARQUET_SHARED_IMP_LIB}) - ADD_THIRDPARTY_LIB(parquet_arrow - SHARED_LIB ${PARQUET_ARROW_SHARED_IMP_LIB}) else() ADD_THIRDPARTY_LIB(parquet SHARED_LIB ${PARQUET_SHARED_LIB}) - ADD_THIRDPARTY_LIB(parquet_arrow - SHARED_LIB ${PARQUET_ARROW_SHARED_LIB}) endif() set(LINK_LIBS ${LINK_LIBS} - parquet_shared - parquet_arrow_shared) + parquet_shared) set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _parquet) http://git-wip-us.apache.org/repos/asf/arrow/blob/8229688a/python/cmake_modules/FindParquet.cmake ---------------------------------------------------------------------- diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake index ef3c645..88dca2e 100644 --- a/python/cmake_modules/FindParquet.cmake +++ b/python/cmake_modules/FindParquet.cmake @@ -41,11 +41,7 @@ if (MSVC) PATHS ${PARQUET_HOME} NO_DEFAULT_PATH PATH_SUFFIXES "bin" ) - find_library(PARQUET_ARROW_SHARED_LIBRARIES NAMES parquet_arrow - PATHS ${PARQUET_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "bin" ) get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH ) - get_filename_component(PARQUET_ARROW_SHARED_LIBS ${PARQUET_ARROW_SHARED_LIBRARIES} PATH ) endif () if(PARQUET_HOME) @@ -63,9 +59,6 @@ if(PARQUET_HOME) find_library(PARQUET_LIBRARIES NAMES parquet PATHS ${PARQUET_HOME} NO_DEFAULT_PATH PATH_SUFFIXES "lib") - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow - PATHS ${PARQUET_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "lib") get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) else() pkg_check_modules(PARQUET parquet) @@ -80,13 +73,9 @@ else() message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}") find_library(PARQUET_LIBRARIES NAMES parquet PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH) - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow - PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH) - message(STATUS "${PARQUET_ARROW_LIBRARIES}") else() find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h ) find_library(PARQUET_LIBRARIES NAMES parquet) - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow) get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) endif() endif() @@ -106,28 +95,9 @@ else () set(PARQUET_FOUND FALSE) endif () -if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES) - set(PARQUET_ARROW_FOUND TRUE) - get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH) - set(PARQUET_ARROW_LIB_NAME parquet_arrow) - if (MSVC) - set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_SHARED_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}") - set(PARQUET_ARROW_SHARED_IMP_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.lib") - else() - set(PARQUET_ARROW_STATIC_LIB - ${PARQUET_ARROW_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}.a) - set(PARQUET_ARROW_SHARED_LIB - ${PARQUET_ARROW_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - endif() -else () - set(PARQUET_ARROW_FOUND FALSE) -endif () - -if (PARQUET_FOUND AND PARQUET_ARROW_FOUND) +if (PARQUET_FOUND) if (NOT Parquet_FIND_QUIETLY) message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}") - message(STATUS "Found the Parquet Arrow library: ${PARQUET_ARROW_LIBS}") endif () else () if (NOT Parquet_FIND_QUIETLY) @@ -135,9 +105,6 @@ else () set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet library.") endif() - if (NOT PARQUET_ARROW_FOUND) - set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet_arrow library. Did you build with -DPARQUET_ARROW=on?") - endif() set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Looked in ") if ( _parquet_roots ) set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.") @@ -159,9 +126,4 @@ mark_as_advanced( PARQUET_LIBRARIES PARQUET_STATIC_LIB PARQUET_SHARED_LIB - - PARQUET_ARROW_FOUND - PARQUET_ARROW_LIBS - PARQUET_ARROW_STATIC_LIB - PARQUET_ARROW_SHARED_LIB ) http://git-wip-us.apache.org/repos/asf/arrow/blob/8229688a/python/manylinux1/Dockerfile-x86_64 ---------------------------------------------------------------------- diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 index 363308b..11e66f7 100644 --- a/python/manylinux1/Dockerfile-x86_64 +++ b/python/manylinux1/Dockerfile-x86_64 @@ -21,5 +21,5 @@ RUN make -j5 install WORKDIR / RUN git clone https://github.com/apache/parquet-cpp.git WORKDIR /parquet-cpp -RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF . +RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BOOST_USE_SHARED=OFF . RUN make -j5 install http://git-wip-us.apache.org/repos/asf/arrow/blob/8229688a/python/setup.py ---------------------------------------------------------------------- diff --git a/python/setup.py b/python/setup.py index 7e612c9..b64e259 100644 --- a/python/setup.py +++ b/python/setup.py @@ -248,7 +248,6 @@ class build_ext(_build_ext): move_lib("arrow_jemalloc") if self.with_parquet: move_lib("parquet") - move_lib("parquet_arrow") # Move the built C-extension to the place expected by the Python build self._found_names = []
