This is an automated email from the ASF dual-hosted git repository. hubgeter pushed a commit to branch revert-66546-worktree/dual-arrow-17-24 in repository https://gitbox.apache.org/repos/asf/doris.git
commit b3b89e3824301c798cd4bd25f1ab0f9b5e536e7c Author: daidai <[email protected]> AuthorDate: Mon Aug 31 14:36:32 2026 +0800 Revert "[fix](build) Keep Arrow 17 and 24 in shared thirdparty (#66546)" This reverts commit e5f6299340f2ba8984e6bc347a2a5836c6b60dd7. --- be/CMakeLists.txt | 42 +-- be/cmake/thirdparty.cmake | 59 ++-- build.sh | 53 +--- .../adbc/test_adbc_type_mapping.groovy | 7 +- thirdparty/arrow-paimon-vars.sh | 341 ++------------------- thirdparty/build-thirdparty.sh | 121 ++------ thirdparty/download-thirdparty.sh | 67 +--- thirdparty/paimon-cpp-cache.cmake | 27 +- ...arrow-17.0.0-force-write-int96-timestamps.patch | 98 ------ thirdparty/patches/apache-arrow-17.0.0-lzo.patch | 84 ----- .../patches/apache-arrow-17.0.0-paimon.patch | 224 -------------- thirdparty/test/arrow-paimon-lifecycle-test.sh | 265 +--------------- thirdparty/vars.sh | 3 - 13 files changed, 130 insertions(+), 1261 deletions(-) diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index a795923540b..524d0322873 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -200,6 +200,7 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF for message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}") option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON) +set(PAIMON_HOME "" CACHE PATH "Paimon install prefix") option(ENABLE_TDE "Enable TDE feature module" OFF) set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src") option(ENABLE_TLS "Enable TLS feature module" OFF) @@ -221,31 +222,8 @@ endif() if (DEFINED ENV{ENABLE_PAIMON_CPP}) set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE) endif() - -# Refresh automatic selections on every configure so a reused CMake cache -# follows the current DORIS_THIRDPARTY. Environment overrides are explicit and -# likewise replace any earlier automatic or explicit cache value. -if (DEFINED ENV{ARROW_HOME}) - set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "Selected Arrow install prefix" FORCE) -else() - set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH - "Selected Arrow install prefix" FORCE) -endif() -if (DEFINED ENV{PAIMON_HOME}) - set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "Selected Paimon install prefix" FORCE) -else() - set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "Selected Paimon install prefix" FORCE) -endif() - -if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h") - message(FATAL_ERROR "Selected Arrow installation is incomplete: ${ARROW_HOME}") -endif() -message(STATUS "ARROW_HOME is ${ARROW_HOME}") -if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon") - message(FATAL_ERROR "Selected Paimon installation is incomplete: ${PAIMON_HOME}") -endif() -if (ENABLE_PAIMON_CPP) - message(STATUS "PAIMON_HOME is ${PAIMON_HOME}") +if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME) + set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE) endif() set(CMAKE_SKIP_RPATH TRUE) @@ -618,12 +596,6 @@ include_directories( ${SRC_DIR}/ ) -include_directories( - BEFORE SYSTEM - ${ARROW_HOME}/include - ${PAIMON_HOME}/include -) - include_directories( SYSTEM ${COMMON_SRC_DIR} @@ -769,10 +741,10 @@ endif() set(PAIMON_FACTORY_REGISTRY_LIBS) if (ENABLE_PAIMON_CPP) - # The Arrow selected for this BE build includes COMPUTE/DATASET/ACERO/FILESYSTEM, - # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY - # via thirdparty.cmake. Paimon uses this same selected Arrow (no paimon_deps); - # the legacy Arrow 17 stack remains installed only for pre-upgrade branch-4.1 consumers. + # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM, + # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in COMMON_THIRDPARTY via + # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps). + # No dual-stack selection needed — single Arrow for everything. # paimon_parquet_file_format depends on Arrow Dataset symbols. # Force-link it with --whole-archive so its factory registration runs. diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 1003dda9289..f3bd9867ee0 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -25,11 +25,10 @@ set(COMMON_THIRDPARTY) # if arg exist noadd, not append to COMMON_THIRDPARTY variable # if arg exist libname, use libname to find library # if arg exist wholelibpath, use wholelibpath to find library -# if arg exist root, resolve the library below that install prefix function(add_thirdparty) cmake_parse_arguments(DORIS_THIRDPARTY "NOTADD;LIB64" - "LIBNAME;WHOLELIBPATH;ROOT" + "LIBNAME;WHOLELIBPATH" "" ${ARGN}) @@ -40,20 +39,14 @@ function(add_thirdparty) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME} PARENT_SCOPE) endif() - if (DORIS_THIRDPARTY_ROOT) - set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT}) - else() - set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR}) - endif() - if (DORIS_THIRDPARTY_LIB64) - set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a) + set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a) elseif (DORIS_THIRDPARTY_LIBNAME) - set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME}) + set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME}) elseif (DORIS_THIRDPARTY_WHOLELIBPATH) set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH}) else() - set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a) + set(DORIS_THIRDPARTY_LIBPATH ${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a) endif() set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH}) endfunction() @@ -107,18 +100,18 @@ add_thirdparty(cares LIB64) add_thirdparty(address_sorting LIB64) add_thirdparty(z LIB64) -add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME}) -add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME}) -add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(brotlicommon LIB64) +add_thirdparty(brotlidec LIB64) +add_thirdparty(brotlienc LIB64) add_thirdparty(zstd LIB64) -add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME}) -add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME}) -add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME}) -add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME}) -add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME}) -add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(arrow LIB64) +add_thirdparty(arrow_compute LIB64) +add_thirdparty(arrow_flight LIB64) +add_thirdparty(arrow_flight_sql LIB64) +add_thirdparty(arrow_dataset LIB64) +add_thirdparty(arrow_acero LIB64) add_thirdparty(adbc_driver_manager LIB64) -add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME}) +add_thirdparty(parquet LIB64) # liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it # so the final linker resolves C math symbols from the system library first. add_thirdparty(lance_c LIB64 NOTADD) @@ -198,16 +191,16 @@ add_thirdparty(icudata LIB64) add_thirdparty(pugixml LIB64) if (ENABLE_PAIMON_CPP) - add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME}) - - add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME}) - add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME}) + add_thirdparty(paimon LIB64) + add_thirdparty(paimon_parquet_file_format LIB64) + add_thirdparty(paimon_orc_file_format LIB64) + add_thirdparty(paimon_blob_file_format LIB64) + add_thirdparty(paimon_local_file_system LIB64) + add_thirdparty(paimon_file_index LIB64) + add_thirdparty(paimon_global_index LIB64) + + add_thirdparty(roaring_bitmap_paimon LIB64) + add_thirdparty(xxhash_paimon LIB64) + add_thirdparty(fmt_paimon LIB64) + add_thirdparty(tbb_paimon LIB64) endif() diff --git a/build.sh b/build.sh index 136715539f1..b212706f54f 100755 --- a/build.sh +++ b/build.sh @@ -472,18 +472,6 @@ if [[ "${HELP}" -eq 1 ]]; then usage fi -# Normalize compile-bench before dependency selection. The mode is a BE build, -# even when --compile-bench is the only command-line target. -if [[ "${COMPILE_BENCH}" -eq 1 ]]; then - BUILD_BE=1 - BUILD_FE=0 - BUILD_CLOUD=0 - BUILD_HIVE_UDF=0 - BUILD_BE_JAVA_EXTENSIONS=0 - BUILD_BE_CDC_CLIENT=0 - OUTPUT_BE_BINARY=0 -fi - if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && ${BUILD_CLOUD} -eq 0 ]]; then clean_gensrc clean_be @@ -503,27 +491,12 @@ fi # shellcheck source=thirdparty/arrow-paimon-vars.sh . "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh" NEED_ARROW_PAIMON_THIRDPARTY=false -if [[ "${BUILD_BE}" -eq 1 || "${BUILD_META_TOOL}" == "ON" || - "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" || +if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 || + "${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" || "${BUILD_INDEX_TOOL}" == "ON" ]]; then NEED_ARROW_PAIMON_THIRDPARTY=true fi -if [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then - DEFAULT_ARROW_PAIMON_HOME="${DORIS_THIRDPARTY}/installed/${ARROW_INSTALL_SUBDIR}" - SELECTED_ARROW_HOME="${ARROW_HOME:-${DEFAULT_ARROW_PAIMON_HOME}}" - SELECTED_PAIMON_HOME="${PAIMON_HOME:-${SELECTED_ARROW_HOME}}" - if [[ "${SELECTED_ARROW_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" || - "${SELECTED_PAIMON_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ]]; then - echo "build.sh only supports the Arrow/Paimon stack selected from DORIS_THIRDPARTY." >&2 - echo "Expected ARROW_HOME=${DEFAULT_ARROW_PAIMON_HOME} and PAIMON_HOME=${DEFAULT_ARROW_PAIMON_HOME}." >&2 - echo "Unset ARROW_HOME and PAIMON_HOME, or point DORIS_THIRDPARTY at the matching thirdparty tree." >&2 - exit 1 - fi - export ARROW_HOME="${DEFAULT_ARROW_PAIMON_HOME}" - export PAIMON_HOME="${DEFAULT_ARROW_PAIMON_HOME}" -fi - rebuild_thirdparty_libraries() { local remove_installed="$1" shift @@ -553,7 +526,7 @@ rebuild_thirdparty_libraries() { build_args+=(--clean) fi bash "${build_script}" "${build_args[@]}" "$@" - if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then + if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's selected inputs." >&2 exit 1 fi @@ -562,12 +535,10 @@ rebuild_thirdparty_libraries() { if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then echo "Thirdparty libraries need to be build ..." rebuild_thirdparty_libraries true -elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then - select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed" - if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then - echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." - rebuild_thirdparty_libraries false "${ARROW_PAIMON_REBUILD_PACKAGES[@]}" - fi +elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] && + ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then + echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..." + rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}" fi update_submodule() { @@ -757,6 +728,16 @@ for ((i = 0; i < ${#CLOUD_EXTRA_FEATURE_KEYS[@]}; i++)); do done if [[ "${COMPILE_BENCH}" -eq 1 ]]; then + # BE compile benchmark mode: measure a cold, cache-free BE C++ build. + # Everything that is not the BE C++ build would only add noise, so force + # a BE-only build regardless of the other options. + BUILD_BE=1 + BUILD_FE=0 + BUILD_CLOUD=0 + BUILD_HIVE_UDF=0 + BUILD_BE_JAVA_EXTENSIONS=0 + BUILD_BE_CDC_CLIENT=0 + OUTPUT_BE_BINARY=0 # shellcheck source=build-support/compile-bench/bench-lib.sh . "${DORIS_HOME}/build-support/compile-bench/bench-lib.sh" compile_bench_init "${DORIS_HOME}" diff --git a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy index c1f47c4a399..9d27447cb70 100644 --- a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy +++ b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy @@ -227,12 +227,7 @@ suite("test_adbc_type_mapping", "p0,external") { // ---- floating point ---- qt_desc_float """DESC ${catalogName}.${dbName}.t_float""" - // Render the double with the historical 16-digit baseline so DBL_MAX is not reparsed by the - // test client. The bit-for-bit comparison below still validates the DOUBLE value inside Doris. - qt_select_float """ - SELECT id, c_float, format('{:.16g}', c_double) AS c_double - FROM ${catalogName}.${dbName}.t_float ORDER BY id - """ + qt_select_float """SELECT id, c_float, c_double FROM ${catalogName}.${dbName}.t_float ORDER BY id""" // Compared INSIDE Doris rather than through sameAsSource, and not because ADBC needs the // help: the fixture's DBL_MAX cannot make the trip to the test client at all. Doris renders a diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh index bee098c0040..d532a740e63 100644 --- a/thirdparty/arrow-paimon-vars.sh +++ b/thirdparty/arrow-paimon-vars.sh @@ -21,24 +21,12 @@ # Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can # distinguish this stack from unrelated thirdparty changes. -# Arrow 24 is installed in a versioned prefix. The unversioned install prefix is -# deliberately reserved for Arrow 17 so the shared thirdparty package remains -# consumable by branch-4.1 revisions from before the Arrow 24 backport. +# arrow ARROW_VERSION="24.0.0" ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz"; ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz" ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}" ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436" -ARROW_INSTALL_SUBDIR="arrow-${ARROW_VERSION}" - -# Arrow 17 compatibility stack for pre-upgrade branch-4.1 revisions. Keep these -# variables separate from ARROW_* so master can build both versions from one -# source bundle. -ARROW_17_VERSION="17.0.0" -ARROW_17_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_17_VERSION}.tar.gz"; -ARROW_17_NAME="apache-arrow-${ARROW_17_VERSION}.tar.gz" -ARROW_17_SOURCE="arrow-apache-arrow-${ARROW_17_VERSION}" -ARROW_17_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0" # Arrow bundled dependencies BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz"; @@ -51,11 +39,6 @@ XSIMD_NAME="14.0.0.tar.gz" XSIMD_SOURCE=xsimd-14.0.0 XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1" -XSIMD_17_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz"; -XSIMD_17_NAME="13.0.0.tar.gz" -XSIMD_17_SOURCE=xsimd-13.0.0 -XSIMD_17_MD5SUM="c661deb91836e82d3070f81032014fe6" - # paimon-cpp PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz"; PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz" @@ -94,25 +77,10 @@ PAIMON_LEGACY_BUILD_FINGERPRINTS=( dbb6ca6e243cb3aa783b7a8011f97afda9e7ea28 ) -# Both Paimon variants use the same archive, but they need independent source -# trees because only the Arrow 24 tree receives the API compatibility patches. -PAIMON_CPP_17_DOWNLOAD="${PAIMON_CPP_DOWNLOAD}" -PAIMON_CPP_17_NAME="${PAIMON_CPP_NAME}" -PAIMON_CPP_17_ARCHIVE_SOURCE="${PAIMON_CPP_SOURCE}" -PAIMON_CPP_17_SOURCE="${PAIMON_CPP_SOURCE}-arrow-17" -PAIMON_CPP_17_MD5SUM="${PAIMON_CPP_MD5SUM}" - # Arrow consumes xsimd and Brotli as bundled source archives, but neither is a # build target in the focused Arrow/Paimon recovery path. -ARROW_PAIMON_17_BUILD_PACKAGES=(arrow_17 paimon_cpp_17) ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp) -ARROW_PAIMON_SHARED_BUILD_PACKAGES=( - "${ARROW_PAIMON_17_BUILD_PACKAGES[@]}" - "${ARROW_PAIMON_BUILD_PACKAGES[@]}" -) -ARROW_PAIMON_REBUILD_PACKAGES=() ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli) -ARROW_17_BUNDLED_SOURCE_PACKAGES=(xsimd_17 brotli) ARROW_PAIMON_DOWNLOAD_PACKAGES=() prepare_arrow_paimon_download_packages() { @@ -121,25 +89,18 @@ prepare_arrow_paimon_download_packages() { local package local source_package local arrow_requested=false - local arrow_17_requested=false local source_requested for package in "$@"; do if [[ "${package}" == "arrow" ]]; then arrow_requested=true - elif [[ "${package}" == "arrow_17" ]]; then - arrow_17_requested=true + break fi done - - local bundled_source_packages=() - if [[ "${arrow_requested}" == "true" ]]; then - bundled_source_packages+=("${ARROW_BUNDLED_SOURCE_PACKAGES[@]}") - fi - if [[ "${arrow_17_requested}" == "true" ]]; then - bundled_source_packages+=("${ARROW_17_BUNDLED_SOURCE_PACKAGES[@]}") + if [[ "${arrow_requested}" != "true" ]]; then + return fi - for source_package in "${bundled_source_packages[@]}"; do + for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do source_requested=false for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do if [[ "${package}" == "${source_package}" ]]; then @@ -153,10 +114,6 @@ prepare_arrow_paimon_download_packages() { done } -arrow_install_dir() { - printf '%s/%s\n' "$1" "${ARROW_INSTALL_SUBDIR}" -} - # Print stable path-and-content records for fingerprint inputs. Including the path # makes patch selection and ordering part of the contract, not only file contents. arrow_paimon_fingerprint_files() { @@ -215,9 +172,7 @@ paimon_build_fingerprint() { printf 'PAIMON_CPP_MD5SUM=%s\n' "${PAIMON_CPP_MD5SUM}" arrow_paimon_fingerprint_files \ paimon-cpp-cache.cmake \ - patches/paimon-cpp-buildutils-static-deps.patch \ - patches/paimon-cpp-arrow-24-compatibility.patch \ - patches/paimon-cpp-arrow-24-compute.patch + patches/paimon-cpp-*.patch } | git hash-object --stdin ) } @@ -252,57 +207,6 @@ arrow_paimon_fingerprint_matches() { return 1 } -arrow_17_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - printf 'schema=%s\n' "${ARROW_BUILD_SCHEMA_VERSION}" - printf 'ARROW_17_VERSION=%s\n' "${ARROW_17_VERSION}" - printf 'ARROW_17_NAME=%s\n' "${ARROW_17_NAME}" - printf 'ARROW_17_SOURCE=%s\n' "${ARROW_17_SOURCE}" - printf 'ARROW_17_MD5SUM=%s\n' "${ARROW_17_MD5SUM}" - printf 'BROTLI_NAME=%s\n' "${BROTLI_NAME}" - printf 'BROTLI_SOURCE=%s\n' "${BROTLI_SOURCE}" - printf 'BROTLI_MD5SUM=%s\n' "${BROTLI_MD5SUM}" - printf 'XSIMD_17_NAME=%s\n' "${XSIMD_17_NAME}" - printf 'XSIMD_17_SOURCE=%s\n' "${XSIMD_17_SOURCE}" - printf 'XSIMD_17_MD5SUM=%s\n' "${XSIMD_17_MD5SUM}" - arrow_paimon_fingerprint_files \ - patches/apache-arrow-"${ARROW_17_VERSION}"-*.patch - } | git hash-object --stdin - ) -} - -paimon_17_build_fingerprint() { - local vars_dir - vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - ( - set -o pipefail - cd "${vars_dir}" || return 1 - { - arrow_17_build_fingerprint - printf 'schema=%s\n' "${PAIMON_BUILD_SCHEMA_VERSION}" - printf 'PAIMON_CPP_17_NAME=%s\n' "${PAIMON_CPP_17_NAME}" - printf 'PAIMON_CPP_17_ARCHIVE_SOURCE=%s\n' "${PAIMON_CPP_17_ARCHIVE_SOURCE}" - printf 'PAIMON_CPP_17_SOURCE=%s\n' "${PAIMON_CPP_17_SOURCE}" - printf 'PAIMON_CPP_17_MD5SUM=%s\n' "${PAIMON_CPP_17_MD5SUM}" - arrow_paimon_fingerprint_files \ - paimon-cpp-cache.cmake \ - patches/paimon-cpp-buildutils-static-deps.patch - } | git hash-object --stdin - ) -} - -arrow_paimon_17_build_fingerprint() { - { - arrow_17_build_fingerprint - paimon_17_build_fingerprint - } | git hash-object --stdin -} - ARROW_REQUIRED_LIBRARIES=( libbrotlicommon.a libbrotlidec.a @@ -317,19 +221,6 @@ ARROW_REQUIRED_LIBRARIES=( libparquet.a ) -ARROW_17_REQUIRED_LIBRARIES=( - libbrotlicommon.a - libbrotlidec.a - libbrotlienc.a - libarrow.a - libarrow_flight.a - libarrow_flight_sql.a - libarrow_dataset.a - libarrow_acero.a - libarrow_bundled_dependencies.a - libparquet.a -) - PAIMON_REQUIRED_LIBRARIES=( libpaimon.a libpaimon_parquet_file_format.a @@ -344,64 +235,13 @@ PAIMON_REQUIRED_LIBRARIES=( libtbb_paimon.a ) -# Remove only artifacts owned by the selected Arrow/Paimon stack before an -# install. This matters for the legacy prefix: installing Arrow 17 over an -# existing Arrow 24 prefix must not leave Arrow 24-only headers or libraries -# behind and turn it into a mixed, internally inconsistent SDK. -clean_arrow_artifacts_in() { - local install_dir="$1" - : "${install_dir:?Arrow install directory must be set}" - - rm -rf -- \ - "${install_dir}/include/arrow" \ - "${install_dir}/include/parquet" \ - "${install_dir}/share/arrow" \ - "${install_dir}/share/doc/arrow" - - ( - shopt -s nullglob - local generated_artifacts=( - "${install_dir}/lib64"/libarrow* - "${install_dir}/lib64"/libparquet* - "${install_dir}/lib64/cmake"/Arrow* - "${install_dir}/lib64/cmake"/Parquet - "${install_dir}/lib64/pkgconfig"/arrow*.pc - "${install_dir}/lib64/pkgconfig"/parquet.pc - ) - rm -rf -- "${generated_artifacts[@]}" - ) -} - -clean_paimon_artifacts_in() { - local install_dir="$1" - : "${install_dir:?Paimon install directory must be set}" - - rm -rf -- \ - "${install_dir}/include/paimon" \ - "${install_dir}/lib64/cmake/Paimon" \ - "${install_dir}/paimon-cpp" - - ( - shopt -s nullglob - local generated_artifacts=( - "${install_dir}/lib64"/libpaimon* - "${install_dir}/lib64"/libroaring_bitmap_paimon.* - "${install_dir}/lib64"/libxxhash_paimon.* - "${install_dir}/lib64"/libfmt_paimon.* - "${install_dir}/lib64"/libtbb_paimon.* - ) - rm -rf -- "${generated_artifacts[@]}" - ) -} - ARROW_PAIMON_REQUIRED_LIBRARIES=( "${ARROW_REQUIRED_LIBRARIES[@]}" "${PAIMON_REQUIRED_LIBRARIES[@]}" ) arrow_artifacts_valid() { - local install_dir - install_dir="$(arrow_install_dir "$1")" + local install_dir="$1" local installed_arrow_version local library @@ -428,35 +268,7 @@ arrow_artifacts_valid() { return 0 } -arrow_17_artifacts_valid() { - local install_dir="$1" - local installed_arrow_version - local library - - if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then - echo "Missing installed Arrow 17 version header" >&2 - return 1 - fi - installed_arrow_version="$( - awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" { - gsub(/"/, "", $3); print $3; exit - }' "${install_dir}/include/arrow/util/config.h" - )" - if [[ "${installed_arrow_version}" != "${ARROW_17_VERSION}" ]]; then - echo "Installed legacy Arrow version ${installed_arrow_version} does not match ${ARROW_17_VERSION}" >&2 - return 1 - fi - - for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}"; do - if [[ ! -f "${install_dir}/lib64/${library}" ]]; then - echo "Missing Arrow 17 library: ${library}" >&2 - return 1 - fi - done - return 0 -} - -paimon_artifacts_valid_in() { +paimon_artifacts_valid() { local install_dir="$1" local library @@ -469,13 +281,8 @@ paimon_artifacts_valid_in() { return 0 } -paimon_artifacts_valid() { - paimon_artifacts_valid_in "$(arrow_install_dir "$1")" -} - arrow_prebuilt_valid() { - local install_dir - install_dir="$(arrow_install_dir "$1")" + local install_dir="$1" local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt" local expected_fingerprint local installed_fingerprint @@ -492,12 +299,11 @@ arrow_prebuilt_valid() { echo "Arrow build fingerprint does not match selected inputs" >&2 return 1 fi - arrow_artifacts_valid "$1" + arrow_artifacts_valid "${install_dir}" } paimon_prebuilt_valid() { - local install_dir - install_dir="$(arrow_install_dir "$1")" + local install_dir="$1" local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt" local expected_fingerprint local installed_fingerprint @@ -514,7 +320,7 @@ paimon_prebuilt_valid() { echo "Paimon build fingerprint does not match selected inputs" >&2 return 1 fi - paimon_artifacts_valid "$1" + paimon_artifacts_valid "${install_dir}" } arrow_paimon_prebuilt_valid() { @@ -523,32 +329,26 @@ arrow_paimon_prebuilt_valid() { } invalidate_arrow_prebuilt_marker() { - local install_dir - install_dir="$(arrow_install_dir "$1")" - mkdir -p "${install_dir}" + local install_dir="$1" rm -f "${install_dir}/arrow-build-fingerprint.txt" \ "${install_dir}/arrow-paimon-build-fingerprint.txt" } publish_arrow_prebuilt_marker() { - local install_dir - install_dir="$(arrow_install_dir "$1")" - arrow_artifacts_valid "$1" + local install_dir="$1" + arrow_artifacts_valid "${install_dir}" arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt" } invalidate_paimon_prebuilt_marker() { - local install_dir - install_dir="$(arrow_install_dir "$1")" - mkdir -p "${install_dir}" + local install_dir="$1" rm -f "${install_dir}/paimon-build-fingerprint.txt" \ "${install_dir}/arrow-paimon-build-fingerprint.txt" } publish_paimon_prebuilt_marker() { - local install_dir - install_dir="$(arrow_install_dir "$1")" - paimon_artifacts_valid "$1" + local install_dir="$1" + paimon_artifacts_valid "${install_dir}" paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt" } @@ -559,108 +359,3 @@ require_arrow_prebuilt_for_paimon() { return 1 fi } - -invalidate_arrow_17_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/arrow-17-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-17-build-fingerprint.txt" -} - -publish_arrow_17_prebuilt_marker() { - local install_dir="$1" - arrow_17_artifacts_valid "${install_dir}" - arrow_17_build_fingerprint >"${install_dir}/arrow-17-build-fingerprint.txt" -} - -arrow_17_prebuilt_valid() { - local install_dir="$1" - local fingerprint_mark="${install_dir}/arrow-17-build-fingerprint.txt" - local expected_fingerprint - - if [[ ! -f "${fingerprint_mark}" ]]; then - echo "Missing Arrow 17 build fingerprint: ${fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(arrow_17_build_fingerprint)" - if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then - echo "Arrow 17 build fingerprint does not match selected inputs" >&2 - return 1 - fi - arrow_17_artifacts_valid "${install_dir}" -} - -require_arrow_17_prebuilt_for_paimon() { - local install_dir="$1" - if ! arrow_17_prebuilt_valid "${install_dir}"; then - echo "Paimon for pre-upgrade branch-4.1 requires Arrow 17 to be built first" >&2 - return 1 - fi -} - -invalidate_paimon_17_prebuilt_marker() { - local install_dir="$1" - rm -f "${install_dir}/paimon-arrow-17-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-17-build-fingerprint.txt" -} - -# A legacy-prefix Arrow downgrade must first remove Paimon built against the -# previous Arrow ABI. If the Arrow build is interrupted, the incomplete prefix -# then fails at link time instead of exposing a mixed Arrow/Paimon SDK. -prepare_arrow_17_install_prefix() { - local install_dir="$1" - - rm -f "${install_dir}/arrow-build-fingerprint.txt" \ - "${install_dir}/paimon-build-fingerprint.txt" \ - "${install_dir}/arrow-paimon-build-fingerprint.txt" - invalidate_paimon_17_prebuilt_marker "${install_dir}" - clean_paimon_artifacts_in "${install_dir}" - invalidate_arrow_17_prebuilt_marker "${install_dir}" - clean_arrow_artifacts_in "${install_dir}" -} - -publish_paimon_17_prebuilt_marker() { - local install_dir="$1" - paimon_artifacts_valid_in "${install_dir}" - paimon_17_build_fingerprint >"${install_dir}/paimon-arrow-17-build-fingerprint.txt" -} - -paimon_17_prebuilt_valid() { - local install_dir="$1" - local fingerprint_mark="${install_dir}/paimon-arrow-17-build-fingerprint.txt" - local expected_fingerprint - - if [[ ! -f "${fingerprint_mark}" ]]; then - echo "Missing Paimon Arrow 17 build fingerprint: ${fingerprint_mark}" >&2 - return 1 - fi - expected_fingerprint="$(paimon_17_build_fingerprint)" - if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then - echo "Paimon Arrow 17 build fingerprint does not match selected inputs" >&2 - return 1 - fi - paimon_artifacts_valid_in "${install_dir}" -} - -arrow_paimon_17_prebuilt_valid() { - local install_dir="$1" - arrow_17_prebuilt_valid "${install_dir}" && - paimon_17_prebuilt_valid "${install_dir}" -} - -shared_arrow_paimon_prebuilt_valid() { - local install_dir="$1" - arrow_paimon_17_prebuilt_valid "${install_dir}" && - arrow_paimon_prebuilt_valid "${install_dir}" -} - -select_arrow_paimon_rebuild_packages() { - local install_dir="$1" - ARROW_PAIMON_REBUILD_PACKAGES=() - - if ! arrow_paimon_17_prebuilt_valid "${install_dir}"; then - ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_17_BUILD_PACKAGES[@]}") - fi - if ! arrow_paimon_prebuilt_valid "${install_dir}"; then - ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_BUILD_PACKAGES[@]}") - fi -} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 6597f91ed1e..df4bfe6d1fa 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -354,26 +354,20 @@ else echo "Do not strip thirdparty libraries" fi -strip_lib_at() { - local install_dir="$1" - local library="$2" +strip_lib() { if [[ "${STRIP_TP_LIB}" = "ON" ]]; then - if [[ -z "${library}" ]]; then + if [[ -z $1 ]]; then echo "Must specify the library to be stripped." exit 1 fi - if [[ ! -f "${install_dir}/lib/${library}" ]]; then - echo "Library to be stripped (${install_dir}/lib/${library}) does not exist." + if [[ ! -f "${TP_LIB_DIR}/$1" ]]; then + echo "Library to be stripped (${TP_LIB_DIR}/$1) does not exist." exit 1 fi - strip --strip-debug --strip-unneeded "${install_dir}/lib/${library}" + strip --strip-debug --strip-unneeded "${TP_LIB_DIR}/$1" fi } -strip_lib() { - strip_lib_at "${TP_INSTALL_DIR}" "$1" -} - #libbacktrace build_libbacktrace() { check_if_source_exist "${LIBBACKTRACE_SOURCE}" @@ -1109,19 +1103,11 @@ build_grpc() { # sed -i 's/find_dependency/find_package/g' "${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake } -# Arrow 17 is installed in the legacy unversioned prefix for pre-upgrade -# branch-4.1 revisions, while Arrow 24 is installed in a versioned prefix -# selected by master. -build_arrow_stack() { - local arrow_source="$1" - local xsimd_archive="$2" - local install_dir="$3" - local has_separate_compute_archive="$4" - - check_if_source_exist "${arrow_source}" - mkdir -p "${install_dir}/lib64" - ln -sfn lib64 "${install_dir}/lib" - cd "${TP_SOURCE_DIR}/${arrow_source}/cpp" +# arrow +build_arrow() { + check_if_source_exist "${ARROW_SOURCE}" + invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" + cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp" mkdir -p release cd release @@ -1134,7 +1120,7 @@ build_arrow_stack() { export ARROW_Thrift_URL="${TP_SOURCE_DIR}/${THRIFT_NAME}" export ARROW_SNAPPY_URL="${TP_SOURCE_DIR}/${SNAPPY_NAME}" export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}" - export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${xsimd_archive}" + export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}" export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}" export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}" export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}" @@ -1156,7 +1142,7 @@ build_arrow_stack() { -DARROW_FILESYSTEM=ON \ -DARROW_DATASET=ON \ -DARROW_ACERO=ON \ - -DCMAKE_INSTALL_PREFIX="${install_dir}" \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DCMAKE_INSTALL_LIBDIR=lib64 \ -DARROW_BOOST_USE_SHARED=OFF \ -DARROW_WITH_GRPC=ON \ @@ -1199,30 +1185,15 @@ build_arrow_stack() { "${BUILD_SYSTEM}" install #copy dep libs - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a "${install_dir}/lib64/libbrotlienc.a" - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${install_dir}/lib64/libbrotlidec.a" - cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${install_dir}/lib64/libbrotlicommon.a" - strip_lib_at "${install_dir}" libarrow.a - if [[ "${has_separate_compute_archive}" == "true" ]]; then - strip_lib_at "${install_dir}" libarrow_compute.a - fi - strip_lib_at "${install_dir}" libparquet.a - strip_lib_at "${install_dir}" libarrow_dataset.a - strip_lib_at "${install_dir}" libarrow_acero.a -} + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a "${TP_INSTALL_DIR}/lib64/libbrotlienc.a" + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a "${TP_INSTALL_DIR}/lib64/libbrotlidec.a" + cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a "${TP_INSTALL_DIR}/lib64/libbrotlicommon.a" + strip_lib libarrow.a + strip_lib libarrow_compute.a + strip_lib libparquet.a + strip_lib libarrow_dataset.a + strip_lib libarrow_acero.a -build_arrow_17() { - prepare_arrow_17_install_prefix "${TP_INSTALL_DIR}" - build_arrow_stack "${ARROW_17_SOURCE}" "${XSIMD_17_NAME}" "${TP_INSTALL_DIR}" false - publish_arrow_17_prebuilt_marker "${TP_INSTALL_DIR}" -} - -build_arrow() { - local install_dir - install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")" - invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}" - clean_arrow_artifacts_in "${install_dir}" - build_arrow_stack "${ARROW_SOURCE}" "${XSIMD_NAME}" "${install_dir}" true publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}" } @@ -2318,18 +2289,12 @@ build_pugixml() { cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/" } -# Build each Paimon variant against the matching Arrow prefix and install it -# beside that Arrow version. Arrow types cross Paimon's public C++ boundary, so -# mixing the two versions is not ABI-safe. -build_paimon_cpp_stack() { - local paimon_source="$1" - local arrow_install_dir="$2" - local install_dir="$3" - - check_if_source_exist "${paimon_source}" - mkdir -p "${install_dir}/lib64" - ln -sfn lib64 "${install_dir}/lib" - cd "${TP_SOURCE_DIR}/${paimon_source}" +# paimon-cpp +build_paimon_cpp() { + check_if_source_exist "${PAIMON_CPP_SOURCE}" + require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" + invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" + cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}" rm -rf "${BUILD_DIR}" mkdir -p "${BUILD_DIR}" @@ -2341,13 +2306,12 @@ build_paimon_cpp_stack() { paimon_linker_flags="${paimon_linker_flags} -lunwind" fi - PAIMON_ARROW_INSTALL_DIR="${arrow_install_dir}" \ CXXFLAGS="-Wno-nontrivial-memcall" \ "${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \ -G "${GENERATOR}" \ -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ -DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \ - -DCMAKE_INSTALL_PREFIX="${install_dir}" \ + -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \ -DPAIMON_BUILD_SHARED=OFF \ -DPAIMON_BUILD_STATIC=ON \ -DPAIMON_BUILD_TESTS=OFF \ @@ -2371,7 +2335,7 @@ build_paimon_cpp_stack() { # reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps # directory is not needed. When building its own Arrow (legacy), copy # arrow artefacts into an isolated directory to avoid clashing with Doris. - local paimon_deps_dir="${install_dir}/paimon-cpp/lib64/paimon_deps" + local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps" if [ -d "arrow_ep-install/lib" ]; then mkdir -p "${paimon_deps_dir}" for paimon_arrow_dep in \ @@ -2391,42 +2355,25 @@ build_paimon_cpp_stack() { # Install roaring_bitmap, renamed to avoid conflict with Doris's croaringbitmap if [ -f "release/libroaring_bitmap.a" ]; then - cp -v "release/libroaring_bitmap.a" "${install_dir}/lib64/libroaring_bitmap_paimon.a" + cp -v "release/libroaring_bitmap.a" "${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a" fi # Install xxhash, renamed to avoid conflict with Doris's xxhash if [ -f "release/libxxhash.a" ]; then - cp -v "release/libxxhash.a" "${install_dir}/lib64/libxxhash_paimon.a" + cp -v "release/libxxhash.a" "${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a" fi # Install fmt v11 (from fmt_ep-install directory, renamed to avoid conflict with Doris's fmt v7) if [ -f "fmt_ep-install/lib/libfmt.a" ]; then - cp -v "fmt_ep-install/lib/libfmt.a" "${install_dir}/lib64/libfmt_paimon.a" + cp -v "fmt_ep-install/lib/libfmt.a" "${TP_INSTALL_DIR}/lib64/libfmt_paimon.a" fi # Install tbb (from tbb_ep-install directory, renamed to avoid conflict with Doris's tbb) if [ -f "tbb_ep-install/lib/libtbb.a" ]; then - cp -v "tbb_ep-install/lib/libtbb.a" "${install_dir}/lib64/libtbb_paimon.a" + cp -v "tbb_ep-install/lib/libtbb.a" "${TP_INSTALL_DIR}/lib64/libtbb_paimon.a" fi echo "Paimon-cpp internal dependencies installed successfully" -} - -build_paimon_cpp_17() { - require_arrow_17_prebuilt_for_paimon "${TP_INSTALL_DIR}" - invalidate_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}" - clean_paimon_artifacts_in "${TP_INSTALL_DIR}" - build_paimon_cpp_stack "${PAIMON_CPP_17_SOURCE}" "${TP_INSTALL_DIR}" "${TP_INSTALL_DIR}" - publish_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}" -} - -build_paimon_cpp() { - local install_dir - install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")" - require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}" - invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}" - clean_paimon_artifacts_in "${install_dir}" - build_paimon_cpp_stack "${PAIMON_CPP_SOURCE}" "${install_dir}" "${install_dir}" publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}" } @@ -2531,7 +2478,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then orc cares grpc # after cares, protobuf - arrow_17 arrow arrow_adbc lance_c @@ -2569,7 +2515,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then brotli icu pugixml - paimon_cpp_17 paimon_cpp ) if [[ "$(uname -s)" == 'Darwin' ]]; then @@ -2625,7 +2570,6 @@ cleanup_package_source() { cyrus_sasl) src_var="CYRUS_SASL_SOURCE" ;; librdkafka) src_var="LIBRDKAFKA_SOURCE" ;; flatbuffers) src_var="FLATBUFFERS_SOURCE" ;; - arrow_17) src_var="ARROW_17_SOURCE" ;; arrow) src_var="ARROW_SOURCE" ;; arrow_adbc) # arrow_adbc also unpacks the prebuilt flightsql driver, clean both @@ -2678,7 +2622,6 @@ cleanup_package_source() { jindofs) src_var="JINDOFS_SOURCE" ;; juicefs) src_var="JUICEFS_SOURCE" ;; pugixml) src_var="PUGIXML_SOURCE" ;; - paimon_cpp_17) src_var="PAIMON_CPP_17_SOURCE" ;; paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;; lance_c) src_var="LANCE_C_SOURCE" ;; aws_sdk) src_var="AWS_SDK_SOURCE" ;; diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 5af7fdcc3e5..8d65b8ce275 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -271,24 +271,13 @@ for TP_ARCH in "${TP_ARCHIVES[@]}"; do fi NAME="${TP_ARCH}_NAME" SOURCE="${TP_ARCH}_SOURCE" - ARCHIVE_SOURCE_VAR="${TP_ARCH}_ARCHIVE_SOURCE" - ARCHIVE_SOURCE="${!ARCHIVE_SOURCE_VAR}" if [[ -z "${!SOURCE}" ]]; then continue fi if [[ ! -d "${TP_SOURCE_DIR}/${!SOURCE}" ]]; then - if [[ -n "${ARCHIVE_SOURCE}" && "${ARCHIVE_SOURCE}" != "${!SOURCE}" ]]; then - alias_unpack_dir="$(mktemp -d "${TP_SOURCE_DIR}/.unpack-${TP_ARCH}.XXXXXX")" - if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${alias_unpack_dir}"; then - echo "Failed to untar ${!NAME} for ${!SOURCE}" - rm -rf "${alias_unpack_dir}" - exit 1 - fi - mv "${alias_unpack_dir}/${ARCHIVE_SOURCE}" "${TP_SOURCE_DIR}/${!SOURCE}" - rm -rf "${alias_unpack_dir}" - elif [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then + if [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then echo "${TP_SOURCE_DIR}/${!NAME}" echo "${TP_SOURCE_DIR}/${!SOURCE}" if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C "${TP_SOURCE_DIR}/"; then @@ -380,31 +369,17 @@ echo "===== Patching thirdparty archives..." PATCHED_MARK="patched_mark" ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint" ARROW_PAIMON_BUILD_FINGERPRINT="" -ARROW_PAIMON_17_BUILD_FINGERPRINT="" if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " || " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)" fi -if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " || - " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then - ARROW_PAIMON_17_BUILD_FINGERPRINT="$(arrow_paimon_17_build_fingerprint)" -fi reset_arrow_paimon_source() { local archive_name="$1" local source_name="$2" - local archive_source="${3:-${source_name}}" echo "Resetting ${source_name} because its patch state is incomplete or stale" rm -rf "${TP_SOURCE_DIR:?}/${source_name}" - if [[ "${archive_source}" == "${source_name}" ]]; then - "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" - else - local alias_unpack_dir - alias_unpack_dir="$(mktemp -d "${TP_SOURCE_DIR}/.reset-${source_name}.XXXXXX")" - "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${alias_unpack_dir}" - mv "${alias_unpack_dir}/${archive_source}" "${TP_SOURCE_DIR}/${source_name}" - rm -rf "${alias_unpack_dir}" - fi + "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/" } # glog patch @@ -500,26 +475,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ROCKSDB " ]]; then echo "Finished patching ${ROCKSDB_SOURCE}" fi -# Keep the Arrow 17 source used by pre-upgrade branch-4.1 revisions independently -# patched from the Arrow 24 source selected by master. -if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " ]]; then - arrow_17_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${PATCHED_MARK}" && - -f "${arrow_17_fingerprint_mark}" ]] || - [[ "$(<"${arrow_17_fingerprint_mark}")" != "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${ARROW_17_NAME}" "${ARROW_17_SOURCE}" - cd "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}" - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch" - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-force-write-int96-timestamps.patch" - patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-lzo.patch" - touch "${PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - - fi - echo "Finished patching ${ARROW_17_SOURCE}" -fi - # arrow patch is used to get the raw orc reader for filter prune. if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then @@ -839,24 +794,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then echo "Finished patching ${AZURE_SOURCE}" fi -# Keep the Arrow 17 Paimon source free of the Arrow 24 API and Compute patches. -if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then - paimon_17_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${PATCHED_MARK}" && - -f "${paimon_17_fingerprint_mark}" ]] || - [[ "$(<"${paimon_17_fingerprint_mark}")" != "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then - reset_arrow_paimon_source "${PAIMON_CPP_17_NAME}" "${PAIMON_CPP_17_SOURCE}" \ - "${PAIMON_CPP_17_ARCHIVE_SOURCE}" - cd "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}" - patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch" - touch "${PATCHED_MARK}" - printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \ - >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}" - cd - - fi - echo "Finished patching ${PAIMON_CPP_17_SOURCE}" -fi - # patch paimon-cpp if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24" diff --git a/thirdparty/paimon-cpp-cache.cmake b/thirdparty/paimon-cpp-cache.cmake index a12540e5d3e..6ab6049c59b 100644 --- a/thirdparty/paimon-cpp-cache.cmake +++ b/thirdparty/paimon-cpp-cache.cmake @@ -21,17 +21,12 @@ # Get the Doris thirdparty installation directory from environment set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty install directory") -set(DORIS_ARROW_DIR "$ENV{PAIMON_ARROW_INSTALL_DIR}" CACHE PATH "Selected Doris Arrow install directory") if(NOT DORIS_THIRDPARTY_DIR) message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set") endif() -if(NOT DORIS_ARROW_DIR) - message(FATAL_ERROR "PAIMON_ARROW_INSTALL_DIR environment variable must be set") -endif() message(STATUS "Using Doris thirdparty libraries from: ${DORIS_THIRDPARTY_DIR}") -message(STATUS "Using selected Arrow libraries from: ${DORIS_ARROW_DIR}") # Set CMAKE_PREFIX_PATH to help find_package locate our libraries set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE STRING "Search path for find_package") @@ -69,21 +64,21 @@ set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4 include directory") # ============================================================================ # ============================================================================ -# Arrow - Reuse the version selected by the caller. Both installed Arrow stacks -# include DATASET/ACERO/FILESYSTEM; Arrow 24 also has a separate Compute archive. +# Arrow - Reuse from Doris (Doris Arrow now includes COMPUTE/DATASET/ACERO/FILESYSTEM) +# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp +# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...). # ============================================================================ set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris instead of building from source") -set(DORIS_ARROW_LIB64_DIR "${DORIS_ARROW_DIR}/lib64" CACHE PATH "Selected Arrow lib64 directory") -set(DORIS_ARROW_INCLUDE_DIR "${DORIS_ARROW_DIR}/include" CACHE PATH "Selected Arrow include directory") +set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64 directory") -set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_ARROW_INCLUDE_DIR}" CACHE PATH "Arrow include directory") -set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") -set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") -set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") -set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") -set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_ARROW_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") -set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") +set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow include directory") +set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH "Arrow core library") +set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute library") +set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset library") +set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero library") +set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE FILEPATH "Parquet library") +set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB "${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow bundled dependencies library") # Protobuf, Thrift - still built separately by paimon-cpp diff --git a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch deleted file mode 100644 index 5a754247566..00000000000 --- a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch +++ /dev/null @@ -1,98 +0,0 @@ -diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc ---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:23:23.651831424 +0800 -+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc 2026-03-27 01:28:36.855281965 +0800 -@@ -178,7 +178,8 @@ - - // The user is explicitly asking for Impala int96 encoding, there is no - // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { -+ if (arrow_properties.force_write_int96_timestamps() || -+ (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO)) { - *physical_type = ParquetType::INT96; - return Status::OK(); - } -diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h ---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h 2026-03-27 01:23:23.643831362 +0800 -+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h 2026-03-27 01:27:47.717897537 +0800 -@@ -980,6 +980,7 @@ - public: - Builder() - : write_timestamps_as_int96_(false), -+ force_write_int96_timestamps_(false), - coerce_timestamps_enabled_(false), - coerce_timestamps_unit_(::arrow::TimeUnit::SECOND), - truncated_timestamps_allowed_(false), -@@ -1005,6 +1006,21 @@ - return this; - } - -+ /// \brief Force writing legacy int96 timestamps. -+ /// -+ /// This bypasses unit-based guards and writes INT96 whenever timestamp -+ /// metadata is resolved. -+ Builder* enable_force_write_int96_timestamps() { -+ force_write_int96_timestamps_ = true; -+ return this; -+ } -+ -+ /// \brief Disable forcing legacy int96 timestamps (default). -+ Builder* disable_force_write_int96_timestamps() { -+ force_write_int96_timestamps_ = false; -+ return this; -+ } -+ - /// \brief Coerce all timestamps to the specified time unit. - /// \param unit time unit to truncate to. - /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to microseconds. -@@ -1085,7 +1101,8 @@ - /// Create the final properties. - std::shared_ptr<ArrowWriterProperties> build() { - return std::shared_ptr<ArrowWriterProperties>(new ArrowWriterProperties( -- write_timestamps_as_int96_, coerce_timestamps_enabled_, coerce_timestamps_unit_, -+ write_timestamps_as_int96_, force_write_int96_timestamps_, -+ coerce_timestamps_enabled_, coerce_timestamps_unit_, - truncated_timestamps_allowed_, store_schema_, compliant_nested_types_, - engine_version_, use_threads_, executor_)); - } -@@ -1093,6 +1110,8 @@ - private: - bool write_timestamps_as_int96_; - -+ bool force_write_int96_timestamps_; -+ - bool coerce_timestamps_enabled_; - ::arrow::TimeUnit::type coerce_timestamps_unit_; - bool truncated_timestamps_allowed_; -@@ -1107,6 +1126,8 @@ - - bool support_deprecated_int96_timestamps() const { return write_timestamps_as_int96_; } - -+ bool force_write_int96_timestamps() const { return force_write_int96_timestamps_; } -+ - bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_; } - ::arrow::TimeUnit::type coerce_timestamps_unit() const { - return coerce_timestamps_unit_; -@@ -1138,6 +1159,7 @@ - - private: - explicit ArrowWriterProperties(bool write_nanos_as_int96, -+ bool force_write_int96_timestamps, - bool coerce_timestamps_enabled, - ::arrow::TimeUnit::type coerce_timestamps_unit, - bool truncated_timestamps_allowed, bool store_schema, -@@ -1145,6 +1167,7 @@ - EngineVersion engine_version, bool use_threads, - ::arrow::internal::Executor* executor) - : write_timestamps_as_int96_(write_nanos_as_int96), -+ force_write_int96_timestamps_(force_write_int96_timestamps), - coerce_timestamps_enabled_(coerce_timestamps_enabled), - coerce_timestamps_unit_(coerce_timestamps_unit), - truncated_timestamps_allowed_(truncated_timestamps_allowed), -@@ -1155,6 +1178,7 @@ - executor_(executor) {} - - const bool write_timestamps_as_int96_; -+ const bool force_write_int96_timestamps_; - const bool coerce_timestamps_enabled_; - const ::arrow::TimeUnit::type coerce_timestamps_unit_; - const bool truncated_timestamps_allowed_; diff --git a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch deleted file mode 100644 index a983818413a..00000000000 --- a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch +++ /dev/null @@ -1,84 +0,0 @@ ---- a/cpp/src/parquet/column_reader.cc -+++ b/cpp/src/parquet/column_reader.cc -@@ -30,0 +31,2 @@ -+ -+#include <lzo/lzo1x.h> -@@ -268,0 +269 @@ -+ compression_codec_(codec), -@@ -279 +282,7 @@ -- decompressor_ = GetCodec(codec); -+ if (compression_codec_ == Compression::LZO) { -+ if (lzo_init() != LZO_E_OK) { -+ throw ParquetException("Failed to initialize LZO codec"); -+ } -+ } else { -+ decompressor_ = GetCodec(codec); -+ } -@@ -315,0 +325 @@ -+ Compression::type compression_codec_; -@@ -585 +595 @@ -- if (decompressor_ == nullptr) { -+ if (decompressor_ == nullptr && compression_codec_ != Compression::LZO) { -@@ -601,0 +612,61 @@ -+ if (compression_codec_ == Compression::LZO) { -+ const uint8_t* input = page_buffer->data() + levels_byte_len; -+ const uint8_t* const input_end = page_buffer->data() + compressed_len; -+ uint8_t* output = decompression_buffer_->mutable_data() + levels_byte_len; -+ uint8_t* const output_end = decompression_buffer_->mutable_data() + uncompressed_len; -+ -+ auto load_big_endian_u32 = [](const uint8_t* data) { -+ return (static_cast<uint32_t>(data[0]) << 24) | -+ (static_cast<uint32_t>(data[1]) << 16) | -+ (static_cast<uint32_t>(data[2]) << 8) | static_cast<uint32_t>(data[3]); -+ }; -+ -+ while (input < input_end) { -+ if (input_end - input < 4) { -+ throw ParquetException("LZO page decompression failed: truncated large block length"); -+ } -+ -+ uint32_t large_block_uncompressed_len = load_big_endian_u32(input); -+ input += 4; -+ if (static_cast<size_t>(output_end - output) < large_block_uncompressed_len) { -+ throw ParquetException("LZO page decompression failed: output buffer too small"); -+ } -+ -+ while (large_block_uncompressed_len > 0) { -+ if (input_end - input < 4) { -+ throw ParquetException("LZO page decompression failed: truncated small block length"); -+ } -+ -+ uint32_t small_block_compressed_len = load_big_endian_u32(input); -+ input += 4; -+ if (static_cast<size_t>(input_end - input) < small_block_compressed_len) { -+ throw ParquetException("LZO page decompression failed: truncated small block data"); -+ } -+ -+ auto small_block_uncompressed_len = -+ static_cast<lzo_uint>(large_block_uncompressed_len); -+ const int result = -+ lzo1x_decompress_safe(input, static_cast<lzo_uint>(small_block_compressed_len), -+ output, &small_block_uncompressed_len, nullptr); -+ if (result != LZO_E_OK) { -+ throw ParquetException("LZO page decompression failed, error: " + -+ std::to_string(result)); -+ } -+ if (small_block_uncompressed_len > large_block_uncompressed_len) { -+ throw ParquetException("LZO page decompression failed: invalid small block size"); -+ } -+ -+ input += small_block_compressed_len; -+ output += small_block_uncompressed_len; -+ large_block_uncompressed_len -= small_block_uncompressed_len; -+ } -+ } -+ if (output != output_end) { -+ throw ParquetException("Page didn't decompress to expected size, expected: " + -+ std::to_string(uncompressed_len - levels_byte_len) + ", but got:" + -+ std::to_string(output - (decompression_buffer_->mutable_data() + -+ levels_byte_len))); -+ } -+ -+ return decompression_buffer_; -+ } -+ diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch deleted file mode 100644 index 4e53117b79b..00000000000 --- a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch +++ /dev/null @@ -1,224 +0,0 @@ -diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc -index ec3890a41f..943f69bb6c 100644 ---- a/cpp/src/parquet/arrow/schema.cc -+++ b/cpp/src/parquet/arrow/schema.cc -@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const ::arrow::TimestampType& type, - - // The user is explicitly asking for Impala int96 encoding, there is no - // logical type. -- if (arrow_properties.support_deprecated_int96_timestamps()) { -+ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit == ::arrow::TimeUnit::NANO) { - *physical_type = ParquetType::INT96; - return Status::OK(); - } - -diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc -index 285e2a5973..aa6f92f077 100644 ---- a/cpp/src/parquet/arrow/reader.cc -+++ b/cpp/src/parquet/arrow/reader.cc -@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const std::vector<int>& row_groups, - return Status::OK(); - } - -- int64_t num_rows = 0; -+ std::vector<int64_t> num_rows; - for (int row_group : row_groups) { -- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows(); -+ num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows()); - } - - using ::arrow::RecordBatchIterator; -+ int row_group_idx = 0; - - // NB: This lambda will be invoked outside the scope of this call to - // `GetRecordBatchReader()`, so it must capture `readers` and `batch_schema` by value. - // `this` is a non-owning pointer so we are relying on the parent FileReader outliving - // this RecordBatchReader. - ::arrow::Iterator<RecordBatchIterator> batches = ::arrow::MakeFunctionIterator( -- [readers, batch_schema, num_rows, -+ [readers, batch_schema, num_rows, row_group_idx, - this]() mutable -> ::arrow::Result<RecordBatchIterator> { - ::arrow::ChunkedArrayVector columns(readers.size()); - -- // don't reserve more rows than necessary -- int64_t batch_size = std::min(properties().batch_size(), num_rows); -- num_rows -= batch_size; -+ int64_t batch_size = 0; -+ if (!num_rows.empty()) { -+ // don't reserve more rows than necessary -+ batch_size = std::min(properties().batch_size(), num_rows[row_group_idx]); -+ num_rows[row_group_idx] -= batch_size; -+ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) != row_group_idx) { -+ row_group_idx++; -+ } -+ } - - RETURN_NOT_OK(::arrow::internal::OptionalParallelFor( - reader_properties_.use_threads(), static_cast<int>(readers.size()), -diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc -index 4fd7ef1b47..87326a54f1 100644 ---- a/cpp/src/parquet/arrow/writer.cc -+++ b/cpp/src/parquet/arrow/writer.cc -@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter { - return Status::OK(); - } - -+ int64_t GetBufferedSize() override { -+ if (row_group_writer_ == nullptr) { -+ return 0; -+ } -+ return row_group_writer_->total_compressed_bytes() + -+ row_group_writer_->total_compressed_bytes_written(); -+ } -+ - Status Close() override { - if (!closed_) { - // Make idempotent -@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter { - - // Max number of rows allowed in a row group. - const int64_t max_row_group_length = this->properties().max_row_group_length(); -+ const int64_t max_row_group_size = this->properties().max_row_group_size(); - - // Initialize a new buffered row group writer if necessary. - if (row_group_writer_ == nullptr || !row_group_writer_->buffered() || -- row_group_writer_->num_rows() >= max_row_group_length) { -+ row_group_writer_->num_rows() >= max_row_group_length || -+ (row_group_writer_->total_compressed_bytes_written() + -+ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) { - RETURN_NOT_OK(NewBufferedRowGroup()); - } - -diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h -index 4a1a033a7b..0f13d05e44 100644 ---- a/cpp/src/parquet/arrow/writer.h -+++ b/cpp/src/parquet/arrow/writer.h -@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter { - /// option in this case. - virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch) = 0; - -+ /// \brief Return the buffered size in bytes. -+ virtual int64_t GetBufferedSize() = 0; -+ - /// \brief Write the footer and close the file. - virtual ::arrow::Status Close() = 0; - virtual ~FileWriter(); -diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h -index 4d3acb491e..3906ff3c59 100644 ---- a/cpp/src/parquet/properties.h -+++ b/cpp/src/parquet/properties.h -@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true; - static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT = kDefaultDataPageSize; - static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024; - static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024; -+static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024; - static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true; - static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096; - static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN; -@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties { - dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT), - write_batch_size_(DEFAULT_WRITE_BATCH_SIZE), - max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH), -+ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE), - pagesize_(kDefaultDataPageSize), - version_(ParquetVersion::PARQUET_2_6), - data_page_version_(ParquetDataPageVersion::V1), -@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties { - dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()), - write_batch_size_(properties.write_batch_size()), - max_row_group_length_(properties.max_row_group_length()), -+ max_row_group_size_(properties.max_row_group_size()), - pagesize_(properties.data_pagesize()), - version_(properties.version()), - data_page_version_(properties.data_page_version()), -@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties { - return this; - } - -+ /// Specify the max bytes size to put in a single row group. -+ /// Default 128 M. -+ Builder* max_row_group_size(int64_t max_row_group_size) { -+ max_row_group_size_ = max_row_group_size; -+ return this; -+ } -+ - /// Specify the data page size. - /// Default 1MB. - Builder* data_pagesize(int64_t pg_size) { -@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties { - - return std::shared_ptr<WriterProperties>(new WriterProperties( - pool_, dictionary_pagesize_limit_, write_batch_size_, max_row_group_length_, -- pagesize_, version_, created_by_, page_checksum_enabled_, -+ max_row_group_size_, pagesize_, version_, created_by_, page_checksum_enabled_, - std::move(file_encryption_properties_), default_column_properties_, - column_properties, data_page_version_, store_decimal_as_integer_, - std::move(sorting_columns_))); -@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties { - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - ParquetVersion::type version_; - ParquetDataPageVersion data_page_version_; -@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties { - - inline int64_t max_row_group_length() const { return max_row_group_length_; } - -+ inline int64_t max_row_group_size() const { return max_row_group_size_; } -+ - inline int64_t data_pagesize() const { return pagesize_; } - - inline ParquetDataPageVersion data_page_version() const { -@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties { - private: - explicit WriterProperties( - MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t write_batch_size, -- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type version, -+ int64_t max_row_group_length, int64_t max_row_group_size, int64_t pagesize, ParquetVersion::type version, - const std::string& created_by, bool page_write_checksum_enabled, - std::shared_ptr<FileEncryptionProperties> file_encryption_properties, - const ColumnProperties& default_column_properties, -@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties { - dictionary_pagesize_limit_(dictionary_pagesize_limit), - write_batch_size_(write_batch_size), - max_row_group_length_(max_row_group_length), -+ max_row_group_size_(max_row_group_size), - pagesize_(pagesize), - parquet_data_page_version_(data_page_version), - parquet_version_(version), -@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties { - int64_t dictionary_pagesize_limit_; - int64_t write_batch_size_; - int64_t max_row_group_length_; -+ int64_t max_row_group_size_; - int64_t pagesize_; - ParquetDataPageVersion parquet_data_page_version_; - ParquetVersion::type parquet_version_; -diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake -index 9df922afa2..5c8b3d4d07 100644 ---- a/cpp/cmake_modules/ThirdpartyToolchain.cmake -+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT) - REQUIRED_VERSION - 0.11.0) - -- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION}) -+ if(NOT Thrift_VERSION) -+ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION) -+ set(Thrift_VERSION "${thrift_PC_VERSION}") -+ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION) -+ set(Thrift_VERSION "${ThriftAlt_VERSION}") -+ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION) -+ set(Thrift_VERSION "${THRIFT_VERSION}") -+ endif() -+ endif() -+ if(NOT Thrift_VERSION) -+ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift dependency") -+ endif() -+ -+ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}") - list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR) - list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR) - list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH) diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh b/thirdparty/test/arrow-paimon-lifecycle-test.sh index 5ea5ac9cf5a..f946be5c763 100755 --- a/thirdparty/test/arrow-paimon-lifecycle-test.sh +++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh @@ -80,8 +80,8 @@ exercise_semantic_fingerprints() { fingerprint_from_fixture "${first_fixture}" paimon first_paimon [[ "${first_arrow}" == "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || fail "the Arrow legacy marker migration target is stale" - [[ "${first_paimon}" != "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || - fail "the dual-prefix Paimon build unexpectedly accepts the legacy root-prefix marker" + [[ "${first_paimon}" == "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}" ]] || + fail "the Paimon legacy marker migration target is stale" fingerprint_from_fixture "${second_fixture}" arrow second_arrow fingerprint_from_fixture "${second_fixture}" paimon second_paimon [[ "${first_arrow}" == "${second_arrow}" ]] || @@ -155,14 +155,10 @@ create_archive() { arrow_source="arrow-apache-arrow-24.0.0" arrow_archive="apache-arrow-24.0.0.tar.gz" -arrow_17_source="arrow-apache-arrow-17.0.0" -arrow_17_archive="apache-arrow-17.0.0.tar.gz" paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2" -paimon_17_source="${paimon_source}-arrow-17" paimon_archive="paimon-cpp-0a4f4e2.tar.gz" create_archive "${arrow_source}" "${arrow_archive}" arrow -create_archive "${arrow_17_source}" "${arrow_17_archive}" arrow17 create_archive "${paimon_source}" "${paimon_archive}" paimon arrow_patches=( @@ -170,11 +166,6 @@ arrow_patches=( apache-arrow-24.0.0-force-write-int96-timestamps.patch apache-arrow-24.0.0-lzo.patch ) -arrow_17_patches=( - apache-arrow-17.0.0-paimon.patch - apache-arrow-17.0.0-force-write-int96-timestamps.patch - apache-arrow-17.0.0-lzo.patch -) paimon_patches=( paimon-cpp-buildutils-static-deps.patch paimon-cpp-arrow-24-compatibility.patch @@ -183,22 +174,16 @@ paimon_patches=( for index in 0 1 2; do create_patch "${harness}/patches/${arrow_patches[${index}]}" "arrow-$((index + 1)).txt" - create_patch "${harness}/patches/${arrow_17_patches[${index}]}" "arrow17-$((index + 1)).txt" create_patch "${harness}/patches/${paimon_patches[${index}]}" "paimon-$((index + 1)).txt" done arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')" -arrow_17_md5="$(md5sum "${harness}/src/${arrow_17_archive}" | awk '{print $1}')" paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" { printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src" printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed" printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches" - printf '%s\n' 'TP_ARCHIVES=(ARROW_17 ARROW PAIMON_CPP_17 PAIMON_CPP)' - printf 'ARROW_17_NAME="%s"\n' "${arrow_17_archive}" - printf 'ARROW_17_SOURCE="%s"\n' "${arrow_17_source}" - printf 'ARROW_17_MD5SUM="%s"\n' "${arrow_17_md5}" - printf '%s\n' 'ARROW_17_DOWNLOAD="unused"' + printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)' printf 'ARROW_NAME="%s"\n' "${arrow_archive}" printf 'ARROW_SOURCE="%s"\n' "${arrow_source}" printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}" @@ -207,13 +192,7 @@ paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')" printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}" printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}" printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"' - printf 'PAIMON_CPP_17_NAME="%s"\n' "${paimon_archive}" - printf 'PAIMON_CPP_17_ARCHIVE_SOURCE="%s"\n' "${paimon_source}" - printf 'PAIMON_CPP_17_SOURCE="%s"\n' "${paimon_17_source}" - printf 'PAIMON_CPP_17_MD5SUM="%s"\n' "${paimon_md5}" - printf '%s\n' 'PAIMON_CPP_17_DOWNLOAD="unused"' printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n" test-fingerprint; }' - printf '%s\n' 'arrow_paimon_17_build_fingerprint() { printf "%s\n" test-17-fingerprint; }' } >"${harness}/vars.sh" exercise_interrupted_patch_set() { @@ -259,34 +238,6 @@ exercise_interrupted_patch_set ARROW "${arrow_source}" "${arrow_archive}" arrow exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}" "${paimon_archive}" paimon \ "${paimon_patches[@]}" -exercise_legacy_source_isolation() { - local index - - touch "${harness}/src/${arrow_source}/current-arrow-sentinel" - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" ARROW_17 >/dev/null - for index in 1 2 3; do - [[ "$(<"${harness}/src/${arrow_17_source}/arrow17-${index}.txt")" == "patched" ]] || - fail "Arrow 17 patch ${index} was not applied" - done - [[ -f "${harness}/src/${arrow_source}/current-arrow-sentinel" ]] || - fail "extracting Arrow 17 modified the Arrow 24 source" - - touch "${harness}/src/${paimon_source}/current-paimon-sentinel" - TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \ - bash "${harness}/download-thirdparty.sh" PAIMON_CPP_17 >/dev/null - [[ "$(<"${harness}/src/${paimon_17_source}/paimon-1.txt")" == "patched" ]] || - fail "Paimon Arrow 17 static dependency patch was not applied" - for index in 2 3; do - [[ "$(<"${harness}/src/${paimon_17_source}/paimon-${index}.txt")" == "original" ]] || - fail "Paimon Arrow 17 received an Arrow 24-only patch" - done - [[ -f "${harness}/src/${paimon_source}/current-paimon-sentinel" ]] || - fail "extracting Paimon for Arrow 17 modified the Arrow 24 source" -} - -exercise_legacy_source_isolation - exercise_generic_recovery_dispatch() { local generic="${tmpdir}/generic-recovery" local thirdparty_dir="${generic}/thirdparty" @@ -303,8 +254,6 @@ exercise_generic_recovery_dispatch() { local clean local package1 local package2 - local package3 - local package4 local extra mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \ @@ -358,18 +307,6 @@ exercise_generic_recovery_dispatch() { fail "${non_native_target} invoked the native Arrow/Paimon builder" done - rm -f "${args_file}" - if DORIS_THIRDPARTY="${external_thirdparty_dir}" \ - bash "${generic}/build.sh" --cloud >"${output_file}" 2>&1; then - fail "--cloud did not reach the generated-source sentinel" - else - status=$? - fi - [[ "${status}" -eq "${non_native_status}" ]] || - fail "--cloud incorrectly required the BE Arrow/Paimon stack" - [[ ! -e "${args_file}" ]] || - fail "--cloud invoked the native Arrow/Paimon builder" - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ bash "${generic}/build.sh" --fe --clean >"${output_file}" 2>&1; then fail "--fe --clean did not reach the generated-source sentinel" @@ -381,32 +318,6 @@ exercise_generic_recovery_dispatch() { [[ ! -e "${args_file}" ]] || fail "--fe --clean invoked the native Arrow/Paimon builder" - rm -f "${args_file}" - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --compile-bench >"${output_file}" 2>&1; then - fail "standalone --compile-bench skipped stale-prebuilt recovery" - else - status=$? - fi - [[ "${status}" -eq 73 ]] || - fail "standalone --compile-bench failed before invoking its builder" - read -r flag parallel package1 package2 package3 package4 extra <"${args_file}" - [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && - "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" && - "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z "${extra}" ]] || - fail "standalone --compile-bench dispatched the wrong build package set" - - rm -f "${args_file}" - if ARROW_HOME="${generic}/explicit-arrow" PAIMON_HOME="${generic}/explicit-paimon" \ - DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ - bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then - fail "build.sh accepted unsupported explicit Arrow/Paimon prefixes" - fi - grep -Fq "only supports the Arrow/Paimon stack selected from DORIS_THIRDPARTY" \ - "${output_file}" || fail "build.sh did not explain its supported Arrow/Paimon selection" - [[ ! -e "${args_file}" ]] || - fail "an unsupported explicit Arrow/Paimon selection invoked the builder" - if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then fail "generic stale-prebuilt recovery did not invoke the focused builder" @@ -414,10 +325,9 @@ exercise_generic_recovery_dispatch() { status=$? fi [[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking its builder" - read -r flag parallel package1 package2 package3 package4 extra <"${args_file}" + read -r flag parallel package1 package2 extra <"${args_file}" [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && - "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" && - "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z "${extra}" ]] || + "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || fail "generic recovery dispatched the wrong build package set" if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ @@ -427,10 +337,9 @@ exercise_generic_recovery_dispatch() { status=$? fi [[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before invoking its builder" - read -r flag parallel clean package1 package2 package3 package4 extra <"${args_file}" + read -r flag parallel clean package1 package2 extra <"${args_file}" [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" == "--clean" && - "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" && - "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z "${extra}" ]] || + "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z "${extra}" ]] || fail "generic clean recovery dispatched the wrong build package set" if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \ @@ -469,24 +378,18 @@ exercise_generic_recovery_dispatch # A Paimon-only build may publish only its own fingerprint. It must not make a # stale Arrow installation pass the shared prebuilt validation. prebuilt="${tmpdir}/prebuilt" -selected_prebuilt="$(arrow_install_dir "${prebuilt}")" -mkdir -p "${selected_prebuilt}/include/arrow/util" "${selected_prebuilt}/lib64" +mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \ - >"${selected_prebuilt}/include/arrow/util/config.h" + >"${prebuilt}/include/arrow/util/config.h" for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${selected_prebuilt}/lib64/${library}" + touch "${prebuilt}/lib64/${library}" done prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}" [[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || fail "focused recovery dispatches a bundled source package as a build target" -[[ "${ARROW_PAIMON_SHARED_BUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 arrow paimon_cpp" ]] || - fail "shared recovery does not cover both installed Arrow/Paimon stacks" [[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli" ]] || fail "focused recovery does not download the complete Arrow source closure" -prepare_arrow_paimon_download_packages arrow_17 paimon_cpp_17 -[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 xsimd_17 brotli" ]] || - fail "Arrow 17 build does not download its independent source closure" # A legacy prebuilt may have the old combined marker but no component markers. # Generic build.sh consumers must reject it before importing Arrow Compute. @@ -496,20 +399,11 @@ if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then fi printf '%s\n' "${ARROW_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${selected_prebuilt}/arrow-build-fingerprint.txt" + >"${prebuilt}/arrow-build-fingerprint.txt" printf '%s\n' "${PAIMON_LEGACY_BUILD_FINGERPRINTS[0]}" \ - >"${selected_prebuilt}/paimon-build-fingerprint.txt" -arrow_prebuilt_valid "${prebuilt}" || - fail "the unchanged Arrow artifacts were rejected during fingerprint migration" -if paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "the root-prefix Paimon marker certified the dual-prefix Paimon build" -fi -if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then - fail "the shared prebuilt accepted a stale Paimon marker" -fi -publish_paimon_prebuilt_marker "${prebuilt}" + >"${prebuilt}/paimon-build-fingerprint.txt" arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "the migrated Arrow and Paimon component markers were rejected" + fail "the complete shared prebuilt was rejected during fingerprint migration" if ( ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed" arrow_prebuilt_valid "${prebuilt}" @@ -525,13 +419,13 @@ fi publish_arrow_prebuilt_marker "${prebuilt}" publish_paimon_prebuilt_marker "${prebuilt}" -rm "${selected_prebuilt}/lib64/libarrow_compute.a" +rm "${prebuilt}/lib64/libarrow_compute.a" if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then fail "prebuilt validation accepted a missing Arrow Compute archive" fi -touch "${selected_prebuilt}/lib64/libarrow_compute.a" +touch "${prebuilt}/lib64/libarrow_compute.a" -printf '%s\n' stale-arrow >"${selected_prebuilt}/arrow-build-fingerprint.txt" +printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt" if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then fail "Paimon-only marker update certified a stale Arrow build" fi @@ -557,132 +451,5 @@ fi publish_arrow_prebuilt_marker "${prebuilt}" arrow_paimon_prebuilt_valid "${prebuilt}" || fail "republished component markers were rejected" -select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 -[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] || - fail "recovery did not select only the missing Arrow 17 stack" - -# The legacy stack remains at the unversioned prefix and is validated -# independently. Rebuilding or invalidating either stack must not affect the -# other branch's artifacts or fingerprints. -mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64" -printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_17_VERSION}" \ - >"${prebuilt}/include/arrow/util/config.h" -for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}" "${PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${prebuilt}/lib64/${library}" -done -publish_arrow_17_prebuilt_marker "${prebuilt}" -publish_paimon_17_prebuilt_marker "${prebuilt}" -arrow_paimon_17_prebuilt_valid "${prebuilt}" || - fail "matching Arrow 17 and Paimon artifacts were rejected" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "publishing the Arrow 17 stack invalidated Arrow 24" -shared_arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "matching shared Arrow/Paimon stacks were rejected" -select_arrow_paimon_rebuild_packages "${prebuilt}" -[[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]] || - fail "recovery rebuilt an already valid shared stack" - -invalidate_arrow_prebuilt_marker "${prebuilt}" -arrow_paimon_17_prebuilt_valid "${prebuilt}" || - fail "invalidating Arrow 24 affected the Arrow 17 stack" -select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 -[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] || - fail "recovery did not isolate an invalid Arrow 24 stack" -publish_arrow_prebuilt_marker "${prebuilt}" - -invalidate_arrow_17_prebuilt_marker "${prebuilt}" -arrow_paimon_prebuilt_valid "${prebuilt}" || - fail "invalidating Arrow 17 affected the Arrow 24 stack" -select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1 -[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] || - fail "recovery did not isolate an invalid Arrow 17 stack" -publish_arrow_17_prebuilt_marker "${prebuilt}" -arrow_paimon_17_prebuilt_valid "${prebuilt}" || - fail "republished Arrow 17 component markers were rejected" - -# Preparing a root-prefix Arrow downgrade removes Paimon first. An interrupted -# Arrow 17 build therefore cannot expose a root Arrow 17/Paimon 24 mixture to an -# unchanged branch-4.1 consumer. -migration_prefix="${tmpdir}/interrupted-arrow-17-migration" -migration_selected_prefix="$(arrow_install_dir "${migration_prefix}")" -mkdir -p \ - "${migration_prefix}/include/arrow" \ - "${migration_prefix}/include/parquet" \ - "${migration_prefix}/include/paimon" \ - "${migration_prefix}/include/unrelated" \ - "${migration_prefix}/lib64" \ - "${migration_selected_prefix}" -for library in "${ARROW_REQUIRED_LIBRARIES[@]}" "${PAIMON_REQUIRED_LIBRARIES[@]}"; do - touch "${migration_prefix}/lib64/${library}" -done -touch \ - "${migration_prefix}/arrow-build-fingerprint.txt" \ - "${migration_prefix}/paimon-build-fingerprint.txt" \ - "${migration_prefix}/arrow-paimon-build-fingerprint.txt" \ - "${migration_prefix}/arrow-17-build-fingerprint.txt" \ - "${migration_prefix}/paimon-arrow-17-build-fingerprint.txt" \ - "${migration_prefix}/arrow-paimon-17-build-fingerprint.txt" \ - "${migration_prefix}/include/unrelated/sentinel" \ - "${migration_selected_prefix}/sentinel" - -prepare_arrow_17_install_prefix "${migration_prefix}" -[[ ! -e "${migration_prefix}/include/arrow" && - ! -e "${migration_prefix}/include/parquet" && - ! -e "${migration_prefix}/include/paimon" && - ! -e "${migration_prefix}/lib64/libarrow.a" && - ! -e "${migration_prefix}/lib64/libpaimon.a" && - ! -e "${migration_prefix}/arrow-build-fingerprint.txt" && - ! -e "${migration_prefix}/paimon-build-fingerprint.txt" && - ! -e "${migration_prefix}/arrow-17-build-fingerprint.txt" && - ! -e "${migration_prefix}/paimon-arrow-17-build-fingerprint.txt" ]] || - fail "an interrupted Arrow 17 migration left a mixed root-prefix stack" -[[ -e "${migration_prefix}/include/unrelated/sentinel" && - -e "${migration_selected_prefix}/sentinel" ]] || - fail "preparing the Arrow 17 migration removed another stack's artifacts" - -# Reinstalling one stack cleans only files owned by that stack. In particular, -# a downgrade of the legacy prefix must remove Arrow 24-only artifacts without -# deleting Paimon or unrelated thirdparty files. -cleanup_prefix="${tmpdir}/cleanup-prefix" -mkdir -p \ - "${cleanup_prefix}/include/arrow" \ - "${cleanup_prefix}/include/parquet" \ - "${cleanup_prefix}/include/paimon" \ - "${cleanup_prefix}/include/unrelated" \ - "${cleanup_prefix}/lib64/cmake/ArrowCompute" \ - "${cleanup_prefix}/lib64/cmake/Paimon" \ - "${cleanup_prefix}/lib64/pkgconfig" \ - "${cleanup_prefix}/share/arrow" \ - "${cleanup_prefix}/share/doc/arrow" -touch \ - "${cleanup_prefix}/lib64/libarrow_compute.a" \ - "${cleanup_prefix}/lib64/libparquet.a" \ - "${cleanup_prefix}/lib64/libpaimon.a" \ - "${cleanup_prefix}/lib64/libfmt_paimon.a" \ - "${cleanup_prefix}/lib64/libunrelated.a" \ - "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" \ - "${cleanup_prefix}/include/unrelated/sentinel" - -clean_arrow_artifacts_in "${cleanup_prefix}" -[[ ! -e "${cleanup_prefix}/include/arrow" && - ! -e "${cleanup_prefix}/include/parquet" && - ! -e "${cleanup_prefix}/lib64/libarrow_compute.a" && - ! -e "${cleanup_prefix}/lib64/cmake/ArrowCompute" && - ! -e "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" ]] || - fail "Arrow cleanup left stale artifacts in the selected prefix" -[[ -e "${cleanup_prefix}/include/paimon" && - -e "${cleanup_prefix}/lib64/libpaimon.a" && - -e "${cleanup_prefix}/include/unrelated/sentinel" ]] || - fail "Arrow cleanup removed another package's artifacts" - -clean_paimon_artifacts_in "${cleanup_prefix}" -[[ ! -e "${cleanup_prefix}/include/paimon" && - ! -e "${cleanup_prefix}/lib64/libpaimon.a" && - ! -e "${cleanup_prefix}/lib64/libfmt_paimon.a" && - ! -e "${cleanup_prefix}/lib64/cmake/Paimon" ]] || - fail "Paimon cleanup left stale artifacts in the selected prefix" -[[ -e "${cleanup_prefix}/lib64/libunrelated.a" && - -e "${cleanup_prefix}/include/unrelated/sentinel" ]] || - fail "Paimon cleanup removed another package's artifacts" echo "PASS" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 13a3a04e6a6..27fca98f3f9 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -618,7 +618,6 @@ export TP_ARCHIVES=( 'CYRUS_SASL' 'LIBRDKAFKA' 'FLATBUFFERS' - 'ARROW_17' 'ARROW' 'ARROW_ADBC' 'BROTLI' @@ -651,7 +650,6 @@ export TP_ARCHIVES=( 'PDQSORT' 'TIMSORT' 'BENCHMARK' - 'XSIMD_17' 'XSIMD' 'SIMDJSON' 'NLOHMANN_JSON' @@ -674,7 +672,6 @@ export TP_ARCHIVES=( 'JINDOFS' 'JUICEFS' 'PUGIXML' - 'PAIMON_CPP_17' 'PAIMON_CPP' 'LANCE_C' ) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
