This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 33e0c0b69f4 Revert "[fix](build) Keep Arrow 17 and 24 in shared
thirdparty" (#67330)
33e0c0b69f4 is described below
commit 33e0c0b69f4241aff176ef22313432d066be6b9b
Author: daidai <[email protected]>
AuthorDate: Tue Sep 1 17:24:55 2026 +0800
Revert "[fix](build) Keep Arrow 17 and 24 in shared thirdparty" (#67330)
Reverts apache/doris#66546.
no need this.
---
be/CMakeLists.txt | 42 +--
be/cmake/thirdparty.cmake | 59 ++--
build.sh | 53 +---
.../adbc/test_adbc_type_mapping.groovy | 7 +-
thirdparty/arrow-paimon-vars.sh | 341 ++-------------------
thirdparty/build-thirdparty.sh | 121 ++------
thirdparty/download-thirdparty.sh | 67 +---
thirdparty/paimon-cpp-cache.cmake | 27 +-
...arrow-17.0.0-force-write-int96-timestamps.patch | 98 ------
thirdparty/patches/apache-arrow-17.0.0-lzo.patch | 84 -----
.../patches/apache-arrow-17.0.0-paimon.patch | 224 --------------
thirdparty/test/arrow-paimon-lifecycle-test.sh | 265 +---------------
thirdparty/vars.sh | 3 -
13 files changed, 130 insertions(+), 1261 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index c1b3fcd2370..0dc2196dc81 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -200,6 +200,7 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file
cache lru tool or OFF for
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")
option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
+set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
option(ENABLE_TDE "Enable TDE feature module" OFF)
set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src")
option(ENABLE_TLS "Enable TLS feature module" OFF)
@@ -221,31 +222,8 @@ endif()
if (DEFINED ENV{ENABLE_PAIMON_CPP})
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
endif()
-
-# Refresh automatic selections on every configure so a reused CMake cache
-# follows the current DORIS_THIRDPARTY. Environment overrides are explicit and
-# likewise replace any earlier automatic or explicit cache value.
-if (DEFINED ENV{ARROW_HOME})
- set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "Selected Arrow install
prefix" FORCE)
-else()
- set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH
- "Selected Arrow install prefix" FORCE)
-endif()
-if (DEFINED ENV{PAIMON_HOME})
- set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "Selected Paimon install
prefix" FORCE)
-else()
- set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "Selected Paimon install
prefix" FORCE)
-endif()
-
-if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h")
- message(FATAL_ERROR "Selected Arrow installation is incomplete:
${ARROW_HOME}")
-endif()
-message(STATUS "ARROW_HOME is ${ARROW_HOME}")
-if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon")
- message(FATAL_ERROR "Selected Paimon installation is incomplete:
${PAIMON_HOME}")
-endif()
-if (ENABLE_PAIMON_CPP)
- message(STATUS "PAIMON_HOME is ${PAIMON_HOME}")
+if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
+ set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
endif()
set(CMAKE_SKIP_RPATH TRUE)
@@ -638,12 +616,6 @@ include_directories(
${SRC_DIR}/
)
-include_directories(
- BEFORE SYSTEM
- ${ARROW_HOME}/include
- ${PAIMON_HOME}/include
-)
-
include_directories(
SYSTEM
${COMMON_SRC_DIR}
@@ -789,10 +761,10 @@ endif()
set(PAIMON_FACTORY_REGISTRY_LIBS)
if (ENABLE_PAIMON_CPP)
- # The Arrow selected for this BE build includes
COMPUTE/DATASET/ACERO/FILESYSTEM,
- # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in
COMMON_THIRDPARTY
- # via thirdparty.cmake. Paimon uses this same selected Arrow (no
paimon_deps);
- # the legacy Arrow 17 stack remains installed only for pre-upgrade
branch-4.1 consumers.
+ # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
+ # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in
COMMON_THIRDPARTY via
+ # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
+ # No dual-stack selection needed — single Arrow for everything.
# paimon_parquet_file_format depends on Arrow Dataset symbols.
# Force-link it with --whole-archive so its factory registration runs.
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index 1003dda9289..f3bd9867ee0 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -25,11 +25,10 @@ set(COMMON_THIRDPARTY)
# if arg exist noadd, not append to COMMON_THIRDPARTY variable
# if arg exist libname, use libname to find library
# if arg exist wholelibpath, use wholelibpath to find library
-# if arg exist root, resolve the library below that install prefix
function(add_thirdparty)
cmake_parse_arguments(DORIS_THIRDPARTY
"NOTADD;LIB64"
- "LIBNAME;WHOLELIBPATH;ROOT"
+ "LIBNAME;WHOLELIBPATH"
""
${ARGN})
@@ -40,20 +39,14 @@ function(add_thirdparty)
set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME}
PARENT_SCOPE)
endif()
- if (DORIS_THIRDPARTY_ROOT)
- set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT})
- else()
- set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR})
- endif()
-
if (DORIS_THIRDPARTY_LIB64)
- set(DORIS_THIRDPARTY_LIBPATH
${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
+ set(DORIS_THIRDPARTY_LIBPATH
${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
elseif (DORIS_THIRDPARTY_LIBNAME)
- set(DORIS_THIRDPARTY_LIBPATH
${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME})
+ set(DORIS_THIRDPARTY_LIBPATH
${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME})
elseif (DORIS_THIRDPARTY_WHOLELIBPATH)
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH})
else()
- set(DORIS_THIRDPARTY_LIBPATH
${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
+ set(DORIS_THIRDPARTY_LIBPATH
${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
endif()
set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES
IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH})
endfunction()
@@ -107,18 +100,18 @@ add_thirdparty(cares LIB64)
add_thirdparty(address_sorting LIB64)
add_thirdparty(z LIB64)
-add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME})
-add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME})
-add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(brotlicommon LIB64)
+add_thirdparty(brotlidec LIB64)
+add_thirdparty(brotlienc LIB64)
add_thirdparty(zstd LIB64)
-add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME})
-add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME})
-add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME})
-add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME})
-add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME})
-add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(arrow LIB64)
+add_thirdparty(arrow_compute LIB64)
+add_thirdparty(arrow_flight LIB64)
+add_thirdparty(arrow_flight_sql LIB64)
+add_thirdparty(arrow_dataset LIB64)
+add_thirdparty(arrow_acero LIB64)
add_thirdparty(adbc_driver_manager LIB64)
-add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(parquet LIB64)
# liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it
# so the final linker resolves C math symbols from the system library first.
add_thirdparty(lance_c LIB64 NOTADD)
@@ -198,16 +191,16 @@ add_thirdparty(icudata LIB64)
add_thirdparty(pugixml LIB64)
if (ENABLE_PAIMON_CPP)
- add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME})
-
- add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME})
- add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(paimon LIB64)
+ add_thirdparty(paimon_parquet_file_format LIB64)
+ add_thirdparty(paimon_orc_file_format LIB64)
+ add_thirdparty(paimon_blob_file_format LIB64)
+ add_thirdparty(paimon_local_file_system LIB64)
+ add_thirdparty(paimon_file_index LIB64)
+ add_thirdparty(paimon_global_index LIB64)
+
+ add_thirdparty(roaring_bitmap_paimon LIB64)
+ add_thirdparty(xxhash_paimon LIB64)
+ add_thirdparty(fmt_paimon LIB64)
+ add_thirdparty(tbb_paimon LIB64)
endif()
diff --git a/build.sh b/build.sh
index 4331f9b7aa0..b5b531ce707 100755
--- a/build.sh
+++ b/build.sh
@@ -472,18 +472,6 @@ if [[ "${HELP}" -eq 1 ]]; then
usage
fi
-# Normalize compile-bench before dependency selection. The mode is a BE build,
-# even when --compile-bench is the only command-line target.
-if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
- BUILD_BE=1
- BUILD_FE=0
- BUILD_CLOUD=0
- BUILD_HIVE_UDF=0
- BUILD_BE_JAVA_EXTENSIONS=0
- BUILD_BE_CDC_CLIENT=0
- OUTPUT_BE_BINARY=0
-fi
-
if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 &&
${BUILD_CLOUD} -eq 0 ]]; then
clean_gensrc
clean_be
@@ -503,27 +491,12 @@ fi
# shellcheck source=thirdparty/arrow-paimon-vars.sh
. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh"
NEED_ARROW_PAIMON_THIRDPARTY=false
-if [[ "${BUILD_BE}" -eq 1 || "${BUILD_META_TOOL}" == "ON" ||
- "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
+if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 ||
+ "${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" ==
"ON" ||
"${BUILD_INDEX_TOOL}" == "ON" ]]; then
NEED_ARROW_PAIMON_THIRDPARTY=true
fi
-if [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
-
DEFAULT_ARROW_PAIMON_HOME="${DORIS_THIRDPARTY}/installed/${ARROW_INSTALL_SUBDIR}"
- SELECTED_ARROW_HOME="${ARROW_HOME:-${DEFAULT_ARROW_PAIMON_HOME}}"
- SELECTED_PAIMON_HOME="${PAIMON_HOME:-${SELECTED_ARROW_HOME}}"
- if [[ "${SELECTED_ARROW_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ||
- "${SELECTED_PAIMON_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ]]; then
- echo "build.sh only supports the Arrow/Paimon stack selected from
DORIS_THIRDPARTY." >&2
- echo "Expected ARROW_HOME=${DEFAULT_ARROW_PAIMON_HOME} and
PAIMON_HOME=${DEFAULT_ARROW_PAIMON_HOME}." >&2
- echo "Unset ARROW_HOME and PAIMON_HOME, or point DORIS_THIRDPARTY at
the matching thirdparty tree." >&2
- exit 1
- fi
- export ARROW_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
- export PAIMON_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
-fi
-
rebuild_thirdparty_libraries() {
local remove_installed="$1"
shift
@@ -553,7 +526,7 @@ rebuild_thirdparty_libraries() {
build_args+=(--clean)
fi
bash "${build_script}" "${build_args[@]}" "$@"
- if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed";
then
+ if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's
selected inputs." >&2
exit 1
fi
@@ -562,12 +535,10 @@ rebuild_thirdparty_libraries() {
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
echo "Thirdparty libraries need to be build ..."
rebuild_thirdparty_libraries true
-elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
- select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed"
- if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then
- echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
- rebuild_thirdparty_libraries false
"${ARROW_PAIMON_REBUILD_PACKAGES[@]}"
- fi
+elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] &&
+ ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
+ echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
+ rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
fi
update_submodule() {
@@ -772,6 +743,16 @@ for ((i = 0; i < ${#CLOUD_EXTRA_FEATURE_KEYS[@]}; i++)); do
done
if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
+ # BE compile benchmark mode: measure a cold, cache-free BE C++ build.
+ # Everything that is not the BE C++ build would only add noise, so force
+ # a BE-only build regardless of the other options.
+ BUILD_BE=1
+ BUILD_FE=0
+ BUILD_CLOUD=0
+ BUILD_HIVE_UDF=0
+ BUILD_BE_JAVA_EXTENSIONS=0
+ BUILD_BE_CDC_CLIENT=0
+ OUTPUT_BE_BINARY=0
# shellcheck source=build-support/compile-bench/bench-lib.sh
. "${DORIS_HOME}/build-support/compile-bench/bench-lib.sh"
compile_bench_init "${DORIS_HOME}"
diff --git
a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
index c1f47c4a399..9d27447cb70 100644
---
a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
+++
b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
@@ -227,12 +227,7 @@ suite("test_adbc_type_mapping", "p0,external") {
// ---- floating point ----
qt_desc_float """DESC ${catalogName}.${dbName}.t_float"""
- // Render the double with the historical 16-digit baseline so DBL_MAX
is not reparsed by the
- // test client. The bit-for-bit comparison below still validates the
DOUBLE value inside Doris.
- qt_select_float """
- SELECT id, c_float, format('{:.16g}', c_double) AS c_double
- FROM ${catalogName}.${dbName}.t_float ORDER BY id
- """
+ qt_select_float """SELECT id, c_float, c_double FROM
${catalogName}.${dbName}.t_float ORDER BY id"""
// Compared INSIDE Doris rather than through sameAsSource, and not
because ADBC needs the
// help: the fixture's DBL_MAX cannot make the trip to the test client
at all. Doris renders a
diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh
index bee098c0040..d532a740e63 100644
--- a/thirdparty/arrow-paimon-vars.sh
+++ b/thirdparty/arrow-paimon-vars.sh
@@ -21,24 +21,12 @@
# Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can
# distinguish this stack from unrelated thirdparty changes.
-# Arrow 24 is installed in a versioned prefix. The unversioned install prefix
is
-# deliberately reserved for Arrow 17 so the shared thirdparty package remains
-# consumable by branch-4.1 revisions from before the Arrow 24 backport.
+# arrow
ARROW_VERSION="24.0.0"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz";
ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}"
ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436"
-ARROW_INSTALL_SUBDIR="arrow-${ARROW_VERSION}"
-
-# Arrow 17 compatibility stack for pre-upgrade branch-4.1 revisions. Keep these
-# variables separate from ARROW_* so master can build both versions from one
-# source bundle.
-ARROW_17_VERSION="17.0.0"
-ARROW_17_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_17_VERSION}.tar.gz";
-ARROW_17_NAME="apache-arrow-${ARROW_17_VERSION}.tar.gz"
-ARROW_17_SOURCE="arrow-apache-arrow-${ARROW_17_VERSION}"
-ARROW_17_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0"
# Arrow bundled dependencies
BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz";
@@ -51,11 +39,6 @@ XSIMD_NAME="14.0.0.tar.gz"
XSIMD_SOURCE=xsimd-14.0.0
XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1"
-XSIMD_17_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz";
-XSIMD_17_NAME="13.0.0.tar.gz"
-XSIMD_17_SOURCE=xsimd-13.0.0
-XSIMD_17_MD5SUM="c661deb91836e82d3070f81032014fe6"
-
# paimon-cpp
PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz";
PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz"
@@ -94,25 +77,10 @@ PAIMON_LEGACY_BUILD_FINGERPRINTS=(
dbb6ca6e243cb3aa783b7a8011f97afda9e7ea28
)
-# Both Paimon variants use the same archive, but they need independent source
-# trees because only the Arrow 24 tree receives the API compatibility patches.
-PAIMON_CPP_17_DOWNLOAD="${PAIMON_CPP_DOWNLOAD}"
-PAIMON_CPP_17_NAME="${PAIMON_CPP_NAME}"
-PAIMON_CPP_17_ARCHIVE_SOURCE="${PAIMON_CPP_SOURCE}"
-PAIMON_CPP_17_SOURCE="${PAIMON_CPP_SOURCE}-arrow-17"
-PAIMON_CPP_17_MD5SUM="${PAIMON_CPP_MD5SUM}"
-
# Arrow consumes xsimd and Brotli as bundled source archives, but neither is a
# build target in the focused Arrow/Paimon recovery path.
-ARROW_PAIMON_17_BUILD_PACKAGES=(arrow_17 paimon_cpp_17)
ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp)
-ARROW_PAIMON_SHARED_BUILD_PACKAGES=(
- "${ARROW_PAIMON_17_BUILD_PACKAGES[@]}"
- "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
-)
-ARROW_PAIMON_REBUILD_PACKAGES=()
ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli)
-ARROW_17_BUNDLED_SOURCE_PACKAGES=(xsimd_17 brotli)
ARROW_PAIMON_DOWNLOAD_PACKAGES=()
prepare_arrow_paimon_download_packages() {
@@ -121,25 +89,18 @@ prepare_arrow_paimon_download_packages() {
local package
local source_package
local arrow_requested=false
- local arrow_17_requested=false
local source_requested
for package in "$@"; do
if [[ "${package}" == "arrow" ]]; then
arrow_requested=true
- elif [[ "${package}" == "arrow_17" ]]; then
- arrow_17_requested=true
+ break
fi
done
-
- local bundled_source_packages=()
- if [[ "${arrow_requested}" == "true" ]]; then
- bundled_source_packages+=("${ARROW_BUNDLED_SOURCE_PACKAGES[@]}")
- fi
- if [[ "${arrow_17_requested}" == "true" ]]; then
- bundled_source_packages+=("${ARROW_17_BUNDLED_SOURCE_PACKAGES[@]}")
+ if [[ "${arrow_requested}" != "true" ]]; then
+ return
fi
- for source_package in "${bundled_source_packages[@]}"; do
+ for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do
source_requested=false
for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do
if [[ "${package}" == "${source_package}" ]]; then
@@ -153,10 +114,6 @@ prepare_arrow_paimon_download_packages() {
done
}
-arrow_install_dir() {
- printf '%s/%s\n' "$1" "${ARROW_INSTALL_SUBDIR}"
-}
-
# Print stable path-and-content records for fingerprint inputs. Including the
path
# makes patch selection and ordering part of the contract, not only file
contents.
arrow_paimon_fingerprint_files() {
@@ -215,9 +172,7 @@ paimon_build_fingerprint() {
printf 'PAIMON_CPP_MD5SUM=%s\n' "${PAIMON_CPP_MD5SUM}"
arrow_paimon_fingerprint_files \
paimon-cpp-cache.cmake \
- patches/paimon-cpp-buildutils-static-deps.patch \
- patches/paimon-cpp-arrow-24-compatibility.patch \
- patches/paimon-cpp-arrow-24-compute.patch
+ patches/paimon-cpp-*.patch
} | git hash-object --stdin
)
}
@@ -252,57 +207,6 @@ arrow_paimon_fingerprint_matches() {
return 1
}
-arrow_17_build_fingerprint() {
- local vars_dir
- vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
- (
- set -o pipefail
- cd "${vars_dir}" || return 1
- {
- printf 'schema=%s\n' "${ARROW_BUILD_SCHEMA_VERSION}"
- printf 'ARROW_17_VERSION=%s\n' "${ARROW_17_VERSION}"
- printf 'ARROW_17_NAME=%s\n' "${ARROW_17_NAME}"
- printf 'ARROW_17_SOURCE=%s\n' "${ARROW_17_SOURCE}"
- printf 'ARROW_17_MD5SUM=%s\n' "${ARROW_17_MD5SUM}"
- printf 'BROTLI_NAME=%s\n' "${BROTLI_NAME}"
- printf 'BROTLI_SOURCE=%s\n' "${BROTLI_SOURCE}"
- printf 'BROTLI_MD5SUM=%s\n' "${BROTLI_MD5SUM}"
- printf 'XSIMD_17_NAME=%s\n' "${XSIMD_17_NAME}"
- printf 'XSIMD_17_SOURCE=%s\n' "${XSIMD_17_SOURCE}"
- printf 'XSIMD_17_MD5SUM=%s\n' "${XSIMD_17_MD5SUM}"
- arrow_paimon_fingerprint_files \
- patches/apache-arrow-"${ARROW_17_VERSION}"-*.patch
- } | git hash-object --stdin
- )
-}
-
-paimon_17_build_fingerprint() {
- local vars_dir
- vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
- (
- set -o pipefail
- cd "${vars_dir}" || return 1
- {
- arrow_17_build_fingerprint
- printf 'schema=%s\n' "${PAIMON_BUILD_SCHEMA_VERSION}"
- printf 'PAIMON_CPP_17_NAME=%s\n' "${PAIMON_CPP_17_NAME}"
- printf 'PAIMON_CPP_17_ARCHIVE_SOURCE=%s\n'
"${PAIMON_CPP_17_ARCHIVE_SOURCE}"
- printf 'PAIMON_CPP_17_SOURCE=%s\n' "${PAIMON_CPP_17_SOURCE}"
- printf 'PAIMON_CPP_17_MD5SUM=%s\n' "${PAIMON_CPP_17_MD5SUM}"
- arrow_paimon_fingerprint_files \
- paimon-cpp-cache.cmake \
- patches/paimon-cpp-buildutils-static-deps.patch
- } | git hash-object --stdin
- )
-}
-
-arrow_paimon_17_build_fingerprint() {
- {
- arrow_17_build_fingerprint
- paimon_17_build_fingerprint
- } | git hash-object --stdin
-}
-
ARROW_REQUIRED_LIBRARIES=(
libbrotlicommon.a
libbrotlidec.a
@@ -317,19 +221,6 @@ ARROW_REQUIRED_LIBRARIES=(
libparquet.a
)
-ARROW_17_REQUIRED_LIBRARIES=(
- libbrotlicommon.a
- libbrotlidec.a
- libbrotlienc.a
- libarrow.a
- libarrow_flight.a
- libarrow_flight_sql.a
- libarrow_dataset.a
- libarrow_acero.a
- libarrow_bundled_dependencies.a
- libparquet.a
-)
-
PAIMON_REQUIRED_LIBRARIES=(
libpaimon.a
libpaimon_parquet_file_format.a
@@ -344,64 +235,13 @@ PAIMON_REQUIRED_LIBRARIES=(
libtbb_paimon.a
)
-# Remove only artifacts owned by the selected Arrow/Paimon stack before an
-# install. This matters for the legacy prefix: installing Arrow 17 over an
-# existing Arrow 24 prefix must not leave Arrow 24-only headers or libraries
-# behind and turn it into a mixed, internally inconsistent SDK.
-clean_arrow_artifacts_in() {
- local install_dir="$1"
- : "${install_dir:?Arrow install directory must be set}"
-
- rm -rf -- \
- "${install_dir}/include/arrow" \
- "${install_dir}/include/parquet" \
- "${install_dir}/share/arrow" \
- "${install_dir}/share/doc/arrow"
-
- (
- shopt -s nullglob
- local generated_artifacts=(
- "${install_dir}/lib64"/libarrow*
- "${install_dir}/lib64"/libparquet*
- "${install_dir}/lib64/cmake"/Arrow*
- "${install_dir}/lib64/cmake"/Parquet
- "${install_dir}/lib64/pkgconfig"/arrow*.pc
- "${install_dir}/lib64/pkgconfig"/parquet.pc
- )
- rm -rf -- "${generated_artifacts[@]}"
- )
-}
-
-clean_paimon_artifacts_in() {
- local install_dir="$1"
- : "${install_dir:?Paimon install directory must be set}"
-
- rm -rf -- \
- "${install_dir}/include/paimon" \
- "${install_dir}/lib64/cmake/Paimon" \
- "${install_dir}/paimon-cpp"
-
- (
- shopt -s nullglob
- local generated_artifacts=(
- "${install_dir}/lib64"/libpaimon*
- "${install_dir}/lib64"/libroaring_bitmap_paimon.*
- "${install_dir}/lib64"/libxxhash_paimon.*
- "${install_dir}/lib64"/libfmt_paimon.*
- "${install_dir}/lib64"/libtbb_paimon.*
- )
- rm -rf -- "${generated_artifacts[@]}"
- )
-}
-
ARROW_PAIMON_REQUIRED_LIBRARIES=(
"${ARROW_REQUIRED_LIBRARIES[@]}"
"${PAIMON_REQUIRED_LIBRARIES[@]}"
)
arrow_artifacts_valid() {
- local install_dir
- install_dir="$(arrow_install_dir "$1")"
+ local install_dir="$1"
local installed_arrow_version
local library
@@ -428,35 +268,7 @@ arrow_artifacts_valid() {
return 0
}
-arrow_17_artifacts_valid() {
- local install_dir="$1"
- local installed_arrow_version
- local library
-
- if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then
- echo "Missing installed Arrow 17 version header" >&2
- return 1
- fi
- installed_arrow_version="$(
- awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" {
- gsub(/"/, "", $3); print $3; exit
- }' "${install_dir}/include/arrow/util/config.h"
- )"
- if [[ "${installed_arrow_version}" != "${ARROW_17_VERSION}" ]]; then
- echo "Installed legacy Arrow version ${installed_arrow_version} does
not match ${ARROW_17_VERSION}" >&2
- return 1
- fi
-
- for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}"; do
- if [[ ! -f "${install_dir}/lib64/${library}" ]]; then
- echo "Missing Arrow 17 library: ${library}" >&2
- return 1
- fi
- done
- return 0
-}
-
-paimon_artifacts_valid_in() {
+paimon_artifacts_valid() {
local install_dir="$1"
local library
@@ -469,13 +281,8 @@ paimon_artifacts_valid_in() {
return 0
}
-paimon_artifacts_valid() {
- paimon_artifacts_valid_in "$(arrow_install_dir "$1")"
-}
-
arrow_prebuilt_valid() {
- local install_dir
- install_dir="$(arrow_install_dir "$1")"
+ local install_dir="$1"
local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt"
local expected_fingerprint
local installed_fingerprint
@@ -492,12 +299,11 @@ arrow_prebuilt_valid() {
echo "Arrow build fingerprint does not match selected inputs" >&2
return 1
fi
- arrow_artifacts_valid "$1"
+ arrow_artifacts_valid "${install_dir}"
}
paimon_prebuilt_valid() {
- local install_dir
- install_dir="$(arrow_install_dir "$1")"
+ local install_dir="$1"
local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt"
local expected_fingerprint
local installed_fingerprint
@@ -514,7 +320,7 @@ paimon_prebuilt_valid() {
echo "Paimon build fingerprint does not match selected inputs" >&2
return 1
fi
- paimon_artifacts_valid "$1"
+ paimon_artifacts_valid "${install_dir}"
}
arrow_paimon_prebuilt_valid() {
@@ -523,32 +329,26 @@ arrow_paimon_prebuilt_valid() {
}
invalidate_arrow_prebuilt_marker() {
- local install_dir
- install_dir="$(arrow_install_dir "$1")"
- mkdir -p "${install_dir}"
+ local install_dir="$1"
rm -f "${install_dir}/arrow-build-fingerprint.txt" \
"${install_dir}/arrow-paimon-build-fingerprint.txt"
}
publish_arrow_prebuilt_marker() {
- local install_dir
- install_dir="$(arrow_install_dir "$1")"
- arrow_artifacts_valid "$1"
+ local install_dir="$1"
+ arrow_artifacts_valid "${install_dir}"
arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt"
}
invalidate_paimon_prebuilt_marker() {
- local install_dir
- install_dir="$(arrow_install_dir "$1")"
- mkdir -p "${install_dir}"
+ local install_dir="$1"
rm -f "${install_dir}/paimon-build-fingerprint.txt" \
"${install_dir}/arrow-paimon-build-fingerprint.txt"
}
publish_paimon_prebuilt_marker() {
- local install_dir
- install_dir="$(arrow_install_dir "$1")"
- paimon_artifacts_valid "$1"
+ local install_dir="$1"
+ paimon_artifacts_valid "${install_dir}"
paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt"
}
@@ -559,108 +359,3 @@ require_arrow_prebuilt_for_paimon() {
return 1
fi
}
-
-invalidate_arrow_17_prebuilt_marker() {
- local install_dir="$1"
- rm -f "${install_dir}/arrow-17-build-fingerprint.txt" \
- "${install_dir}/arrow-paimon-17-build-fingerprint.txt"
-}
-
-publish_arrow_17_prebuilt_marker() {
- local install_dir="$1"
- arrow_17_artifacts_valid "${install_dir}"
- arrow_17_build_fingerprint >"${install_dir}/arrow-17-build-fingerprint.txt"
-}
-
-arrow_17_prebuilt_valid() {
- local install_dir="$1"
- local fingerprint_mark="${install_dir}/arrow-17-build-fingerprint.txt"
- local expected_fingerprint
-
- if [[ ! -f "${fingerprint_mark}" ]]; then
- echo "Missing Arrow 17 build fingerprint: ${fingerprint_mark}" >&2
- return 1
- fi
- expected_fingerprint="$(arrow_17_build_fingerprint)"
- if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then
- echo "Arrow 17 build fingerprint does not match selected inputs" >&2
- return 1
- fi
- arrow_17_artifacts_valid "${install_dir}"
-}
-
-require_arrow_17_prebuilt_for_paimon() {
- local install_dir="$1"
- if ! arrow_17_prebuilt_valid "${install_dir}"; then
- echo "Paimon for pre-upgrade branch-4.1 requires Arrow 17 to be built
first" >&2
- return 1
- fi
-}
-
-invalidate_paimon_17_prebuilt_marker() {
- local install_dir="$1"
- rm -f "${install_dir}/paimon-arrow-17-build-fingerprint.txt" \
- "${install_dir}/arrow-paimon-17-build-fingerprint.txt"
-}
-
-# A legacy-prefix Arrow downgrade must first remove Paimon built against the
-# previous Arrow ABI. If the Arrow build is interrupted, the incomplete prefix
-# then fails at link time instead of exposing a mixed Arrow/Paimon SDK.
-prepare_arrow_17_install_prefix() {
- local install_dir="$1"
-
- rm -f "${install_dir}/arrow-build-fingerprint.txt" \
- "${install_dir}/paimon-build-fingerprint.txt" \
- "${install_dir}/arrow-paimon-build-fingerprint.txt"
- invalidate_paimon_17_prebuilt_marker "${install_dir}"
- clean_paimon_artifacts_in "${install_dir}"
- invalidate_arrow_17_prebuilt_marker "${install_dir}"
- clean_arrow_artifacts_in "${install_dir}"
-}
-
-publish_paimon_17_prebuilt_marker() {
- local install_dir="$1"
- paimon_artifacts_valid_in "${install_dir}"
- paimon_17_build_fingerprint
>"${install_dir}/paimon-arrow-17-build-fingerprint.txt"
-}
-
-paimon_17_prebuilt_valid() {
- local install_dir="$1"
- local
fingerprint_mark="${install_dir}/paimon-arrow-17-build-fingerprint.txt"
- local expected_fingerprint
-
- if [[ ! -f "${fingerprint_mark}" ]]; then
- echo "Missing Paimon Arrow 17 build fingerprint: ${fingerprint_mark}"
>&2
- return 1
- fi
- expected_fingerprint="$(paimon_17_build_fingerprint)"
- if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then
- echo "Paimon Arrow 17 build fingerprint does not match selected
inputs" >&2
- return 1
- fi
- paimon_artifacts_valid_in "${install_dir}"
-}
-
-arrow_paimon_17_prebuilt_valid() {
- local install_dir="$1"
- arrow_17_prebuilt_valid "${install_dir}" &&
- paimon_17_prebuilt_valid "${install_dir}"
-}
-
-shared_arrow_paimon_prebuilt_valid() {
- local install_dir="$1"
- arrow_paimon_17_prebuilt_valid "${install_dir}" &&
- arrow_paimon_prebuilt_valid "${install_dir}"
-}
-
-select_arrow_paimon_rebuild_packages() {
- local install_dir="$1"
- ARROW_PAIMON_REBUILD_PACKAGES=()
-
- if ! arrow_paimon_17_prebuilt_valid "${install_dir}"; then
- ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_17_BUILD_PACKAGES[@]}")
- fi
- if ! arrow_paimon_prebuilt_valid "${install_dir}"; then
- ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_BUILD_PACKAGES[@]}")
- fi
-}
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index 032e22a13b5..b1d84b1fe78 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -354,26 +354,20 @@ else
echo "Do not strip thirdparty libraries"
fi
-strip_lib_at() {
- local install_dir="$1"
- local library="$2"
+strip_lib() {
if [[ "${STRIP_TP_LIB}" = "ON" ]]; then
- if [[ -z "${library}" ]]; then
+ if [[ -z $1 ]]; then
echo "Must specify the library to be stripped."
exit 1
fi
- if [[ ! -f "${install_dir}/lib/${library}" ]]; then
- echo "Library to be stripped (${install_dir}/lib/${library}) does
not exist."
+ if [[ ! -f "${TP_LIB_DIR}/$1" ]]; then
+ echo "Library to be stripped (${TP_LIB_DIR}/$1) does not exist."
exit 1
fi
- strip --strip-debug --strip-unneeded "${install_dir}/lib/${library}"
+ strip --strip-debug --strip-unneeded "${TP_LIB_DIR}/$1"
fi
}
-strip_lib() {
- strip_lib_at "${TP_INSTALL_DIR}" "$1"
-}
-
#libbacktrace
build_libbacktrace() {
check_if_source_exist "${LIBBACKTRACE_SOURCE}"
@@ -1113,19 +1107,11 @@ build_grpc() {
# sed -i 's/find_dependency/find_package/g'
"${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}
-# Arrow 17 is installed in the legacy unversioned prefix for pre-upgrade
-# branch-4.1 revisions, while Arrow 24 is installed in a versioned prefix
-# selected by master.
-build_arrow_stack() {
- local arrow_source="$1"
- local xsimd_archive="$2"
- local install_dir="$3"
- local has_separate_compute_archive="$4"
-
- check_if_source_exist "${arrow_source}"
- mkdir -p "${install_dir}/lib64"
- ln -sfn lib64 "${install_dir}/lib"
- cd "${TP_SOURCE_DIR}/${arrow_source}/cpp"
+# arrow
+build_arrow() {
+ check_if_source_exist "${ARROW_SOURCE}"
+ invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}"
+ cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp"
mkdir -p release
cd release
@@ -1138,7 +1124,7 @@ build_arrow_stack() {
export ARROW_Thrift_URL="${TP_SOURCE_DIR}/${THRIFT_NAME}"
export ARROW_SNAPPY_URL="${TP_SOURCE_DIR}/${SNAPPY_NAME}"
export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
- export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${xsimd_archive}"
+ export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"
@@ -1160,7 +1146,7 @@ build_arrow_stack() {
-DARROW_FILESYSTEM=ON \
-DARROW_DATASET=ON \
-DARROW_ACERO=ON \
- -DCMAKE_INSTALL_PREFIX="${install_dir}" \
+ -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DARROW_BOOST_USE_SHARED=OFF \
-DARROW_WITH_GRPC=ON \
@@ -1203,30 +1189,15 @@ build_arrow_stack() {
"${BUILD_SYSTEM}" install
#copy dep libs
- cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a
"${install_dir}/lib64/libbrotlienc.a"
- cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a
"${install_dir}/lib64/libbrotlidec.a"
- cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a
"${install_dir}/lib64/libbrotlicommon.a"
- strip_lib_at "${install_dir}" libarrow.a
- if [[ "${has_separate_compute_archive}" == "true" ]]; then
- strip_lib_at "${install_dir}" libarrow_compute.a
- fi
- strip_lib_at "${install_dir}" libparquet.a
- strip_lib_at "${install_dir}" libarrow_dataset.a
- strip_lib_at "${install_dir}" libarrow_acero.a
-}
+ cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlienc.a"
+ cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlidec.a"
+ cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlicommon.a"
+ strip_lib libarrow.a
+ strip_lib libarrow_compute.a
+ strip_lib libparquet.a
+ strip_lib libarrow_dataset.a
+ strip_lib libarrow_acero.a
-build_arrow_17() {
- prepare_arrow_17_install_prefix "${TP_INSTALL_DIR}"
- build_arrow_stack "${ARROW_17_SOURCE}" "${XSIMD_17_NAME}"
"${TP_INSTALL_DIR}" false
- publish_arrow_17_prebuilt_marker "${TP_INSTALL_DIR}"
-}
-
-build_arrow() {
- local install_dir
- install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")"
- invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}"
- clean_arrow_artifacts_in "${install_dir}"
- build_arrow_stack "${ARROW_SOURCE}" "${XSIMD_NAME}" "${install_dir}" true
publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}"
}
@@ -2334,18 +2305,12 @@ build_pugixml() {
cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp"
"${TP_INSTALL_DIR}/include/"
}
-# Build each Paimon variant against the matching Arrow prefix and install it
-# beside that Arrow version. Arrow types cross Paimon's public C++ boundary, so
-# mixing the two versions is not ABI-safe.
-build_paimon_cpp_stack() {
- local paimon_source="$1"
- local arrow_install_dir="$2"
- local install_dir="$3"
-
- check_if_source_exist "${paimon_source}"
- mkdir -p "${install_dir}/lib64"
- ln -sfn lib64 "${install_dir}/lib"
- cd "${TP_SOURCE_DIR}/${paimon_source}"
+# paimon-cpp
+build_paimon_cpp() {
+ check_if_source_exist "${PAIMON_CPP_SOURCE}"
+ require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}"
+ invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}"
+ cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}"
rm -rf "${BUILD_DIR}"
mkdir -p "${BUILD_DIR}"
@@ -2357,13 +2322,12 @@ build_paimon_cpp_stack() {
paimon_linker_flags="${paimon_linker_flags} -lunwind"
fi
- PAIMON_ARROW_INSTALL_DIR="${arrow_install_dir}" \
CXXFLAGS="-Wno-nontrivial-memcall" \
"${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \
-G "${GENERATOR}" \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \
- -DCMAKE_INSTALL_PREFIX="${install_dir}" \
+ -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
-DPAIMON_BUILD_SHARED=OFF \
-DPAIMON_BUILD_STATIC=ON \
-DPAIMON_BUILD_TESTS=OFF \
@@ -2387,7 +2351,7 @@ build_paimon_cpp_stack() {
# reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps
# directory is not needed. When building its own Arrow (legacy), copy
# arrow artefacts into an isolated directory to avoid clashing with Doris.
- local paimon_deps_dir="${install_dir}/paimon-cpp/lib64/paimon_deps"
+ local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps"
if [ -d "arrow_ep-install/lib" ]; then
mkdir -p "${paimon_deps_dir}"
for paimon_arrow_dep in \
@@ -2407,42 +2371,25 @@ build_paimon_cpp_stack() {
# Install roaring_bitmap, renamed to avoid conflict with Doris's
croaringbitmap
if [ -f "release/libroaring_bitmap.a" ]; then
- cp -v "release/libroaring_bitmap.a"
"${install_dir}/lib64/libroaring_bitmap_paimon.a"
+ cp -v "release/libroaring_bitmap.a"
"${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a"
fi
# Install xxhash, renamed to avoid conflict with Doris's xxhash
if [ -f "release/libxxhash.a" ]; then
- cp -v "release/libxxhash.a" "${install_dir}/lib64/libxxhash_paimon.a"
+ cp -v "release/libxxhash.a"
"${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a"
fi
# Install fmt v11 (from fmt_ep-install directory, renamed to avoid
conflict with Doris's fmt v7)
if [ -f "fmt_ep-install/lib/libfmt.a" ]; then
- cp -v "fmt_ep-install/lib/libfmt.a"
"${install_dir}/lib64/libfmt_paimon.a"
+ cp -v "fmt_ep-install/lib/libfmt.a"
"${TP_INSTALL_DIR}/lib64/libfmt_paimon.a"
fi
# Install tbb (from tbb_ep-install directory, renamed to avoid conflict
with Doris's tbb)
if [ -f "tbb_ep-install/lib/libtbb.a" ]; then
- cp -v "tbb_ep-install/lib/libtbb.a"
"${install_dir}/lib64/libtbb_paimon.a"
+ cp -v "tbb_ep-install/lib/libtbb.a"
"${TP_INSTALL_DIR}/lib64/libtbb_paimon.a"
fi
echo "Paimon-cpp internal dependencies installed successfully"
-}
-
-build_paimon_cpp_17() {
- require_arrow_17_prebuilt_for_paimon "${TP_INSTALL_DIR}"
- invalidate_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}"
- clean_paimon_artifacts_in "${TP_INSTALL_DIR}"
- build_paimon_cpp_stack "${PAIMON_CPP_17_SOURCE}" "${TP_INSTALL_DIR}"
"${TP_INSTALL_DIR}"
- publish_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}"
-}
-
-build_paimon_cpp() {
- local install_dir
- install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")"
- require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}"
- invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}"
- clean_paimon_artifacts_in "${install_dir}"
- build_paimon_cpp_stack "${PAIMON_CPP_SOURCE}" "${install_dir}"
"${install_dir}"
publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}"
}
@@ -2547,7 +2494,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
orc
cares
grpc # after cares, protobuf
- arrow_17
arrow
arrow_adbc
lance_c
@@ -2586,7 +2532,6 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
icu
mecab_ipadic
pugixml
- paimon_cpp_17
paimon_cpp
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
@@ -2642,7 +2587,6 @@ cleanup_package_source() {
cyrus_sasl) src_var="CYRUS_SASL_SOURCE" ;;
librdkafka) src_var="LIBRDKAFKA_SOURCE" ;;
flatbuffers) src_var="FLATBUFFERS_SOURCE" ;;
- arrow_17) src_var="ARROW_17_SOURCE" ;;
arrow) src_var="ARROW_SOURCE" ;;
arrow_adbc)
# arrow_adbc also unpacks the prebuilt flightsql driver, clean both
@@ -2696,7 +2640,6 @@ cleanup_package_source() {
jindofs) src_var="JINDOFS_SOURCE" ;;
juicefs) src_var="JUICEFS_SOURCE" ;;
pugixml) src_var="PUGIXML_SOURCE" ;;
- paimon_cpp_17) src_var="PAIMON_CPP_17_SOURCE" ;;
paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;;
lance_c) src_var="LANCE_C_SOURCE" ;;
aws_sdk) src_var="AWS_SDK_SOURCE" ;;
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index 5ac55f7985f..b377cbf08a7 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -271,24 +271,13 @@ for TP_ARCH in "${TP_ARCHIVES[@]}"; do
fi
NAME="${TP_ARCH}_NAME"
SOURCE="${TP_ARCH}_SOURCE"
- ARCHIVE_SOURCE_VAR="${TP_ARCH}_ARCHIVE_SOURCE"
- ARCHIVE_SOURCE="${!ARCHIVE_SOURCE_VAR}"
if [[ -z "${!SOURCE}" ]]; then
continue
fi
if [[ ! -d "${TP_SOURCE_DIR}/${!SOURCE}" ]]; then
- if [[ -n "${ARCHIVE_SOURCE}" && "${ARCHIVE_SOURCE}" != "${!SOURCE}"
]]; then
- alias_unpack_dir="$(mktemp -d
"${TP_SOURCE_DIR}/.unpack-${TP_ARCH}.XXXXXX")"
- if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C
"${alias_unpack_dir}"; then
- echo "Failed to untar ${!NAME} for ${!SOURCE}"
- rm -rf "${alias_unpack_dir}"
- exit 1
- fi
- mv "${alias_unpack_dir}/${ARCHIVE_SOURCE}"
"${TP_SOURCE_DIR}/${!SOURCE}"
- rm -rf "${alias_unpack_dir}"
- elif [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then
+ if [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then
echo "${TP_SOURCE_DIR}/${!NAME}"
echo "${TP_SOURCE_DIR}/${!SOURCE}"
if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C
"${TP_SOURCE_DIR}/"; then
@@ -380,31 +369,17 @@ echo "===== Patching thirdparty archives..."
PATCHED_MARK="patched_mark"
ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint"
ARROW_PAIMON_BUILD_FINGERPRINT=""
-ARROW_PAIMON_17_BUILD_FINGERPRINT=""
if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ||
" ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then
ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)"
fi
-if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " ||
- " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then
- ARROW_PAIMON_17_BUILD_FINGERPRINT="$(arrow_paimon_17_build_fingerprint)"
-fi
reset_arrow_paimon_source() {
local archive_name="$1"
local source_name="$2"
- local archive_source="${3:-${source_name}}"
echo "Resetting ${source_name} because its patch state is incomplete or
stale"
rm -rf "${TP_SOURCE_DIR:?}/${source_name}"
- if [[ "${archive_source}" == "${source_name}" ]]; then
- "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C
"${TP_SOURCE_DIR}/"
- else
- local alias_unpack_dir
- alias_unpack_dir="$(mktemp -d
"${TP_SOURCE_DIR}/.reset-${source_name}.XXXXXX")"
- "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C
"${alias_unpack_dir}"
- mv "${alias_unpack_dir}/${archive_source}"
"${TP_SOURCE_DIR}/${source_name}"
- rm -rf "${alias_unpack_dir}"
- fi
+ "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/"
}
# glog patch
@@ -500,26 +475,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ROCKSDB " ]]; then
echo "Finished patching ${ROCKSDB_SOURCE}"
fi
-# Keep the Arrow 17 source used by pre-upgrade branch-4.1 revisions
independently
-# patched from the Arrow 24 source selected by master.
-if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " ]]; then
-
arrow_17_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
- if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${PATCHED_MARK}" &&
- -f "${arrow_17_fingerprint_mark}" ]] ||
- [[ "$(<"${arrow_17_fingerprint_mark}")" !=
"${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then
- reset_arrow_paimon_source "${ARROW_17_NAME}" "${ARROW_17_SOURCE}"
- cd "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}"
- patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch"
- patch -p1
<"${TP_PATCH_DIR}/apache-arrow-17.0.0-force-write-int96-timestamps.patch"
- patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-lzo.patch"
- touch "${PATCHED_MARK}"
- printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \
- >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
- cd -
- fi
- echo "Finished patching ${ARROW_17_SOURCE}"
-fi
-
# arrow patch is used to get the raw orc reader for filter prune.
if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
@@ -823,24 +778,6 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then
echo "Finished patching ${AZURE_SOURCE}"
fi
-# Keep the Arrow 17 Paimon source free of the Arrow 24 API and Compute patches.
-if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then
-
paimon_17_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
- if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${PATCHED_MARK}" &&
- -f "${paimon_17_fingerprint_mark}" ]] ||
- [[ "$(<"${paimon_17_fingerprint_mark}")" !=
"${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then
- reset_arrow_paimon_source "${PAIMON_CPP_17_NAME}"
"${PAIMON_CPP_17_SOURCE}" \
- "${PAIMON_CPP_17_ARCHIVE_SOURCE}"
- cd "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}"
- patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch"
- touch "${PATCHED_MARK}"
- printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \
- >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
- cd -
- fi
- echo "Finished patching ${PAIMON_CPP_17_SOURCE}"
-fi
-
# patch paimon-cpp
if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then
PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24"
diff --git a/thirdparty/paimon-cpp-cache.cmake
b/thirdparty/paimon-cpp-cache.cmake
index a12540e5d3e..6ab6049c59b 100644
--- a/thirdparty/paimon-cpp-cache.cmake
+++ b/thirdparty/paimon-cpp-cache.cmake
@@ -21,17 +21,12 @@
# Get the Doris thirdparty installation directory from environment
set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty
install directory")
-set(DORIS_ARROW_DIR "$ENV{PAIMON_ARROW_INSTALL_DIR}" CACHE PATH "Selected
Doris Arrow install directory")
if(NOT DORIS_THIRDPARTY_DIR)
message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set")
endif()
-if(NOT DORIS_ARROW_DIR)
- message(FATAL_ERROR "PAIMON_ARROW_INSTALL_DIR environment variable must be
set")
-endif()
message(STATUS "Using Doris thirdparty libraries from:
${DORIS_THIRDPARTY_DIR}")
-message(STATUS "Using selected Arrow libraries from: ${DORIS_ARROW_DIR}")
# Set CMAKE_PREFIX_PATH to help find_package locate our libraries
set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE
STRING "Search path for find_package")
@@ -69,21 +64,21 @@ set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4
include directory")
# ============================================================================
# ============================================================================
-# Arrow - Reuse the version selected by the caller. Both installed Arrow stacks
-# include DATASET/ACERO/FILESYSTEM; Arrow 24 also has a separate Compute
archive.
+# Arrow - Reuse from Doris (Doris Arrow now includes
COMPUTE/DATASET/ACERO/FILESYSTEM)
+# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp
+# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...).
# ============================================================================
set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris
instead of building from source")
-set(DORIS_ARROW_LIB64_DIR "${DORIS_ARROW_DIR}/lib64" CACHE PATH "Selected
Arrow lib64 directory")
-set(DORIS_ARROW_INCLUDE_DIR "${DORIS_ARROW_DIR}/include" CACHE PATH "Selected
Arrow include directory")
+set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64
directory")
-set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_ARROW_INCLUDE_DIR}" CACHE PATH
"Arrow include directory")
-set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow.a" CACHE
FILEPATH "Arrow core library")
-set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute
library")
-set(PAIMON_EXTERNAL_ARROW_DATASET_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset
library")
-set(PAIMON_EXTERNAL_ARROW_ACERO_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero
library")
-set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_ARROW_LIB64_DIR}/libparquet.a" CACHE
FILEPATH "Parquet library")
-set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH
"Arrow bundled dependencies library")
+set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow
include directory")
+set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH
"Arrow core library")
+set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a"
CACHE FILEPATH "Arrow Compute library")
+set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a"
CACHE FILEPATH "Arrow Dataset library")
+set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a"
CACHE FILEPATH "Arrow Acero library")
+set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE
FILEPATH "Parquet library")
+set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB
"${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow
bundled dependencies library")
# Protobuf, Thrift - still built separately by paimon-cpp
diff --git
a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch
b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch
deleted file mode 100644
index 5a754247566..00000000000
--- a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch
+++ /dev/null
@@ -1,98 +0,0 @@
-diff -ruN
arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc
arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc
---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc
2026-03-27 01:23:23.651831424 +0800
-+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc 2026-03-27
01:28:36.855281965 +0800
-@@ -178,7 +178,8 @@
-
- // The user is explicitly asking for Impala int96 encoding, there is no
- // logical type.
-- if (arrow_properties.support_deprecated_int96_timestamps() && target_unit
== ::arrow::TimeUnit::NANO) {
-+ if (arrow_properties.force_write_int96_timestamps() ||
-+ (arrow_properties.support_deprecated_int96_timestamps() && target_unit
== ::arrow::TimeUnit::NANO)) {
- *physical_type = ParquetType::INT96;
- return Status::OK();
- }
-diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h
arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h
---- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h
2026-03-27 01:23:23.643831362 +0800
-+++ arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h 2026-03-27
01:27:47.717897537 +0800
-@@ -980,6 +980,7 @@
- public:
- Builder()
- : write_timestamps_as_int96_(false),
-+ force_write_int96_timestamps_(false),
- coerce_timestamps_enabled_(false),
- coerce_timestamps_unit_(::arrow::TimeUnit::SECOND),
- truncated_timestamps_allowed_(false),
-@@ -1005,6 +1006,21 @@
- return this;
- }
-
-+ /// \brief Force writing legacy int96 timestamps.
-+ ///
-+ /// This bypasses unit-based guards and writes INT96 whenever timestamp
-+ /// metadata is resolved.
-+ Builder* enable_force_write_int96_timestamps() {
-+ force_write_int96_timestamps_ = true;
-+ return this;
-+ }
-+
-+ /// \brief Disable forcing legacy int96 timestamps (default).
-+ Builder* disable_force_write_int96_timestamps() {
-+ force_write_int96_timestamps_ = false;
-+ return this;
-+ }
-+
- /// \brief Coerce all timestamps to the specified time unit.
- /// \param unit time unit to truncate to.
- /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to
microseconds.
-@@ -1085,7 +1101,8 @@
- /// Create the final properties.
- std::shared_ptr<ArrowWriterProperties> build() {
- return std::shared_ptr<ArrowWriterProperties>(new ArrowWriterProperties(
-- write_timestamps_as_int96_, coerce_timestamps_enabled_,
coerce_timestamps_unit_,
-+ write_timestamps_as_int96_, force_write_int96_timestamps_,
-+ coerce_timestamps_enabled_, coerce_timestamps_unit_,
- truncated_timestamps_allowed_, store_schema_,
compliant_nested_types_,
- engine_version_, use_threads_, executor_));
- }
-@@ -1093,6 +1110,8 @@
- private:
- bool write_timestamps_as_int96_;
-
-+ bool force_write_int96_timestamps_;
-+
- bool coerce_timestamps_enabled_;
- ::arrow::TimeUnit::type coerce_timestamps_unit_;
- bool truncated_timestamps_allowed_;
-@@ -1107,6 +1126,8 @@
-
- bool support_deprecated_int96_timestamps() const { return
write_timestamps_as_int96_; }
-
-+ bool force_write_int96_timestamps() const { return
force_write_int96_timestamps_; }
-+
- bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_;
}
- ::arrow::TimeUnit::type coerce_timestamps_unit() const {
- return coerce_timestamps_unit_;
-@@ -1138,6 +1159,7 @@
-
- private:
- explicit ArrowWriterProperties(bool write_nanos_as_int96,
-+ bool force_write_int96_timestamps,
- bool coerce_timestamps_enabled,
- ::arrow::TimeUnit::type
coerce_timestamps_unit,
- bool truncated_timestamps_allowed, bool
store_schema,
-@@ -1145,6 +1167,7 @@
- EngineVersion engine_version, bool
use_threads,
- ::arrow::internal::Executor* executor)
- : write_timestamps_as_int96_(write_nanos_as_int96),
-+ force_write_int96_timestamps_(force_write_int96_timestamps),
- coerce_timestamps_enabled_(coerce_timestamps_enabled),
- coerce_timestamps_unit_(coerce_timestamps_unit),
- truncated_timestamps_allowed_(truncated_timestamps_allowed),
-@@ -1155,6 +1178,7 @@
- executor_(executor) {}
-
- const bool write_timestamps_as_int96_;
-+ const bool force_write_int96_timestamps_;
- const bool coerce_timestamps_enabled_;
- const ::arrow::TimeUnit::type coerce_timestamps_unit_;
- const bool truncated_timestamps_allowed_;
diff --git a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch
b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch
deleted file mode 100644
index a983818413a..00000000000
--- a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch
+++ /dev/null
@@ -1,84 +0,0 @@
---- a/cpp/src/parquet/column_reader.cc
-+++ b/cpp/src/parquet/column_reader.cc
-@@ -30,0 +31,2 @@
-+
-+#include <lzo/lzo1x.h>
-@@ -268,0 +269 @@
-+ compression_codec_(codec),
-@@ -279 +282,7 @@
-- decompressor_ = GetCodec(codec);
-+ if (compression_codec_ == Compression::LZO) {
-+ if (lzo_init() != LZO_E_OK) {
-+ throw ParquetException("Failed to initialize LZO codec");
-+ }
-+ } else {
-+ decompressor_ = GetCodec(codec);
-+ }
-@@ -315,0 +325 @@
-+ Compression::type compression_codec_;
-@@ -585 +595 @@
-- if (decompressor_ == nullptr) {
-+ if (decompressor_ == nullptr && compression_codec_ != Compression::LZO) {
-@@ -601,0 +612,61 @@
-+ if (compression_codec_ == Compression::LZO) {
-+ const uint8_t* input = page_buffer->data() + levels_byte_len;
-+ const uint8_t* const input_end = page_buffer->data() + compressed_len;
-+ uint8_t* output = decompression_buffer_->mutable_data() + levels_byte_len;
-+ uint8_t* const output_end = decompression_buffer_->mutable_data() +
uncompressed_len;
-+
-+ auto load_big_endian_u32 = [](const uint8_t* data) {
-+ return (static_cast<uint32_t>(data[0]) << 24) |
-+ (static_cast<uint32_t>(data[1]) << 16) |
-+ (static_cast<uint32_t>(data[2]) << 8) |
static_cast<uint32_t>(data[3]);
-+ };
-+
-+ while (input < input_end) {
-+ if (input_end - input < 4) {
-+ throw ParquetException("LZO page decompression failed: truncated
large block length");
-+ }
-+
-+ uint32_t large_block_uncompressed_len = load_big_endian_u32(input);
-+ input += 4;
-+ if (static_cast<size_t>(output_end - output) <
large_block_uncompressed_len) {
-+ throw ParquetException("LZO page decompression failed: output buffer
too small");
-+ }
-+
-+ while (large_block_uncompressed_len > 0) {
-+ if (input_end - input < 4) {
-+ throw ParquetException("LZO page decompression failed: truncated
small block length");
-+ }
-+
-+ uint32_t small_block_compressed_len = load_big_endian_u32(input);
-+ input += 4;
-+ if (static_cast<size_t>(input_end - input) <
small_block_compressed_len) {
-+ throw ParquetException("LZO page decompression failed: truncated
small block data");
-+ }
-+
-+ auto small_block_uncompressed_len =
-+ static_cast<lzo_uint>(large_block_uncompressed_len);
-+ const int result =
-+ lzo1x_decompress_safe(input,
static_cast<lzo_uint>(small_block_compressed_len),
-+ output, &small_block_uncompressed_len,
nullptr);
-+ if (result != LZO_E_OK) {
-+ throw ParquetException("LZO page decompression failed, error: " +
-+ std::to_string(result));
-+ }
-+ if (small_block_uncompressed_len > large_block_uncompressed_len) {
-+ throw ParquetException("LZO page decompression failed: invalid
small block size");
-+ }
-+
-+ input += small_block_compressed_len;
-+ output += small_block_uncompressed_len;
-+ large_block_uncompressed_len -= small_block_uncompressed_len;
-+ }
-+ }
-+ if (output != output_end) {
-+ throw ParquetException("Page didn't decompress to expected size,
expected: " +
-+ std::to_string(uncompressed_len -
levels_byte_len) + ", but got:" +
-+ std::to_string(output -
(decompression_buffer_->mutable_data() +
-+ levels_byte_len)));
-+ }
-+
-+ return decompression_buffer_;
-+ }
-+
diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
deleted file mode 100644
index 4e53117b79b..00000000000
--- a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
+++ /dev/null
@@ -1,224 +0,0 @@
-diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc
-index ec3890a41f..943f69bb6c 100644
---- a/cpp/src/parquet/arrow/schema.cc
-+++ b/cpp/src/parquet/arrow/schema.cc
-@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const
::arrow::TimestampType& type,
-
- // The user is explicitly asking for Impala int96 encoding, there is no
- // logical type.
-- if (arrow_properties.support_deprecated_int96_timestamps()) {
-+ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit
== ::arrow::TimeUnit::NANO) {
- *physical_type = ParquetType::INT96;
- return Status::OK();
- }
-
-diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc
-index 285e2a5973..aa6f92f077 100644
---- a/cpp/src/parquet/arrow/reader.cc
-+++ b/cpp/src/parquet/arrow/reader.cc
-@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const
std::vector<int>& row_groups,
- return Status::OK();
- }
-
-- int64_t num_rows = 0;
-+ std::vector<int64_t> num_rows;
- for (int row_group : row_groups) {
-- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows();
-+
num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows());
- }
-
- using ::arrow::RecordBatchIterator;
-+ int row_group_idx = 0;
-
- // NB: This lambda will be invoked outside the scope of this call to
- // `GetRecordBatchReader()`, so it must capture `readers` and
`batch_schema` by value.
- // `this` is a non-owning pointer so we are relying on the parent
FileReader outliving
- // this RecordBatchReader.
- ::arrow::Iterator<RecordBatchIterator> batches =
::arrow::MakeFunctionIterator(
-- [readers, batch_schema, num_rows,
-+ [readers, batch_schema, num_rows, row_group_idx,
- this]() mutable -> ::arrow::Result<RecordBatchIterator> {
- ::arrow::ChunkedArrayVector columns(readers.size());
-
-- // don't reserve more rows than necessary
-- int64_t batch_size = std::min(properties().batch_size(), num_rows);
-- num_rows -= batch_size;
-+ int64_t batch_size = 0;
-+ if (!num_rows.empty()) {
-+ // don't reserve more rows than necessary
-+ batch_size = std::min(properties().batch_size(),
num_rows[row_group_idx]);
-+ num_rows[row_group_idx] -= batch_size;
-+ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) !=
row_group_idx) {
-+ row_group_idx++;
-+ }
-+ }
-
- RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
- reader_properties_.use_threads(),
static_cast<int>(readers.size()),
-diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
-index 4fd7ef1b47..87326a54f1 100644
---- a/cpp/src/parquet/arrow/writer.cc
-+++ b/cpp/src/parquet/arrow/writer.cc
-@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter {
- return Status::OK();
- }
-
-+ int64_t GetBufferedSize() override {
-+ if (row_group_writer_ == nullptr) {
-+ return 0;
-+ }
-+ return row_group_writer_->total_compressed_bytes() +
-+ row_group_writer_->total_compressed_bytes_written();
-+ }
-+
- Status Close() override {
- if (!closed_) {
- // Make idempotent
-@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter {
-
- // Max number of rows allowed in a row group.
- const int64_t max_row_group_length =
this->properties().max_row_group_length();
-+ const int64_t max_row_group_size =
this->properties().max_row_group_size();
-
- // Initialize a new buffered row group writer if necessary.
- if (row_group_writer_ == nullptr || !row_group_writer_->buffered() ||
-- row_group_writer_->num_rows() >= max_row_group_length) {
-+ row_group_writer_->num_rows() >= max_row_group_length ||
-+ (row_group_writer_->total_compressed_bytes_written() +
-+ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) {
- RETURN_NOT_OK(NewBufferedRowGroup());
- }
-
-diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h
-index 4a1a033a7b..0f13d05e44 100644
---- a/cpp/src/parquet/arrow/writer.h
-+++ b/cpp/src/parquet/arrow/writer.h
-@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter {
- /// option in this case.
- virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch)
= 0;
-
-+ /// \brief Return the buffered size in bytes.
-+ virtual int64_t GetBufferedSize() = 0;
-+
- /// \brief Write the footer and close the file.
- virtual ::arrow::Status Close() = 0;
- virtual ~FileWriter();
-diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h
-index 4d3acb491e..3906ff3c59 100644
---- a/cpp/src/parquet/properties.h
-+++ b/cpp/src/parquet/properties.h
-@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
- static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT =
kDefaultDataPageSize;
- static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
- static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024;
-+static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024;
- static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
- static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
- static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN;
-@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties {
- dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
- write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
- max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
-+ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE),
- pagesize_(kDefaultDataPageSize),
- version_(ParquetVersion::PARQUET_2_6),
- data_page_version_(ParquetDataPageVersion::V1),
-@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties {
- dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()),
- write_batch_size_(properties.write_batch_size()),
- max_row_group_length_(properties.max_row_group_length()),
-+ max_row_group_size_(properties.max_row_group_size()),
- pagesize_(properties.data_pagesize()),
- version_(properties.version()),
- data_page_version_(properties.data_page_version()),
-@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties {
- return this;
- }
-
-+ /// Specify the max bytes size to put in a single row group.
-+ /// Default 128 M.
-+ Builder* max_row_group_size(int64_t max_row_group_size) {
-+ max_row_group_size_ = max_row_group_size;
-+ return this;
-+ }
-+
- /// Specify the data page size.
- /// Default 1MB.
- Builder* data_pagesize(int64_t pg_size) {
-@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties {
-
- return std::shared_ptr<WriterProperties>(new WriterProperties(
- pool_, dictionary_pagesize_limit_, write_batch_size_,
max_row_group_length_,
-- pagesize_, version_, created_by_, page_checksum_enabled_,
-+ max_row_group_size_, pagesize_, version_, created_by_,
page_checksum_enabled_,
- std::move(file_encryption_properties_), default_column_properties_,
- column_properties, data_page_version_, store_decimal_as_integer_,
- std::move(sorting_columns_)));
-@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties {
- int64_t dictionary_pagesize_limit_;
- int64_t write_batch_size_;
- int64_t max_row_group_length_;
-+ int64_t max_row_group_size_;
- int64_t pagesize_;
- ParquetVersion::type version_;
- ParquetDataPageVersion data_page_version_;
-@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties {
-
- inline int64_t max_row_group_length() const { return max_row_group_length_;
}
-
-+ inline int64_t max_row_group_size() const { return max_row_group_size_; }
-+
- inline int64_t data_pagesize() const { return pagesize_; }
-
- inline ParquetDataPageVersion data_page_version() const {
-@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties {
- private:
- explicit WriterProperties(
- MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t
write_batch_size,
-- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type
version,
-+ int64_t max_row_group_length, int64_t max_row_group_size, int64_t
pagesize, ParquetVersion::type version,
- const std::string& created_by, bool page_write_checksum_enabled,
- std::shared_ptr<FileEncryptionProperties> file_encryption_properties,
- const ColumnProperties& default_column_properties,
-@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties {
- dictionary_pagesize_limit_(dictionary_pagesize_limit),
- write_batch_size_(write_batch_size),
- max_row_group_length_(max_row_group_length),
-+ max_row_group_size_(max_row_group_size),
- pagesize_(pagesize),
- parquet_data_page_version_(data_page_version),
- parquet_version_(version),
-@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties {
- int64_t dictionary_pagesize_limit_;
- int64_t write_batch_size_;
- int64_t max_row_group_length_;
-+ int64_t max_row_group_size_;
- int64_t pagesize_;
- ParquetDataPageVersion parquet_data_page_version_;
- ParquetVersion::type parquet_version_;
-diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
-index 9df922afa2..5c8b3d4d07 100644
---- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
-+++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
-@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT)
- REQUIRED_VERSION
- 0.11.0)
-
-- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION})
-+ if(NOT Thrift_VERSION)
-+ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION)
-+ set(Thrift_VERSION "${thrift_PC_VERSION}")
-+ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION)
-+ set(Thrift_VERSION "${ThriftAlt_VERSION}")
-+ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION)
-+ set(Thrift_VERSION "${THRIFT_VERSION}")
-+ endif()
-+ endif()
-+ if(NOT Thrift_VERSION)
-+ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift
dependency")
-+ endif()
-+
-+ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}")
- list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR)
- list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR)
- list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH)
diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh
b/thirdparty/test/arrow-paimon-lifecycle-test.sh
index 5ea5ac9cf5a..f946be5c763 100755
--- a/thirdparty/test/arrow-paimon-lifecycle-test.sh
+++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh
@@ -80,8 +80,8 @@ exercise_semantic_fingerprints() {
fingerprint_from_fixture "${first_fixture}" paimon first_paimon
[[ "${first_arrow}" == "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}"
]] ||
fail "the Arrow legacy marker migration target is stale"
- [[ "${first_paimon}" != "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}"
]] ||
- fail "the dual-prefix Paimon build unexpectedly accepts the legacy
root-prefix marker"
+ [[ "${first_paimon}" == "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}"
]] ||
+ fail "the Paimon legacy marker migration target is stale"
fingerprint_from_fixture "${second_fixture}" arrow second_arrow
fingerprint_from_fixture "${second_fixture}" paimon second_paimon
[[ "${first_arrow}" == "${second_arrow}" ]] ||
@@ -155,14 +155,10 @@ create_archive() {
arrow_source="arrow-apache-arrow-24.0.0"
arrow_archive="apache-arrow-24.0.0.tar.gz"
-arrow_17_source="arrow-apache-arrow-17.0.0"
-arrow_17_archive="apache-arrow-17.0.0.tar.gz"
paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2"
-paimon_17_source="${paimon_source}-arrow-17"
paimon_archive="paimon-cpp-0a4f4e2.tar.gz"
create_archive "${arrow_source}" "${arrow_archive}" arrow
-create_archive "${arrow_17_source}" "${arrow_17_archive}" arrow17
create_archive "${paimon_source}" "${paimon_archive}" paimon
arrow_patches=(
@@ -170,11 +166,6 @@ arrow_patches=(
apache-arrow-24.0.0-force-write-int96-timestamps.patch
apache-arrow-24.0.0-lzo.patch
)
-arrow_17_patches=(
- apache-arrow-17.0.0-paimon.patch
- apache-arrow-17.0.0-force-write-int96-timestamps.patch
- apache-arrow-17.0.0-lzo.patch
-)
paimon_patches=(
paimon-cpp-buildutils-static-deps.patch
paimon-cpp-arrow-24-compatibility.patch
@@ -183,22 +174,16 @@ paimon_patches=(
for index in 0 1 2; do
create_patch "${harness}/patches/${arrow_patches[${index}]}"
"arrow-$((index + 1)).txt"
- create_patch "${harness}/patches/${arrow_17_patches[${index}]}"
"arrow17-$((index + 1)).txt"
create_patch "${harness}/patches/${paimon_patches[${index}]}"
"paimon-$((index + 1)).txt"
done
arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')"
-arrow_17_md5="$(md5sum "${harness}/src/${arrow_17_archive}" | awk '{print
$1}')"
paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')"
{
printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src"
printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed"
printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches"
- printf '%s\n' 'TP_ARCHIVES=(ARROW_17 ARROW PAIMON_CPP_17 PAIMON_CPP)'
- printf 'ARROW_17_NAME="%s"\n' "${arrow_17_archive}"
- printf 'ARROW_17_SOURCE="%s"\n' "${arrow_17_source}"
- printf 'ARROW_17_MD5SUM="%s"\n' "${arrow_17_md5}"
- printf '%s\n' 'ARROW_17_DOWNLOAD="unused"'
+ printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)'
printf 'ARROW_NAME="%s"\n' "${arrow_archive}"
printf 'ARROW_SOURCE="%s"\n' "${arrow_source}"
printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}"
@@ -207,13 +192,7 @@ paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" |
awk '{print $1}')"
printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}"
printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}"
printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"'
- printf 'PAIMON_CPP_17_NAME="%s"\n' "${paimon_archive}"
- printf 'PAIMON_CPP_17_ARCHIVE_SOURCE="%s"\n' "${paimon_source}"
- printf 'PAIMON_CPP_17_SOURCE="%s"\n' "${paimon_17_source}"
- printf 'PAIMON_CPP_17_MD5SUM="%s"\n' "${paimon_md5}"
- printf '%s\n' 'PAIMON_CPP_17_DOWNLOAD="unused"'
printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n"
test-fingerprint; }'
- printf '%s\n' 'arrow_paimon_17_build_fingerprint() { printf "%s\n"
test-17-fingerprint; }'
} >"${harness}/vars.sh"
exercise_interrupted_patch_set() {
@@ -259,34 +238,6 @@ exercise_interrupted_patch_set ARROW "${arrow_source}"
"${arrow_archive}" arrow
exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}"
"${paimon_archive}" paimon \
"${paimon_patches[@]}"
-exercise_legacy_source_isolation() {
- local index
-
- touch "${harness}/src/${arrow_source}/current-arrow-sentinel"
- TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \
- bash "${harness}/download-thirdparty.sh" ARROW_17 >/dev/null
- for index in 1 2 3; do
- [[ "$(<"${harness}/src/${arrow_17_source}/arrow17-${index}.txt")" ==
"patched" ]] ||
- fail "Arrow 17 patch ${index} was not applied"
- done
- [[ -f "${harness}/src/${arrow_source}/current-arrow-sentinel" ]] ||
- fail "extracting Arrow 17 modified the Arrow 24 source"
-
- touch "${harness}/src/${paimon_source}/current-paimon-sentinel"
- TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \
- bash "${harness}/download-thirdparty.sh" PAIMON_CPP_17 >/dev/null
- [[ "$(<"${harness}/src/${paimon_17_source}/paimon-1.txt")" == "patched" ]]
||
- fail "Paimon Arrow 17 static dependency patch was not applied"
- for index in 2 3; do
- [[ "$(<"${harness}/src/${paimon_17_source}/paimon-${index}.txt")" ==
"original" ]] ||
- fail "Paimon Arrow 17 received an Arrow 24-only patch"
- done
- [[ -f "${harness}/src/${paimon_source}/current-paimon-sentinel" ]] ||
- fail "extracting Paimon for Arrow 17 modified the Arrow 24 source"
-}
-
-exercise_legacy_source_isolation
-
exercise_generic_recovery_dispatch() {
local generic="${tmpdir}/generic-recovery"
local thirdparty_dir="${generic}/thirdparty"
@@ -303,8 +254,6 @@ exercise_generic_recovery_dispatch() {
local clean
local package1
local package2
- local package3
- local package4
local extra
mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \
@@ -358,18 +307,6 @@ exercise_generic_recovery_dispatch() {
fail "${non_native_target} invoked the native Arrow/Paimon builder"
done
- rm -f "${args_file}"
- if DORIS_THIRDPARTY="${external_thirdparty_dir}" \
- bash "${generic}/build.sh" --cloud >"${output_file}" 2>&1; then
- fail "--cloud did not reach the generated-source sentinel"
- else
- status=$?
- fi
- [[ "${status}" -eq "${non_native_status}" ]] ||
- fail "--cloud incorrectly required the BE Arrow/Paimon stack"
- [[ ! -e "${args_file}" ]] ||
- fail "--cloud invoked the native Arrow/Paimon builder"
-
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
bash "${generic}/build.sh" --fe --clean >"${output_file}" 2>&1; then
fail "--fe --clean did not reach the generated-source sentinel"
@@ -381,32 +318,6 @@ exercise_generic_recovery_dispatch() {
[[ ! -e "${args_file}" ]] ||
fail "--fe --clean invoked the native Arrow/Paimon builder"
- rm -f "${args_file}"
- if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
- bash "${generic}/build.sh" --compile-bench >"${output_file}" 2>&1; then
- fail "standalone --compile-bench skipped stale-prebuilt recovery"
- else
- status=$?
- fi
- [[ "${status}" -eq 73 ]] ||
- fail "standalone --compile-bench failed before invoking its builder"
- read -r flag parallel package1 package2 package3 package4 extra
<"${args_file}"
- [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ &&
- "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" &&
- "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z
"${extra}" ]] ||
- fail "standalone --compile-bench dispatched the wrong build package
set"
-
- rm -f "${args_file}"
- if ARROW_HOME="${generic}/explicit-arrow"
PAIMON_HOME="${generic}/explicit-paimon" \
- DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}"
\
- bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then
- fail "build.sh accepted unsupported explicit Arrow/Paimon prefixes"
- fi
- grep -Fq "only supports the Arrow/Paimon stack selected from
DORIS_THIRDPARTY" \
- "${output_file}" || fail "build.sh did not explain its supported
Arrow/Paimon selection"
- [[ ! -e "${args_file}" ]] ||
- fail "an unsupported explicit Arrow/Paimon selection invoked the
builder"
-
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then
fail "generic stale-prebuilt recovery did not invoke the focused
builder"
@@ -414,10 +325,9 @@ exercise_generic_recovery_dispatch() {
status=$?
fi
[[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking
its builder"
- read -r flag parallel package1 package2 package3 package4 extra
<"${args_file}"
+ read -r flag parallel package1 package2 extra <"${args_file}"
[[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ &&
- "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" &&
- "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z
"${extra}" ]] ||
+ "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z
"${extra}" ]] ||
fail "generic recovery dispatched the wrong build package set"
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
@@ -427,10 +337,9 @@ exercise_generic_recovery_dispatch() {
status=$?
fi
[[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before
invoking its builder"
- read -r flag parallel clean package1 package2 package3 package4 extra
<"${args_file}"
+ read -r flag parallel clean package1 package2 extra <"${args_file}"
[[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" ==
"--clean" &&
- "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" &&
- "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z
"${extra}" ]] ||
+ "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z
"${extra}" ]] ||
fail "generic clean recovery dispatched the wrong build package set"
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
@@ -469,24 +378,18 @@ exercise_generic_recovery_dispatch
# A Paimon-only build may publish only its own fingerprint. It must not make a
# stale Arrow installation pass the shared prebuilt validation.
prebuilt="${tmpdir}/prebuilt"
-selected_prebuilt="$(arrow_install_dir "${prebuilt}")"
-mkdir -p "${selected_prebuilt}/include/arrow/util" "${selected_prebuilt}/lib64"
+mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64"
printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \
- >"${selected_prebuilt}/include/arrow/util/config.h"
+ >"${prebuilt}/include/arrow/util/config.h"
for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do
- touch "${selected_prebuilt}/lib64/${library}"
+ touch "${prebuilt}/lib64/${library}"
done
prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
[[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] ||
fail "focused recovery dispatches a bundled source package as a build
target"
-[[ "${ARROW_PAIMON_SHARED_BUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 arrow
paimon_cpp" ]] ||
- fail "shared recovery does not cover both installed Arrow/Paimon stacks"
[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli"
]] ||
fail "focused recovery does not download the complete Arrow source closure"
-prepare_arrow_paimon_download_packages arrow_17 paimon_cpp_17
-[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 xsimd_17
brotli" ]] ||
- fail "Arrow 17 build does not download its independent source closure"
# A legacy prebuilt may have the old combined marker but no component markers.
# Generic build.sh consumers must reject it before importing Arrow Compute.
@@ -496,20 +399,11 @@ if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null
2>&1; then
fi
printf '%s\n' "${ARROW_LEGACY_BUILD_FINGERPRINTS[0]}" \
- >"${selected_prebuilt}/arrow-build-fingerprint.txt"
+ >"${prebuilt}/arrow-build-fingerprint.txt"
printf '%s\n' "${PAIMON_LEGACY_BUILD_FINGERPRINTS[0]}" \
- >"${selected_prebuilt}/paimon-build-fingerprint.txt"
-arrow_prebuilt_valid "${prebuilt}" ||
- fail "the unchanged Arrow artifacts were rejected during fingerprint
migration"
-if paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
- fail "the root-prefix Paimon marker certified the dual-prefix Paimon build"
-fi
-if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
- fail "the shared prebuilt accepted a stale Paimon marker"
-fi
-publish_paimon_prebuilt_marker "${prebuilt}"
+ >"${prebuilt}/paimon-build-fingerprint.txt"
arrow_paimon_prebuilt_valid "${prebuilt}" ||
- fail "the migrated Arrow and Paimon component markers were rejected"
+ fail "the complete shared prebuilt was rejected during fingerprint
migration"
if (
ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed"
arrow_prebuilt_valid "${prebuilt}"
@@ -525,13 +419,13 @@ fi
publish_arrow_prebuilt_marker "${prebuilt}"
publish_paimon_prebuilt_marker "${prebuilt}"
-rm "${selected_prebuilt}/lib64/libarrow_compute.a"
+rm "${prebuilt}/lib64/libarrow_compute.a"
if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
fail "prebuilt validation accepted a missing Arrow Compute archive"
fi
-touch "${selected_prebuilt}/lib64/libarrow_compute.a"
+touch "${prebuilt}/lib64/libarrow_compute.a"
-printf '%s\n' stale-arrow >"${selected_prebuilt}/arrow-build-fingerprint.txt"
+printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt"
if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
fail "Paimon-only marker update certified a stale Arrow build"
fi
@@ -557,132 +451,5 @@ fi
publish_arrow_prebuilt_marker "${prebuilt}"
arrow_paimon_prebuilt_valid "${prebuilt}" ||
fail "republished component markers were rejected"
-select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1
-[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] ||
- fail "recovery did not select only the missing Arrow 17 stack"
-
-# The legacy stack remains at the unversioned prefix and is validated
-# independently. Rebuilding or invalidating either stack must not affect the
-# other branch's artifacts or fingerprints.
-mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64"
-printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_17_VERSION}" \
- >"${prebuilt}/include/arrow/util/config.h"
-for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}"
"${PAIMON_REQUIRED_LIBRARIES[@]}"; do
- touch "${prebuilt}/lib64/${library}"
-done
-publish_arrow_17_prebuilt_marker "${prebuilt}"
-publish_paimon_17_prebuilt_marker "${prebuilt}"
-arrow_paimon_17_prebuilt_valid "${prebuilt}" ||
- fail "matching Arrow 17 and Paimon artifacts were rejected"
-arrow_paimon_prebuilt_valid "${prebuilt}" ||
- fail "publishing the Arrow 17 stack invalidated Arrow 24"
-shared_arrow_paimon_prebuilt_valid "${prebuilt}" ||
- fail "matching shared Arrow/Paimon stacks were rejected"
-select_arrow_paimon_rebuild_packages "${prebuilt}"
-[[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]] ||
- fail "recovery rebuilt an already valid shared stack"
-
-invalidate_arrow_prebuilt_marker "${prebuilt}"
-arrow_paimon_17_prebuilt_valid "${prebuilt}" ||
- fail "invalidating Arrow 24 affected the Arrow 17 stack"
-select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1
-[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] ||
- fail "recovery did not isolate an invalid Arrow 24 stack"
-publish_arrow_prebuilt_marker "${prebuilt}"
-
-invalidate_arrow_17_prebuilt_marker "${prebuilt}"
-arrow_paimon_prebuilt_valid "${prebuilt}" ||
- fail "invalidating Arrow 17 affected the Arrow 24 stack"
-select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1
-[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] ||
- fail "recovery did not isolate an invalid Arrow 17 stack"
-publish_arrow_17_prebuilt_marker "${prebuilt}"
-arrow_paimon_17_prebuilt_valid "${prebuilt}" ||
- fail "republished Arrow 17 component markers were rejected"
-
-# Preparing a root-prefix Arrow downgrade removes Paimon first. An interrupted
-# Arrow 17 build therefore cannot expose a root Arrow 17/Paimon 24 mixture to
an
-# unchanged branch-4.1 consumer.
-migration_prefix="${tmpdir}/interrupted-arrow-17-migration"
-migration_selected_prefix="$(arrow_install_dir "${migration_prefix}")"
-mkdir -p \
- "${migration_prefix}/include/arrow" \
- "${migration_prefix}/include/parquet" \
- "${migration_prefix}/include/paimon" \
- "${migration_prefix}/include/unrelated" \
- "${migration_prefix}/lib64" \
- "${migration_selected_prefix}"
-for library in "${ARROW_REQUIRED_LIBRARIES[@]}"
"${PAIMON_REQUIRED_LIBRARIES[@]}"; do
- touch "${migration_prefix}/lib64/${library}"
-done
-touch \
- "${migration_prefix}/arrow-build-fingerprint.txt" \
- "${migration_prefix}/paimon-build-fingerprint.txt" \
- "${migration_prefix}/arrow-paimon-build-fingerprint.txt" \
- "${migration_prefix}/arrow-17-build-fingerprint.txt" \
- "${migration_prefix}/paimon-arrow-17-build-fingerprint.txt" \
- "${migration_prefix}/arrow-paimon-17-build-fingerprint.txt" \
- "${migration_prefix}/include/unrelated/sentinel" \
- "${migration_selected_prefix}/sentinel"
-
-prepare_arrow_17_install_prefix "${migration_prefix}"
-[[ ! -e "${migration_prefix}/include/arrow" &&
- ! -e "${migration_prefix}/include/parquet" &&
- ! -e "${migration_prefix}/include/paimon" &&
- ! -e "${migration_prefix}/lib64/libarrow.a" &&
- ! -e "${migration_prefix}/lib64/libpaimon.a" &&
- ! -e "${migration_prefix}/arrow-build-fingerprint.txt" &&
- ! -e "${migration_prefix}/paimon-build-fingerprint.txt" &&
- ! -e "${migration_prefix}/arrow-17-build-fingerprint.txt" &&
- ! -e "${migration_prefix}/paimon-arrow-17-build-fingerprint.txt" ]] ||
- fail "an interrupted Arrow 17 migration left a mixed root-prefix stack"
-[[ -e "${migration_prefix}/include/unrelated/sentinel" &&
- -e "${migration_selected_prefix}/sentinel" ]] ||
- fail "preparing the Arrow 17 migration removed another stack's artifacts"
-
-# Reinstalling one stack cleans only files owned by that stack. In particular,
-# a downgrade of the legacy prefix must remove Arrow 24-only artifacts without
-# deleting Paimon or unrelated thirdparty files.
-cleanup_prefix="${tmpdir}/cleanup-prefix"
-mkdir -p \
- "${cleanup_prefix}/include/arrow" \
- "${cleanup_prefix}/include/parquet" \
- "${cleanup_prefix}/include/paimon" \
- "${cleanup_prefix}/include/unrelated" \
- "${cleanup_prefix}/lib64/cmake/ArrowCompute" \
- "${cleanup_prefix}/lib64/cmake/Paimon" \
- "${cleanup_prefix}/lib64/pkgconfig" \
- "${cleanup_prefix}/share/arrow" \
- "${cleanup_prefix}/share/doc/arrow"
-touch \
- "${cleanup_prefix}/lib64/libarrow_compute.a" \
- "${cleanup_prefix}/lib64/libparquet.a" \
- "${cleanup_prefix}/lib64/libpaimon.a" \
- "${cleanup_prefix}/lib64/libfmt_paimon.a" \
- "${cleanup_prefix}/lib64/libunrelated.a" \
- "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" \
- "${cleanup_prefix}/include/unrelated/sentinel"
-
-clean_arrow_artifacts_in "${cleanup_prefix}"
-[[ ! -e "${cleanup_prefix}/include/arrow" &&
- ! -e "${cleanup_prefix}/include/parquet" &&
- ! -e "${cleanup_prefix}/lib64/libarrow_compute.a" &&
- ! -e "${cleanup_prefix}/lib64/cmake/ArrowCompute" &&
- ! -e "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" ]] ||
- fail "Arrow cleanup left stale artifacts in the selected prefix"
-[[ -e "${cleanup_prefix}/include/paimon" &&
- -e "${cleanup_prefix}/lib64/libpaimon.a" &&
- -e "${cleanup_prefix}/include/unrelated/sentinel" ]] ||
- fail "Arrow cleanup removed another package's artifacts"
-
-clean_paimon_artifacts_in "${cleanup_prefix}"
-[[ ! -e "${cleanup_prefix}/include/paimon" &&
- ! -e "${cleanup_prefix}/lib64/libpaimon.a" &&
- ! -e "${cleanup_prefix}/lib64/libfmt_paimon.a" &&
- ! -e "${cleanup_prefix}/lib64/cmake/Paimon" ]] ||
- fail "Paimon cleanup left stale artifacts in the selected prefix"
-[[ -e "${cleanup_prefix}/lib64/libunrelated.a" &&
- -e "${cleanup_prefix}/include/unrelated/sentinel" ]] ||
- fail "Paimon cleanup removed another package's artifacts"
echo "PASS"
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 33e1bb6e7ce..c9fec64e032 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -625,7 +625,6 @@ export TP_ARCHIVES=(
'CYRUS_SASL'
'LIBRDKAFKA'
'FLATBUFFERS'
- 'ARROW_17'
'ARROW'
'ARROW_ADBC'
'BROTLI'
@@ -658,7 +657,6 @@ export TP_ARCHIVES=(
'PDQSORT'
'TIMSORT'
'BENCHMARK'
- 'XSIMD_17'
'XSIMD'
'SIMDJSON'
'NLOHMANN_JSON'
@@ -681,7 +679,6 @@ export TP_ARCHIVES=(
'JINDOFS'
'JUICEFS'
'PUGIXML'
- 'PAIMON_CPP_17'
'PAIMON_CPP'
'LANCE_C'
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
