This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e5f6299340f [fix](build) Keep Arrow 17 and 24 in shared thirdparty
(#66546)
e5f6299340f is described below
commit e5f6299340f2ba8984e6bc347a2a5836c6b60dd7
Author: daidai <[email protected]>
AuthorDate: Mon Aug 31 09:53:48 2026 +0800
[fix](build) Keep Arrow 17 and 24 in shared thirdparty (#66546)
### What problem does this PR solve?
Related PR: #66221
Problem Summary: master and branch-4.1 consume the same prebuilt
thirdparty prefix. Upgrading the unversioned Arrow/Paimon stack to Arrow
24 makes unchanged branch-4.1 compile against incompatible headers and
libraries. Keep Arrow 17/Paimon 17 in the legacy unversioned prefix,
install Arrow 24/Paimon 24 in a versioned prefix selected by master,
build and validate both source closures independently, and recover only
stale stacks.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [x] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [x] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [x] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [x] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
be/CMakeLists.txt | 42 ++-
be/cmake/thirdparty.cmake | 59 ++--
build.sh | 53 +++-
.../adbc/test_adbc_type_mapping.groovy | 7 +-
thirdparty/arrow-paimon-vars.sh | 341 +++++++++++++++++++--
thirdparty/build-thirdparty.sh | 121 ++++++--
thirdparty/download-thirdparty.sh | 67 +++-
thirdparty/paimon-cpp-cache.cmake | 27 +-
...arrow-17.0.0-force-write-int96-timestamps.patch | 98 ++++++
thirdparty/patches/apache-arrow-17.0.0-lzo.patch | 84 +++++
.../patches/apache-arrow-17.0.0-paimon.patch | 224 ++++++++++++++
thirdparty/test/arrow-paimon-lifecycle-test.sh | 265 +++++++++++++++-
thirdparty/vars.sh | 3 +
13 files changed, 1261 insertions(+), 130 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 524d0322873..a795923540b 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -200,7 +200,6 @@ option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file
cache lru tool or OFF for
message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")
option(ENABLE_PAIMON_CPP "Enable Paimon C++ integration" ON)
-set(PAIMON_HOME "" CACHE PATH "Paimon install prefix")
option(ENABLE_TDE "Enable TDE feature module" OFF)
set(TDE_MODULE_DIR "" CACHE STRING "TDE feature module directory under be/src")
option(ENABLE_TLS "Enable TLS feature module" OFF)
@@ -222,8 +221,31 @@ endif()
if (DEFINED ENV{ENABLE_PAIMON_CPP})
set(ENABLE_PAIMON_CPP "$ENV{ENABLE_PAIMON_CPP}" CACHE BOOL "" FORCE)
endif()
-if (DEFINED ENV{PAIMON_HOME} AND NOT PAIMON_HOME)
- set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "" FORCE)
+
+# Refresh automatic selections on every configure so a reused CMake cache
+# follows the current DORIS_THIRDPARTY. Environment overrides are explicit and
+# likewise replace any earlier automatic or explicit cache value.
+if (DEFINED ENV{ARROW_HOME})
+ set(ARROW_HOME "$ENV{ARROW_HOME}" CACHE PATH "Selected Arrow install
prefix" FORCE)
+else()
+ set(ARROW_HOME "${THIRDPARTY_DIR}/arrow-24.0.0" CACHE PATH
+ "Selected Arrow install prefix" FORCE)
+endif()
+if (DEFINED ENV{PAIMON_HOME})
+ set(PAIMON_HOME "$ENV{PAIMON_HOME}" CACHE PATH "Selected Paimon install
prefix" FORCE)
+else()
+ set(PAIMON_HOME "${ARROW_HOME}" CACHE PATH "Selected Paimon install
prefix" FORCE)
+endif()
+
+if (NOT EXISTS "${ARROW_HOME}/include/arrow/util/config.h")
+ message(FATAL_ERROR "Selected Arrow installation is incomplete:
${ARROW_HOME}")
+endif()
+message(STATUS "ARROW_HOME is ${ARROW_HOME}")
+if (ENABLE_PAIMON_CPP AND NOT EXISTS "${PAIMON_HOME}/include/paimon")
+ message(FATAL_ERROR "Selected Paimon installation is incomplete:
${PAIMON_HOME}")
+endif()
+if (ENABLE_PAIMON_CPP)
+ message(STATUS "PAIMON_HOME is ${PAIMON_HOME}")
endif()
set(CMAKE_SKIP_RPATH TRUE)
@@ -596,6 +618,12 @@ include_directories(
${SRC_DIR}/
)
+include_directories(
+ BEFORE SYSTEM
+ ${ARROW_HOME}/include
+ ${PAIMON_HOME}/include
+)
+
include_directories(
SYSTEM
${COMMON_SRC_DIR}
@@ -741,10 +769,10 @@ endif()
set(PAIMON_FACTORY_REGISTRY_LIBS)
if (ENABLE_PAIMON_CPP)
- # Plan B: Doris Arrow is now built with COMPUTE/DATASET/ACERO/FILESYSTEM,
- # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in
COMMON_THIRDPARTY via
- # thirdparty.cmake. paimon-cpp reuses the same Arrow (no paimon_deps).
- # No dual-stack selection needed — single Arrow for everything.
+ # The Arrow selected for this BE build includes
COMPUTE/DATASET/ACERO/FILESYSTEM,
+ # so arrow, arrow_compute, arrow_dataset, arrow_acero are all in
COMMON_THIRDPARTY
+ # via thirdparty.cmake. Paimon uses this same selected Arrow (no
paimon_deps);
+ # the legacy Arrow 17 stack remains installed only for pre-upgrade
branch-4.1 consumers.
# paimon_parquet_file_format depends on Arrow Dataset symbols.
# Force-link it with --whole-archive so its factory registration runs.
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
index f3bd9867ee0..1003dda9289 100644
--- a/be/cmake/thirdparty.cmake
+++ b/be/cmake/thirdparty.cmake
@@ -25,10 +25,11 @@ set(COMMON_THIRDPARTY)
# if arg exist noadd, not append to COMMON_THIRDPARTY variable
# if arg exist libname, use libname to find library
# if arg exist wholelibpath, use wholelibpath to find library
+# if arg exist root, resolve the library below that install prefix
function(add_thirdparty)
cmake_parse_arguments(DORIS_THIRDPARTY
"NOTADD;LIB64"
- "LIBNAME;WHOLELIBPATH"
+ "LIBNAME;WHOLELIBPATH;ROOT"
""
${ARGN})
@@ -39,14 +40,20 @@ function(add_thirdparty)
set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} ${DORIS_THIRDPARTY_NAME}
PARENT_SCOPE)
endif()
+ if (DORIS_THIRDPARTY_ROOT)
+ set(DORIS_THIRDPARTY_ROOT_DIR ${DORIS_THIRDPARTY_ROOT})
+ else()
+ set(DORIS_THIRDPARTY_ROOT_DIR ${THIRDPARTY_DIR})
+ endif()
+
if (DORIS_THIRDPARTY_LIB64)
- set(DORIS_THIRDPARTY_LIBPATH
${THIRDPARTY_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
+ set(DORIS_THIRDPARTY_LIBPATH
${DORIS_THIRDPARTY_ROOT_DIR}/lib64/lib${DORIS_THIRDPARTY_NAME}.a)
elseif (DORIS_THIRDPARTY_LIBNAME)
- set(DORIS_THIRDPARTY_LIBPATH
${THIRDPARTY_DIR}/${DORIS_THIRDPARTY_LIBNAME})
+ set(DORIS_THIRDPARTY_LIBPATH
${DORIS_THIRDPARTY_ROOT_DIR}/${DORIS_THIRDPARTY_LIBNAME})
elseif (DORIS_THIRDPARTY_WHOLELIBPATH)
set(DORIS_THIRDPARTY_LIBPATH ${DORIS_THIRDPARTY_WHOLELIBPATH})
else()
- set(DORIS_THIRDPARTY_LIBPATH
${THIRDPARTY_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
+ set(DORIS_THIRDPARTY_LIBPATH
${DORIS_THIRDPARTY_ROOT_DIR}/lib/lib${DORIS_THIRDPARTY_NAME}.a)
endif()
set_target_properties(${DORIS_THIRDPARTY_NAME} PROPERTIES
IMPORTED_LOCATION ${DORIS_THIRDPARTY_LIBPATH})
endfunction()
@@ -100,18 +107,18 @@ add_thirdparty(cares LIB64)
add_thirdparty(address_sorting LIB64)
add_thirdparty(z LIB64)
-add_thirdparty(brotlicommon LIB64)
-add_thirdparty(brotlidec LIB64)
-add_thirdparty(brotlienc LIB64)
+add_thirdparty(brotlicommon LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(brotlidec LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(brotlienc LIB64 ROOT ${ARROW_HOME})
add_thirdparty(zstd LIB64)
-add_thirdparty(arrow LIB64)
-add_thirdparty(arrow_compute LIB64)
-add_thirdparty(arrow_flight LIB64)
-add_thirdparty(arrow_flight_sql LIB64)
-add_thirdparty(arrow_dataset LIB64)
-add_thirdparty(arrow_acero LIB64)
+add_thirdparty(arrow LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(arrow_compute LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(arrow_flight LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(arrow_flight_sql LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(arrow_dataset LIB64 ROOT ${ARROW_HOME})
+add_thirdparty(arrow_acero LIB64 ROOT ${ARROW_HOME})
add_thirdparty(adbc_driver_manager LIB64)
-add_thirdparty(parquet LIB64)
+add_thirdparty(parquet LIB64 ROOT ${ARROW_HOME})
# liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it
# so the final linker resolves C math symbols from the system library first.
add_thirdparty(lance_c LIB64 NOTADD)
@@ -191,16 +198,16 @@ add_thirdparty(icudata LIB64)
add_thirdparty(pugixml LIB64)
if (ENABLE_PAIMON_CPP)
- add_thirdparty(paimon LIB64)
- add_thirdparty(paimon_parquet_file_format LIB64)
- add_thirdparty(paimon_orc_file_format LIB64)
- add_thirdparty(paimon_blob_file_format LIB64)
- add_thirdparty(paimon_local_file_system LIB64)
- add_thirdparty(paimon_file_index LIB64)
- add_thirdparty(paimon_global_index LIB64)
-
- add_thirdparty(roaring_bitmap_paimon LIB64)
- add_thirdparty(xxhash_paimon LIB64)
- add_thirdparty(fmt_paimon LIB64)
- add_thirdparty(tbb_paimon LIB64)
+ add_thirdparty(paimon LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(paimon_parquet_file_format LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(paimon_orc_file_format LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(paimon_blob_file_format LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(paimon_local_file_system LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(paimon_file_index LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(paimon_global_index LIB64 ROOT ${PAIMON_HOME})
+
+ add_thirdparty(roaring_bitmap_paimon LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(xxhash_paimon LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(fmt_paimon LIB64 ROOT ${PAIMON_HOME})
+ add_thirdparty(tbb_paimon LIB64 ROOT ${PAIMON_HOME})
endif()
diff --git a/build.sh b/build.sh
index b212706f54f..136715539f1 100755
--- a/build.sh
+++ b/build.sh
@@ -472,6 +472,18 @@ if [[ "${HELP}" -eq 1 ]]; then
usage
fi
+# Normalize compile-bench before dependency selection. The mode is a BE build,
+# even when --compile-bench is the only command-line target.
+if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
+ BUILD_BE=1
+ BUILD_FE=0
+ BUILD_CLOUD=0
+ BUILD_HIVE_UDF=0
+ BUILD_BE_JAVA_EXTENSIONS=0
+ BUILD_BE_CDC_CLIENT=0
+ OUTPUT_BE_BINARY=0
+fi
+
if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 &&
${BUILD_CLOUD} -eq 0 ]]; then
clean_gensrc
clean_be
@@ -491,12 +503,27 @@ fi
# shellcheck source=thirdparty/arrow-paimon-vars.sh
. "${DORIS_HOME}/thirdparty/arrow-paimon-vars.sh"
NEED_ARROW_PAIMON_THIRDPARTY=false
-if [[ "${BUILD_BE}" -eq 1 || "${BUILD_CLOUD}" -eq 1 ||
- "${BUILD_META_TOOL}" == "ON" || "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" ==
"ON" ||
+if [[ "${BUILD_BE}" -eq 1 || "${BUILD_META_TOOL}" == "ON" ||
+ "${BUILD_FILE_CACHE_MICROBENCH_TOOL}" == "ON" ||
"${BUILD_INDEX_TOOL}" == "ON" ]]; then
NEED_ARROW_PAIMON_THIRDPARTY=true
fi
+if [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
+
DEFAULT_ARROW_PAIMON_HOME="${DORIS_THIRDPARTY}/installed/${ARROW_INSTALL_SUBDIR}"
+ SELECTED_ARROW_HOME="${ARROW_HOME:-${DEFAULT_ARROW_PAIMON_HOME}}"
+ SELECTED_PAIMON_HOME="${PAIMON_HOME:-${SELECTED_ARROW_HOME}}"
+ if [[ "${SELECTED_ARROW_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ||
+ "${SELECTED_PAIMON_HOME}" != "${DEFAULT_ARROW_PAIMON_HOME}" ]]; then
+ echo "build.sh only supports the Arrow/Paimon stack selected from
DORIS_THIRDPARTY." >&2
+ echo "Expected ARROW_HOME=${DEFAULT_ARROW_PAIMON_HOME} and
PAIMON_HOME=${DEFAULT_ARROW_PAIMON_HOME}." >&2
+ echo "Unset ARROW_HOME and PAIMON_HOME, or point DORIS_THIRDPARTY at
the matching thirdparty tree." >&2
+ exit 1
+ fi
+ export ARROW_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
+ export PAIMON_HOME="${DEFAULT_ARROW_PAIMON_HOME}"
+fi
+
rebuild_thirdparty_libraries() {
local remove_installed="$1"
shift
@@ -526,7 +553,7 @@ rebuild_thirdparty_libraries() {
build_args+=(--clean)
fi
bash "${build_script}" "${build_args[@]}" "$@"
- if ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
+ if ! shared_arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed";
then
echo "Rebuilt Arrow/Paimon artifacts do not match this checkout's
selected inputs." >&2
exit 1
fi
@@ -535,10 +562,12 @@ rebuild_thirdparty_libraries() {
if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" ]]; then
echo "Thirdparty libraries need to be build ..."
rebuild_thirdparty_libraries true
-elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]] &&
- ! arrow_paimon_prebuilt_valid "${DORIS_THIRDPARTY}/installed"; then
- echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
- rebuild_thirdparty_libraries false "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
+elif [[ "${NEED_ARROW_PAIMON_THIRDPARTY}" == "true" ]]; then
+ select_arrow_paimon_rebuild_packages "${DORIS_THIRDPARTY}/installed"
+ if [[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -gt 0 ]]; then
+ echo "Arrow/Paimon thirdparty libraries need to be rebuilt ..."
+ rebuild_thirdparty_libraries false
"${ARROW_PAIMON_REBUILD_PACKAGES[@]}"
+ fi
fi
update_submodule() {
@@ -728,16 +757,6 @@ for ((i = 0; i < ${#CLOUD_EXTRA_FEATURE_KEYS[@]}; i++)); do
done
if [[ "${COMPILE_BENCH}" -eq 1 ]]; then
- # BE compile benchmark mode: measure a cold, cache-free BE C++ build.
- # Everything that is not the BE C++ build would only add noise, so force
- # a BE-only build regardless of the other options.
- BUILD_BE=1
- BUILD_FE=0
- BUILD_CLOUD=0
- BUILD_HIVE_UDF=0
- BUILD_BE_JAVA_EXTENSIONS=0
- BUILD_BE_CDC_CLIENT=0
- OUTPUT_BE_BINARY=0
# shellcheck source=build-support/compile-bench/bench-lib.sh
. "${DORIS_HOME}/build-support/compile-bench/bench-lib.sh"
compile_bench_init "${DORIS_HOME}"
diff --git
a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
index 9d27447cb70..c1f47c4a399 100644
---
a/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
+++
b/regression-test/suites/external_table_p0/adbc/test_adbc_type_mapping.groovy
@@ -227,7 +227,12 @@ suite("test_adbc_type_mapping", "p0,external") {
// ---- floating point ----
qt_desc_float """DESC ${catalogName}.${dbName}.t_float"""
- qt_select_float """SELECT id, c_float, c_double FROM
${catalogName}.${dbName}.t_float ORDER BY id"""
+ // Render the double with the historical 16-digit baseline so DBL_MAX
is not reparsed by the
+ // test client. The bit-for-bit comparison below still validates the
DOUBLE value inside Doris.
+ qt_select_float """
+ SELECT id, c_float, format('{:.16g}', c_double) AS c_double
+ FROM ${catalogName}.${dbName}.t_float ORDER BY id
+ """
// Compared INSIDE Doris rather than through sameAsSource, and not
because ADBC needs the
// help: the fixture's DBL_MAX cannot make the trip to the test client
at all. Doris renders a
diff --git a/thirdparty/arrow-paimon-vars.sh b/thirdparty/arrow-paimon-vars.sh
index d532a740e63..bee098c0040 100644
--- a/thirdparty/arrow-paimon-vars.sh
+++ b/thirdparty/arrow-paimon-vars.sh
@@ -21,12 +21,24 @@
# Keep the Arrow/Paimon source closure in a dedicated file so targeted CI can
# distinguish this stack from unrelated thirdparty changes.
-# arrow
+# Arrow 24 is installed in a versioned prefix. The unversioned install prefix
is
+# deliberately reserved for Arrow 17 so the shared thirdparty package remains
+# consumable by branch-4.1 revisions from before the Arrow 24 backport.
ARROW_VERSION="24.0.0"
ARROW_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_VERSION}.tar.gz";
ARROW_NAME="apache-arrow-${ARROW_VERSION}.tar.gz"
ARROW_SOURCE="arrow-apache-arrow-${ARROW_VERSION}"
ARROW_MD5SUM="66c53bd00baa79034bd2ca167beea436"
+ARROW_INSTALL_SUBDIR="arrow-${ARROW_VERSION}"
+
+# Arrow 17 compatibility stack for pre-upgrade branch-4.1 revisions. Keep these
+# variables separate from ARROW_* so master can build both versions from one
+# source bundle.
+ARROW_17_VERSION="17.0.0"
+ARROW_17_DOWNLOAD="https://github.com/apache/arrow/archive/refs/tags/apache-arrow-${ARROW_17_VERSION}.tar.gz";
+ARROW_17_NAME="apache-arrow-${ARROW_17_VERSION}.tar.gz"
+ARROW_17_SOURCE="arrow-apache-arrow-${ARROW_17_VERSION}"
+ARROW_17_MD5SUM="ba18bf83e2164abd34b9ac4cb164f0f0"
# Arrow bundled dependencies
BROTLI_DOWNLOAD="https://github.com/google/brotli/archive/v1.0.9.tar.gz";
@@ -39,6 +51,11 @@ XSIMD_NAME="14.0.0.tar.gz"
XSIMD_SOURCE=xsimd-14.0.0
XSIMD_MD5SUM="75c0d34cf7011924ba19978076c76dc1"
+XSIMD_17_DOWNLOAD="https://github.com/xtensor-stack/xsimd/archive/refs/tags/13.0.0.tar.gz";
+XSIMD_17_NAME="13.0.0.tar.gz"
+XSIMD_17_SOURCE=xsimd-13.0.0
+XSIMD_17_MD5SUM="c661deb91836e82d3070f81032014fe6"
+
# paimon-cpp
PAIMON_CPP_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/paimon-cpp-0a4f4e2.tar.gz";
PAIMON_CPP_NAME="paimon-cpp-0a4f4e2.tar.gz"
@@ -77,10 +94,25 @@ PAIMON_LEGACY_BUILD_FINGERPRINTS=(
dbb6ca6e243cb3aa783b7a8011f97afda9e7ea28
)
+# Both Paimon variants use the same archive, but they need independent source
+# trees because only the Arrow 24 tree receives the API compatibility patches.
+PAIMON_CPP_17_DOWNLOAD="${PAIMON_CPP_DOWNLOAD}"
+PAIMON_CPP_17_NAME="${PAIMON_CPP_NAME}"
+PAIMON_CPP_17_ARCHIVE_SOURCE="${PAIMON_CPP_SOURCE}"
+PAIMON_CPP_17_SOURCE="${PAIMON_CPP_SOURCE}-arrow-17"
+PAIMON_CPP_17_MD5SUM="${PAIMON_CPP_MD5SUM}"
+
# Arrow consumes xsimd and Brotli as bundled source archives, but neither is a
# build target in the focused Arrow/Paimon recovery path.
+ARROW_PAIMON_17_BUILD_PACKAGES=(arrow_17 paimon_cpp_17)
ARROW_PAIMON_BUILD_PACKAGES=(arrow paimon_cpp)
+ARROW_PAIMON_SHARED_BUILD_PACKAGES=(
+ "${ARROW_PAIMON_17_BUILD_PACKAGES[@]}"
+ "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
+)
+ARROW_PAIMON_REBUILD_PACKAGES=()
ARROW_BUNDLED_SOURCE_PACKAGES=(xsimd brotli)
+ARROW_17_BUNDLED_SOURCE_PACKAGES=(xsimd_17 brotli)
ARROW_PAIMON_DOWNLOAD_PACKAGES=()
prepare_arrow_paimon_download_packages() {
@@ -89,18 +121,25 @@ prepare_arrow_paimon_download_packages() {
local package
local source_package
local arrow_requested=false
+ local arrow_17_requested=false
local source_requested
for package in "$@"; do
if [[ "${package}" == "arrow" ]]; then
arrow_requested=true
- break
+ elif [[ "${package}" == "arrow_17" ]]; then
+ arrow_17_requested=true
fi
done
- if [[ "${arrow_requested}" != "true" ]]; then
- return
+
+ local bundled_source_packages=()
+ if [[ "${arrow_requested}" == "true" ]]; then
+ bundled_source_packages+=("${ARROW_BUNDLED_SOURCE_PACKAGES[@]}")
+ fi
+ if [[ "${arrow_17_requested}" == "true" ]]; then
+ bundled_source_packages+=("${ARROW_17_BUNDLED_SOURCE_PACKAGES[@]}")
fi
- for source_package in "${ARROW_BUNDLED_SOURCE_PACKAGES[@]}"; do
+ for source_package in "${bundled_source_packages[@]}"; do
source_requested=false
for package in "${ARROW_PAIMON_DOWNLOAD_PACKAGES[@]}"; do
if [[ "${package}" == "${source_package}" ]]; then
@@ -114,6 +153,10 @@ prepare_arrow_paimon_download_packages() {
done
}
+arrow_install_dir() {
+ printf '%s/%s\n' "$1" "${ARROW_INSTALL_SUBDIR}"
+}
+
# Print stable path-and-content records for fingerprint inputs. Including the
path
# makes patch selection and ordering part of the contract, not only file
contents.
arrow_paimon_fingerprint_files() {
@@ -172,7 +215,9 @@ paimon_build_fingerprint() {
printf 'PAIMON_CPP_MD5SUM=%s\n' "${PAIMON_CPP_MD5SUM}"
arrow_paimon_fingerprint_files \
paimon-cpp-cache.cmake \
- patches/paimon-cpp-*.patch
+ patches/paimon-cpp-buildutils-static-deps.patch \
+ patches/paimon-cpp-arrow-24-compatibility.patch \
+ patches/paimon-cpp-arrow-24-compute.patch
} | git hash-object --stdin
)
}
@@ -207,6 +252,57 @@ arrow_paimon_fingerprint_matches() {
return 1
}
+arrow_17_build_fingerprint() {
+ local vars_dir
+ vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ (
+ set -o pipefail
+ cd "${vars_dir}" || return 1
+ {
+ printf 'schema=%s\n' "${ARROW_BUILD_SCHEMA_VERSION}"
+ printf 'ARROW_17_VERSION=%s\n' "${ARROW_17_VERSION}"
+ printf 'ARROW_17_NAME=%s\n' "${ARROW_17_NAME}"
+ printf 'ARROW_17_SOURCE=%s\n' "${ARROW_17_SOURCE}"
+ printf 'ARROW_17_MD5SUM=%s\n' "${ARROW_17_MD5SUM}"
+ printf 'BROTLI_NAME=%s\n' "${BROTLI_NAME}"
+ printf 'BROTLI_SOURCE=%s\n' "${BROTLI_SOURCE}"
+ printf 'BROTLI_MD5SUM=%s\n' "${BROTLI_MD5SUM}"
+ printf 'XSIMD_17_NAME=%s\n' "${XSIMD_17_NAME}"
+ printf 'XSIMD_17_SOURCE=%s\n' "${XSIMD_17_SOURCE}"
+ printf 'XSIMD_17_MD5SUM=%s\n' "${XSIMD_17_MD5SUM}"
+ arrow_paimon_fingerprint_files \
+ patches/apache-arrow-"${ARROW_17_VERSION}"-*.patch
+ } | git hash-object --stdin
+ )
+}
+
+paimon_17_build_fingerprint() {
+ local vars_dir
+ vars_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ (
+ set -o pipefail
+ cd "${vars_dir}" || return 1
+ {
+ arrow_17_build_fingerprint
+ printf 'schema=%s\n' "${PAIMON_BUILD_SCHEMA_VERSION}"
+ printf 'PAIMON_CPP_17_NAME=%s\n' "${PAIMON_CPP_17_NAME}"
+ printf 'PAIMON_CPP_17_ARCHIVE_SOURCE=%s\n'
"${PAIMON_CPP_17_ARCHIVE_SOURCE}"
+ printf 'PAIMON_CPP_17_SOURCE=%s\n' "${PAIMON_CPP_17_SOURCE}"
+ printf 'PAIMON_CPP_17_MD5SUM=%s\n' "${PAIMON_CPP_17_MD5SUM}"
+ arrow_paimon_fingerprint_files \
+ paimon-cpp-cache.cmake \
+ patches/paimon-cpp-buildutils-static-deps.patch
+ } | git hash-object --stdin
+ )
+}
+
+arrow_paimon_17_build_fingerprint() {
+ {
+ arrow_17_build_fingerprint
+ paimon_17_build_fingerprint
+ } | git hash-object --stdin
+}
+
ARROW_REQUIRED_LIBRARIES=(
libbrotlicommon.a
libbrotlidec.a
@@ -221,6 +317,19 @@ ARROW_REQUIRED_LIBRARIES=(
libparquet.a
)
+ARROW_17_REQUIRED_LIBRARIES=(
+ libbrotlicommon.a
+ libbrotlidec.a
+ libbrotlienc.a
+ libarrow.a
+ libarrow_flight.a
+ libarrow_flight_sql.a
+ libarrow_dataset.a
+ libarrow_acero.a
+ libarrow_bundled_dependencies.a
+ libparquet.a
+)
+
PAIMON_REQUIRED_LIBRARIES=(
libpaimon.a
libpaimon_parquet_file_format.a
@@ -235,13 +344,64 @@ PAIMON_REQUIRED_LIBRARIES=(
libtbb_paimon.a
)
+# Remove only artifacts owned by the selected Arrow/Paimon stack before an
+# install. This matters for the legacy prefix: installing Arrow 17 over an
+# existing Arrow 24 prefix must not leave Arrow 24-only headers or libraries
+# behind and turn it into a mixed, internally inconsistent SDK.
+clean_arrow_artifacts_in() {
+ local install_dir="$1"
+ : "${install_dir:?Arrow install directory must be set}"
+
+ rm -rf -- \
+ "${install_dir}/include/arrow" \
+ "${install_dir}/include/parquet" \
+ "${install_dir}/share/arrow" \
+ "${install_dir}/share/doc/arrow"
+
+ (
+ shopt -s nullglob
+ local generated_artifacts=(
+ "${install_dir}/lib64"/libarrow*
+ "${install_dir}/lib64"/libparquet*
+ "${install_dir}/lib64/cmake"/Arrow*
+ "${install_dir}/lib64/cmake"/Parquet
+ "${install_dir}/lib64/pkgconfig"/arrow*.pc
+ "${install_dir}/lib64/pkgconfig"/parquet.pc
+ )
+ rm -rf -- "${generated_artifacts[@]}"
+ )
+}
+
+clean_paimon_artifacts_in() {
+ local install_dir="$1"
+ : "${install_dir:?Paimon install directory must be set}"
+
+ rm -rf -- \
+ "${install_dir}/include/paimon" \
+ "${install_dir}/lib64/cmake/Paimon" \
+ "${install_dir}/paimon-cpp"
+
+ (
+ shopt -s nullglob
+ local generated_artifacts=(
+ "${install_dir}/lib64"/libpaimon*
+ "${install_dir}/lib64"/libroaring_bitmap_paimon.*
+ "${install_dir}/lib64"/libxxhash_paimon.*
+ "${install_dir}/lib64"/libfmt_paimon.*
+ "${install_dir}/lib64"/libtbb_paimon.*
+ )
+ rm -rf -- "${generated_artifacts[@]}"
+ )
+}
+
ARROW_PAIMON_REQUIRED_LIBRARIES=(
"${ARROW_REQUIRED_LIBRARIES[@]}"
"${PAIMON_REQUIRED_LIBRARIES[@]}"
)
arrow_artifacts_valid() {
- local install_dir="$1"
+ local install_dir
+ install_dir="$(arrow_install_dir "$1")"
local installed_arrow_version
local library
@@ -268,7 +428,35 @@ arrow_artifacts_valid() {
return 0
}
-paimon_artifacts_valid() {
+arrow_17_artifacts_valid() {
+ local install_dir="$1"
+ local installed_arrow_version
+ local library
+
+ if [[ ! -f "${install_dir}/include/arrow/util/config.h" ]]; then
+ echo "Missing installed Arrow 17 version header" >&2
+ return 1
+ fi
+ installed_arrow_version="$(
+ awk '$1 == "#define" && $2 == "ARROW_VERSION_STRING" {
+ gsub(/"/, "", $3); print $3; exit
+ }' "${install_dir}/include/arrow/util/config.h"
+ )"
+ if [[ "${installed_arrow_version}" != "${ARROW_17_VERSION}" ]]; then
+ echo "Installed legacy Arrow version ${installed_arrow_version} does
not match ${ARROW_17_VERSION}" >&2
+ return 1
+ fi
+
+ for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}"; do
+ if [[ ! -f "${install_dir}/lib64/${library}" ]]; then
+ echo "Missing Arrow 17 library: ${library}" >&2
+ return 1
+ fi
+ done
+ return 0
+}
+
+paimon_artifacts_valid_in() {
local install_dir="$1"
local library
@@ -281,8 +469,13 @@ paimon_artifacts_valid() {
return 0
}
+paimon_artifacts_valid() {
+ paimon_artifacts_valid_in "$(arrow_install_dir "$1")"
+}
+
arrow_prebuilt_valid() {
- local install_dir="$1"
+ local install_dir
+ install_dir="$(arrow_install_dir "$1")"
local arrow_fingerprint_mark="${install_dir}/arrow-build-fingerprint.txt"
local expected_fingerprint
local installed_fingerprint
@@ -299,11 +492,12 @@ arrow_prebuilt_valid() {
echo "Arrow build fingerprint does not match selected inputs" >&2
return 1
fi
- arrow_artifacts_valid "${install_dir}"
+ arrow_artifacts_valid "$1"
}
paimon_prebuilt_valid() {
- local install_dir="$1"
+ local install_dir
+ install_dir="$(arrow_install_dir "$1")"
local paimon_fingerprint_mark="${install_dir}/paimon-build-fingerprint.txt"
local expected_fingerprint
local installed_fingerprint
@@ -320,7 +514,7 @@ paimon_prebuilt_valid() {
echo "Paimon build fingerprint does not match selected inputs" >&2
return 1
fi
- paimon_artifacts_valid "${install_dir}"
+ paimon_artifacts_valid "$1"
}
arrow_paimon_prebuilt_valid() {
@@ -329,26 +523,32 @@ arrow_paimon_prebuilt_valid() {
}
invalidate_arrow_prebuilt_marker() {
- local install_dir="$1"
+ local install_dir
+ install_dir="$(arrow_install_dir "$1")"
+ mkdir -p "${install_dir}"
rm -f "${install_dir}/arrow-build-fingerprint.txt" \
"${install_dir}/arrow-paimon-build-fingerprint.txt"
}
publish_arrow_prebuilt_marker() {
- local install_dir="$1"
- arrow_artifacts_valid "${install_dir}"
+ local install_dir
+ install_dir="$(arrow_install_dir "$1")"
+ arrow_artifacts_valid "$1"
arrow_build_fingerprint >"${install_dir}/arrow-build-fingerprint.txt"
}
invalidate_paimon_prebuilt_marker() {
- local install_dir="$1"
+ local install_dir
+ install_dir="$(arrow_install_dir "$1")"
+ mkdir -p "${install_dir}"
rm -f "${install_dir}/paimon-build-fingerprint.txt" \
"${install_dir}/arrow-paimon-build-fingerprint.txt"
}
publish_paimon_prebuilt_marker() {
- local install_dir="$1"
- paimon_artifacts_valid "${install_dir}"
+ local install_dir
+ install_dir="$(arrow_install_dir "$1")"
+ paimon_artifacts_valid "$1"
paimon_build_fingerprint >"${install_dir}/paimon-build-fingerprint.txt"
}
@@ -359,3 +559,108 @@ require_arrow_prebuilt_for_paimon() {
return 1
fi
}
+
+invalidate_arrow_17_prebuilt_marker() {
+ local install_dir="$1"
+ rm -f "${install_dir}/arrow-17-build-fingerprint.txt" \
+ "${install_dir}/arrow-paimon-17-build-fingerprint.txt"
+}
+
+publish_arrow_17_prebuilt_marker() {
+ local install_dir="$1"
+ arrow_17_artifacts_valid "${install_dir}"
+ arrow_17_build_fingerprint >"${install_dir}/arrow-17-build-fingerprint.txt"
+}
+
+arrow_17_prebuilt_valid() {
+ local install_dir="$1"
+ local fingerprint_mark="${install_dir}/arrow-17-build-fingerprint.txt"
+ local expected_fingerprint
+
+ if [[ ! -f "${fingerprint_mark}" ]]; then
+ echo "Missing Arrow 17 build fingerprint: ${fingerprint_mark}" >&2
+ return 1
+ fi
+ expected_fingerprint="$(arrow_17_build_fingerprint)"
+ if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then
+ echo "Arrow 17 build fingerprint does not match selected inputs" >&2
+ return 1
+ fi
+ arrow_17_artifacts_valid "${install_dir}"
+}
+
+require_arrow_17_prebuilt_for_paimon() {
+ local install_dir="$1"
+ if ! arrow_17_prebuilt_valid "${install_dir}"; then
+ echo "Paimon for pre-upgrade branch-4.1 requires Arrow 17 to be built
first" >&2
+ return 1
+ fi
+}
+
+invalidate_paimon_17_prebuilt_marker() {
+ local install_dir="$1"
+ rm -f "${install_dir}/paimon-arrow-17-build-fingerprint.txt" \
+ "${install_dir}/arrow-paimon-17-build-fingerprint.txt"
+}
+
+# A legacy-prefix Arrow downgrade must first remove Paimon built against the
+# previous Arrow ABI. If the Arrow build is interrupted, the incomplete prefix
+# then fails at link time instead of exposing a mixed Arrow/Paimon SDK.
+prepare_arrow_17_install_prefix() {
+ local install_dir="$1"
+
+ rm -f "${install_dir}/arrow-build-fingerprint.txt" \
+ "${install_dir}/paimon-build-fingerprint.txt" \
+ "${install_dir}/arrow-paimon-build-fingerprint.txt"
+ invalidate_paimon_17_prebuilt_marker "${install_dir}"
+ clean_paimon_artifacts_in "${install_dir}"
+ invalidate_arrow_17_prebuilt_marker "${install_dir}"
+ clean_arrow_artifacts_in "${install_dir}"
+}
+
+publish_paimon_17_prebuilt_marker() {
+ local install_dir="$1"
+ paimon_artifacts_valid_in "${install_dir}"
+ paimon_17_build_fingerprint
>"${install_dir}/paimon-arrow-17-build-fingerprint.txt"
+}
+
+paimon_17_prebuilt_valid() {
+ local install_dir="$1"
+ local
fingerprint_mark="${install_dir}/paimon-arrow-17-build-fingerprint.txt"
+ local expected_fingerprint
+
+ if [[ ! -f "${fingerprint_mark}" ]]; then
+ echo "Missing Paimon Arrow 17 build fingerprint: ${fingerprint_mark}"
>&2
+ return 1
+ fi
+ expected_fingerprint="$(paimon_17_build_fingerprint)"
+ if [[ "$(<"${fingerprint_mark}")" != "${expected_fingerprint}" ]]; then
+ echo "Paimon Arrow 17 build fingerprint does not match selected
inputs" >&2
+ return 1
+ fi
+ paimon_artifacts_valid_in "${install_dir}"
+}
+
+arrow_paimon_17_prebuilt_valid() {
+ local install_dir="$1"
+ arrow_17_prebuilt_valid "${install_dir}" &&
+ paimon_17_prebuilt_valid "${install_dir}"
+}
+
+shared_arrow_paimon_prebuilt_valid() {
+ local install_dir="$1"
+ arrow_paimon_17_prebuilt_valid "${install_dir}" &&
+ arrow_paimon_prebuilt_valid "${install_dir}"
+}
+
+select_arrow_paimon_rebuild_packages() {
+ local install_dir="$1"
+ ARROW_PAIMON_REBUILD_PACKAGES=()
+
+ if ! arrow_paimon_17_prebuilt_valid "${install_dir}"; then
+ ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_17_BUILD_PACKAGES[@]}")
+ fi
+ if ! arrow_paimon_prebuilt_valid "${install_dir}"; then
+ ARROW_PAIMON_REBUILD_PACKAGES+=("${ARROW_PAIMON_BUILD_PACKAGES[@]}")
+ fi
+}
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
index df4bfe6d1fa..6597f91ed1e 100755
--- a/thirdparty/build-thirdparty.sh
+++ b/thirdparty/build-thirdparty.sh
@@ -354,20 +354,26 @@ else
echo "Do not strip thirdparty libraries"
fi
-strip_lib() {
+strip_lib_at() {
+ local install_dir="$1"
+ local library="$2"
if [[ "${STRIP_TP_LIB}" = "ON" ]]; then
- if [[ -z $1 ]]; then
+ if [[ -z "${library}" ]]; then
echo "Must specify the library to be stripped."
exit 1
fi
- if [[ ! -f "${TP_LIB_DIR}/$1" ]]; then
- echo "Library to be stripped (${TP_LIB_DIR}/$1) does not exist."
+ if [[ ! -f "${install_dir}/lib/${library}" ]]; then
+ echo "Library to be stripped (${install_dir}/lib/${library}) does
not exist."
exit 1
fi
- strip --strip-debug --strip-unneeded "${TP_LIB_DIR}/$1"
+ strip --strip-debug --strip-unneeded "${install_dir}/lib/${library}"
fi
}
+strip_lib() {
+ strip_lib_at "${TP_INSTALL_DIR}" "$1"
+}
+
#libbacktrace
build_libbacktrace() {
check_if_source_exist "${LIBBACKTRACE_SOURCE}"
@@ -1103,11 +1109,19 @@ build_grpc() {
# sed -i 's/find_dependency/find_package/g'
"${TP_INSTALL_DIR}"/lib64/cmake/grpc/gRPCConfig.cmake
}
-# arrow
-build_arrow() {
- check_if_source_exist "${ARROW_SOURCE}"
- invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}"
- cd "${TP_SOURCE_DIR}/${ARROW_SOURCE}/cpp"
+# Arrow 17 is installed in the legacy unversioned prefix for pre-upgrade
+# branch-4.1 revisions, while Arrow 24 is installed in a versioned prefix
+# selected by master.
+build_arrow_stack() {
+ local arrow_source="$1"
+ local xsimd_archive="$2"
+ local install_dir="$3"
+ local has_separate_compute_archive="$4"
+
+ check_if_source_exist "${arrow_source}"
+ mkdir -p "${install_dir}/lib64"
+ ln -sfn lib64 "${install_dir}/lib"
+ cd "${TP_SOURCE_DIR}/${arrow_source}/cpp"
mkdir -p release
cd release
@@ -1120,7 +1134,7 @@ build_arrow() {
export ARROW_Thrift_URL="${TP_SOURCE_DIR}/${THRIFT_NAME}"
export ARROW_SNAPPY_URL="${TP_SOURCE_DIR}/${SNAPPY_NAME}"
export ARROW_ZLIB_URL="${TP_SOURCE_DIR}/${ZLIB_NAME}"
- export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${XSIMD_NAME}"
+ export ARROW_XSIMD_URL="${TP_SOURCE_DIR}/${xsimd_archive}"
export ARROW_ORC_URL="${TP_SOURCE_DIR}/${ORC_NAME}"
export ARROW_GRPC_URL="${TP_SOURCE_DIR}/${GRPC_NAME}"
export ARROW_PROTOBUF_URL="${TP_SOURCE_DIR}/${PROTOBUF_NAME}"
@@ -1142,7 +1156,7 @@ build_arrow() {
-DARROW_FILESYSTEM=ON \
-DARROW_DATASET=ON \
-DARROW_ACERO=ON \
- -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
+ -DCMAKE_INSTALL_PREFIX="${install_dir}" \
-DCMAKE_INSTALL_LIBDIR=lib64 \
-DARROW_BOOST_USE_SHARED=OFF \
-DARROW_WITH_GRPC=ON \
@@ -1185,15 +1199,30 @@ build_arrow() {
"${BUILD_SYSTEM}" install
#copy dep libs
- cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlienc.a"
- cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlidec.a"
- cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a
"${TP_INSTALL_DIR}/lib64/libbrotlicommon.a"
- strip_lib libarrow.a
- strip_lib libarrow_compute.a
- strip_lib libparquet.a
- strip_lib libarrow_dataset.a
- strip_lib libarrow_acero.a
+ cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlienc-static.a
"${install_dir}/lib64/libbrotlienc.a"
+ cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlidec-static.a
"${install_dir}/lib64/libbrotlidec.a"
+ cp -rf ./brotli_ep/src/brotli_ep-install/lib/libbrotlicommon-static.a
"${install_dir}/lib64/libbrotlicommon.a"
+ strip_lib_at "${install_dir}" libarrow.a
+ if [[ "${has_separate_compute_archive}" == "true" ]]; then
+ strip_lib_at "${install_dir}" libarrow_compute.a
+ fi
+ strip_lib_at "${install_dir}" libparquet.a
+ strip_lib_at "${install_dir}" libarrow_dataset.a
+ strip_lib_at "${install_dir}" libarrow_acero.a
+}
+build_arrow_17() {
+ prepare_arrow_17_install_prefix "${TP_INSTALL_DIR}"
+ build_arrow_stack "${ARROW_17_SOURCE}" "${XSIMD_17_NAME}"
"${TP_INSTALL_DIR}" false
+ publish_arrow_17_prebuilt_marker "${TP_INSTALL_DIR}"
+}
+
+build_arrow() {
+ local install_dir
+ install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")"
+ invalidate_arrow_prebuilt_marker "${TP_INSTALL_DIR}"
+ clean_arrow_artifacts_in "${install_dir}"
+ build_arrow_stack "${ARROW_SOURCE}" "${XSIMD_NAME}" "${install_dir}" true
publish_arrow_prebuilt_marker "${TP_INSTALL_DIR}"
}
@@ -2289,12 +2318,18 @@ build_pugixml() {
cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp"
"${TP_INSTALL_DIR}/include/"
}
-# paimon-cpp
-build_paimon_cpp() {
- check_if_source_exist "${PAIMON_CPP_SOURCE}"
- require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}"
- invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}"
- cd "${TP_SOURCE_DIR}/${PAIMON_CPP_SOURCE}"
+# Build each Paimon variant against the matching Arrow prefix and install it
+# beside that Arrow version. Arrow types cross Paimon's public C++ boundary, so
+# mixing the two versions is not ABI-safe.
+build_paimon_cpp_stack() {
+ local paimon_source="$1"
+ local arrow_install_dir="$2"
+ local install_dir="$3"
+
+ check_if_source_exist "${paimon_source}"
+ mkdir -p "${install_dir}/lib64"
+ ln -sfn lib64 "${install_dir}/lib"
+ cd "${TP_SOURCE_DIR}/${paimon_source}"
rm -rf "${BUILD_DIR}"
mkdir -p "${BUILD_DIR}"
@@ -2306,12 +2341,13 @@ build_paimon_cpp() {
paimon_linker_flags="${paimon_linker_flags} -lunwind"
fi
+ PAIMON_ARROW_INSTALL_DIR="${arrow_install_dir}" \
CXXFLAGS="-Wno-nontrivial-memcall" \
"${CMAKE_CMD}" -C "${TP_DIR}/paimon-cpp-cache.cmake" \
-G "${GENERATOR}" \
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
-DCMAKE_CXX_STANDARD="${TP_CXX_STANDARD}" \
- -DCMAKE_INSTALL_PREFIX="${TP_INSTALL_DIR}" \
+ -DCMAKE_INSTALL_PREFIX="${install_dir}" \
-DPAIMON_BUILD_SHARED=OFF \
-DPAIMON_BUILD_STATIC=ON \
-DPAIMON_BUILD_TESTS=OFF \
@@ -2335,7 +2371,7 @@ build_paimon_cpp() {
# reuses Doris's Arrow and does NOT build arrow_ep, so the paimon_deps
# directory is not needed. When building its own Arrow (legacy), copy
# arrow artefacts into an isolated directory to avoid clashing with Doris.
- local paimon_deps_dir="${TP_INSTALL_DIR}/paimon-cpp/lib64/paimon_deps"
+ local paimon_deps_dir="${install_dir}/paimon-cpp/lib64/paimon_deps"
if [ -d "arrow_ep-install/lib" ]; then
mkdir -p "${paimon_deps_dir}"
for paimon_arrow_dep in \
@@ -2355,25 +2391,42 @@ build_paimon_cpp() {
# Install roaring_bitmap, renamed to avoid conflict with Doris's
croaringbitmap
if [ -f "release/libroaring_bitmap.a" ]; then
- cp -v "release/libroaring_bitmap.a"
"${TP_INSTALL_DIR}/lib64/libroaring_bitmap_paimon.a"
+ cp -v "release/libroaring_bitmap.a"
"${install_dir}/lib64/libroaring_bitmap_paimon.a"
fi
# Install xxhash, renamed to avoid conflict with Doris's xxhash
if [ -f "release/libxxhash.a" ]; then
- cp -v "release/libxxhash.a"
"${TP_INSTALL_DIR}/lib64/libxxhash_paimon.a"
+ cp -v "release/libxxhash.a" "${install_dir}/lib64/libxxhash_paimon.a"
fi
# Install fmt v11 (from fmt_ep-install directory, renamed to avoid
conflict with Doris's fmt v7)
if [ -f "fmt_ep-install/lib/libfmt.a" ]; then
- cp -v "fmt_ep-install/lib/libfmt.a"
"${TP_INSTALL_DIR}/lib64/libfmt_paimon.a"
+ cp -v "fmt_ep-install/lib/libfmt.a"
"${install_dir}/lib64/libfmt_paimon.a"
fi
# Install tbb (from tbb_ep-install directory, renamed to avoid conflict
with Doris's tbb)
if [ -f "tbb_ep-install/lib/libtbb.a" ]; then
- cp -v "tbb_ep-install/lib/libtbb.a"
"${TP_INSTALL_DIR}/lib64/libtbb_paimon.a"
+ cp -v "tbb_ep-install/lib/libtbb.a"
"${install_dir}/lib64/libtbb_paimon.a"
fi
echo "Paimon-cpp internal dependencies installed successfully"
+}
+
+build_paimon_cpp_17() {
+ require_arrow_17_prebuilt_for_paimon "${TP_INSTALL_DIR}"
+ invalidate_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}"
+ clean_paimon_artifacts_in "${TP_INSTALL_DIR}"
+ build_paimon_cpp_stack "${PAIMON_CPP_17_SOURCE}" "${TP_INSTALL_DIR}"
"${TP_INSTALL_DIR}"
+ publish_paimon_17_prebuilt_marker "${TP_INSTALL_DIR}"
+}
+
+build_paimon_cpp() {
+ local install_dir
+ install_dir="$(arrow_install_dir "${TP_INSTALL_DIR}")"
+ require_arrow_prebuilt_for_paimon "${TP_INSTALL_DIR}"
+ invalidate_paimon_prebuilt_marker "${TP_INSTALL_DIR}"
+ clean_paimon_artifacts_in "${install_dir}"
+ build_paimon_cpp_stack "${PAIMON_CPP_SOURCE}" "${install_dir}"
"${install_dir}"
publish_paimon_prebuilt_marker "${TP_INSTALL_DIR}"
}
@@ -2478,6 +2531,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
orc
cares
grpc # after cares, protobuf
+ arrow_17
arrow
arrow_adbc
lance_c
@@ -2515,6 +2569,7 @@ if [[ "${#packages[@]}" -eq 0 ]]; then
brotli
icu
pugixml
+ paimon_cpp_17
paimon_cpp
)
if [[ "$(uname -s)" == 'Darwin' ]]; then
@@ -2570,6 +2625,7 @@ cleanup_package_source() {
cyrus_sasl) src_var="CYRUS_SASL_SOURCE" ;;
librdkafka) src_var="LIBRDKAFKA_SOURCE" ;;
flatbuffers) src_var="FLATBUFFERS_SOURCE" ;;
+ arrow_17) src_var="ARROW_17_SOURCE" ;;
arrow) src_var="ARROW_SOURCE" ;;
arrow_adbc)
# arrow_adbc also unpacks the prebuilt flightsql driver, clean both
@@ -2622,6 +2678,7 @@ cleanup_package_source() {
jindofs) src_var="JINDOFS_SOURCE" ;;
juicefs) src_var="JUICEFS_SOURCE" ;;
pugixml) src_var="PUGIXML_SOURCE" ;;
+ paimon_cpp_17) src_var="PAIMON_CPP_17_SOURCE" ;;
paimon_cpp) src_var="PAIMON_CPP_SOURCE" ;;
lance_c) src_var="LANCE_C_SOURCE" ;;
aws_sdk) src_var="AWS_SDK_SOURCE" ;;
diff --git a/thirdparty/download-thirdparty.sh
b/thirdparty/download-thirdparty.sh
index 8d65b8ce275..5af7fdcc3e5 100755
--- a/thirdparty/download-thirdparty.sh
+++ b/thirdparty/download-thirdparty.sh
@@ -271,13 +271,24 @@ for TP_ARCH in "${TP_ARCHIVES[@]}"; do
fi
NAME="${TP_ARCH}_NAME"
SOURCE="${TP_ARCH}_SOURCE"
+ ARCHIVE_SOURCE_VAR="${TP_ARCH}_ARCHIVE_SOURCE"
+ ARCHIVE_SOURCE="${!ARCHIVE_SOURCE_VAR}"
if [[ -z "${!SOURCE}" ]]; then
continue
fi
if [[ ! -d "${TP_SOURCE_DIR}/${!SOURCE}" ]]; then
- if [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then
+ if [[ -n "${ARCHIVE_SOURCE}" && "${ARCHIVE_SOURCE}" != "${!SOURCE}"
]]; then
+ alias_unpack_dir="$(mktemp -d
"${TP_SOURCE_DIR}/.unpack-${TP_ARCH}.XXXXXX")"
+ if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C
"${alias_unpack_dir}"; then
+ echo "Failed to untar ${!NAME} for ${!SOURCE}"
+ rm -rf "${alias_unpack_dir}"
+ exit 1
+ fi
+ mv "${alias_unpack_dir}/${ARCHIVE_SOURCE}"
"${TP_SOURCE_DIR}/${!SOURCE}"
+ rm -rf "${alias_unpack_dir}"
+ elif [[ "${!NAME}" =~ ${SUFFIX_TGZ} ]]; then
echo "${TP_SOURCE_DIR}/${!NAME}"
echo "${TP_SOURCE_DIR}/${!SOURCE}"
if ! "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${!NAME}" -C
"${TP_SOURCE_DIR}/"; then
@@ -369,17 +380,31 @@ echo "===== Patching thirdparty archives..."
PATCHED_MARK="patched_mark"
ARROW_PAIMON_PATCH_FINGERPRINT_MARK="patched_mark_arrow_paimon_fingerprint"
ARROW_PAIMON_BUILD_FINGERPRINT=""
+ARROW_PAIMON_17_BUILD_FINGERPRINT=""
if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ||
" ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then
ARROW_PAIMON_BUILD_FINGERPRINT="$(arrow_paimon_build_fingerprint)"
fi
+if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " ||
+ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then
+ ARROW_PAIMON_17_BUILD_FINGERPRINT="$(arrow_paimon_17_build_fingerprint)"
+fi
reset_arrow_paimon_source() {
local archive_name="$1"
local source_name="$2"
+ local archive_source="${3:-${source_name}}"
echo "Resetting ${source_name} because its patch state is incomplete or
stale"
rm -rf "${TP_SOURCE_DIR:?}/${source_name}"
- "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C "${TP_SOURCE_DIR}/"
+ if [[ "${archive_source}" == "${source_name}" ]]; then
+ "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C
"${TP_SOURCE_DIR}/"
+ else
+ local alias_unpack_dir
+ alias_unpack_dir="$(mktemp -d
"${TP_SOURCE_DIR}/.reset-${source_name}.XXXXXX")"
+ "${TAR_CMD}" xzf "${TP_SOURCE_DIR}/${archive_name}" -C
"${alias_unpack_dir}"
+ mv "${alias_unpack_dir}/${archive_source}"
"${TP_SOURCE_DIR}/${source_name}"
+ rm -rf "${alias_unpack_dir}"
+ fi
}
# glog patch
@@ -475,6 +500,26 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " ROCKSDB " ]]; then
echo "Finished patching ${ROCKSDB_SOURCE}"
fi
+# Keep the Arrow 17 source used by pre-upgrade branch-4.1 revisions
independently
+# patched from the Arrow 24 source selected by master.
+if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW_17 " ]]; then
+
arrow_17_fingerprint_mark="${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
+ if ! [[ -f "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}/${PATCHED_MARK}" &&
+ -f "${arrow_17_fingerprint_mark}" ]] ||
+ [[ "$(<"${arrow_17_fingerprint_mark}")" !=
"${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then
+ reset_arrow_paimon_source "${ARROW_17_NAME}" "${ARROW_17_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${ARROW_17_SOURCE}"
+ patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-paimon.patch"
+ patch -p1
<"${TP_PATCH_DIR}/apache-arrow-17.0.0-force-write-int96-timestamps.patch"
+ patch -p1 <"${TP_PATCH_DIR}/apache-arrow-17.0.0-lzo.patch"
+ touch "${PATCHED_MARK}"
+ printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \
+ >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
+ cd -
+ fi
+ echo "Finished patching ${ARROW_17_SOURCE}"
+fi
+
# arrow patch is used to get the raw orc reader for filter prune.
if [[ " ${TP_ARCHIVES[*]} " =~ " ARROW " ]]; then
if [[ "${ARROW_SOURCE}" == "arrow-apache-arrow-13.0.0" ]]; then
@@ -794,6 +839,24 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then
echo "Finished patching ${AZURE_SOURCE}"
fi
+# Keep the Arrow 17 Paimon source free of the Arrow 24 API and Compute patches.
+if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP_17 " ]]; then
+
paimon_17_fingerprint_mark="${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
+ if ! [[ -f "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}/${PATCHED_MARK}" &&
+ -f "${paimon_17_fingerprint_mark}" ]] ||
+ [[ "$(<"${paimon_17_fingerprint_mark}")" !=
"${ARROW_PAIMON_17_BUILD_FINGERPRINT}" ]]; then
+ reset_arrow_paimon_source "${PAIMON_CPP_17_NAME}"
"${PAIMON_CPP_17_SOURCE}" \
+ "${PAIMON_CPP_17_ARCHIVE_SOURCE}"
+ cd "${TP_SOURCE_DIR}/${PAIMON_CPP_17_SOURCE}"
+ patch -p1 <"${TP_PATCH_DIR}/paimon-cpp-buildutils-static-deps.patch"
+ touch "${PATCHED_MARK}"
+ printf '%s\n' "${ARROW_PAIMON_17_BUILD_FINGERPRINT}" \
+ >"${ARROW_PAIMON_PATCH_FINGERPRINT_MARK}"
+ cd -
+ fi
+ echo "Finished patching ${PAIMON_CPP_17_SOURCE}"
+fi
+
# patch paimon-cpp
if [[ " ${TP_ARCHIVES[*]} " =~ " PAIMON_CPP " ]]; then
PAIMON_CPP_ARROW_24_PATCHED_MARK="patched_mark_arrow_24"
diff --git a/thirdparty/paimon-cpp-cache.cmake
b/thirdparty/paimon-cpp-cache.cmake
index 6ab6049c59b..a12540e5d3e 100644
--- a/thirdparty/paimon-cpp-cache.cmake
+++ b/thirdparty/paimon-cpp-cache.cmake
@@ -21,12 +21,17 @@
# Get the Doris thirdparty installation directory from environment
set(DORIS_THIRDPARTY_DIR "$ENV{TP_INSTALL_DIR}" CACHE PATH "Doris thirdparty
install directory")
+set(DORIS_ARROW_DIR "$ENV{PAIMON_ARROW_INSTALL_DIR}" CACHE PATH "Selected
Doris Arrow install directory")
if(NOT DORIS_THIRDPARTY_DIR)
message(FATAL_ERROR "TP_INSTALL_DIR environment variable must be set")
endif()
+if(NOT DORIS_ARROW_DIR)
+ message(FATAL_ERROR "PAIMON_ARROW_INSTALL_DIR environment variable must be
set")
+endif()
message(STATUS "Using Doris thirdparty libraries from:
${DORIS_THIRDPARTY_DIR}")
+message(STATUS "Using selected Arrow libraries from: ${DORIS_ARROW_DIR}")
# Set CMAKE_PREFIX_PATH to help find_package locate our libraries
set(CMAKE_PREFIX_PATH "${DORIS_THIRDPARTY_DIR};${CMAKE_PREFIX_PATH}" CACHE
STRING "Search path for find_package")
@@ -64,21 +69,21 @@ set(LZ4_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "LZ4
include directory")
# ============================================================================
# ============================================================================
-# Arrow - Reuse from Doris (Doris Arrow now includes
COMPUTE/DATASET/ACERO/FILESYSTEM)
-# Doris's Arrow 24.0.0 is built with the full module set that paimon-cpp
-# needs, so we skip paimon-cpp's internal externalproject_add(arrow_ep ...).
+# Arrow - Reuse the version selected by the caller. Both installed Arrow stacks
+# include DATASET/ACERO/FILESYSTEM; Arrow 24 also has a separate Compute
archive.
# ============================================================================
set(PAIMON_USE_EXTERNAL_ARROW ON CACHE BOOL "Use pre-built Arrow from Doris
instead of building from source")
-set(DORIS_LIB64_DIR "${DORIS_THIRDPARTY_DIR}/lib64" CACHE PATH "Doris lib64
directory")
+set(DORIS_ARROW_LIB64_DIR "${DORIS_ARROW_DIR}/lib64" CACHE PATH "Selected
Arrow lib64 directory")
+set(DORIS_ARROW_INCLUDE_DIR "${DORIS_ARROW_DIR}/include" CACHE PATH "Selected
Arrow include directory")
-set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_INCLUDE_DIR}" CACHE PATH "Arrow
include directory")
-set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_LIB64_DIR}/libarrow.a" CACHE FILEPATH
"Arrow core library")
-set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB "${DORIS_LIB64_DIR}/libarrow_compute.a"
CACHE FILEPATH "Arrow Compute library")
-set(PAIMON_EXTERNAL_ARROW_DATASET_LIB "${DORIS_LIB64_DIR}/libarrow_dataset.a"
CACHE FILEPATH "Arrow Dataset library")
-set(PAIMON_EXTERNAL_ARROW_ACERO_LIB "${DORIS_LIB64_DIR}/libarrow_acero.a"
CACHE FILEPATH "Arrow Acero library")
-set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_LIB64_DIR}/libparquet.a" CACHE
FILEPATH "Parquet library")
-set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB
"${DORIS_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH "Arrow
bundled dependencies library")
+set(PAIMON_EXTERNAL_ARROW_INCLUDE_DIR "${DORIS_ARROW_INCLUDE_DIR}" CACHE PATH
"Arrow include directory")
+set(PAIMON_EXTERNAL_ARROW_LIB "${DORIS_ARROW_LIB64_DIR}/libarrow.a" CACHE
FILEPATH "Arrow core library")
+set(PAIMON_EXTERNAL_ARROW_COMPUTE_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_compute.a" CACHE FILEPATH "Arrow Compute
library")
+set(PAIMON_EXTERNAL_ARROW_DATASET_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_dataset.a" CACHE FILEPATH "Arrow Dataset
library")
+set(PAIMON_EXTERNAL_ARROW_ACERO_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_acero.a" CACHE FILEPATH "Arrow Acero
library")
+set(PAIMON_EXTERNAL_PARQUET_LIB "${DORIS_ARROW_LIB64_DIR}/libparquet.a" CACHE
FILEPATH "Parquet library")
+set(PAIMON_EXTERNAL_ARROW_BUNDLED_DEPS_LIB
"${DORIS_ARROW_LIB64_DIR}/libarrow_bundled_dependencies.a" CACHE FILEPATH
"Arrow bundled dependencies library")
# Protobuf, Thrift - still built separately by paimon-cpp
diff --git
a/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch
b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch
new file mode 100644
index 00000000000..5a754247566
--- /dev/null
+++ b/thirdparty/patches/apache-arrow-17.0.0-force-write-int96-timestamps.patch
@@ -0,0 +1,98 @@
+diff -ruN
arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc
arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc
+--- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/arrow/schema.cc
2026-03-27 01:23:23.651831424 +0800
++++ arrow-apache-arrow-17.0.0/cpp/src/parquet/arrow/schema.cc 2026-03-27
01:28:36.855281965 +0800
+@@ -178,7 +178,8 @@
+
+ // The user is explicitly asking for Impala int96 encoding, there is no
+ // logical type.
+- if (arrow_properties.support_deprecated_int96_timestamps() && target_unit
== ::arrow::TimeUnit::NANO) {
++ if (arrow_properties.force_write_int96_timestamps() ||
++ (arrow_properties.support_deprecated_int96_timestamps() && target_unit
== ::arrow::TimeUnit::NANO)) {
+ *physical_type = ParquetType::INT96;
+ return Status::OK();
+ }
+diff -ruN arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h
arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h
+--- arrow-apache-arrow-17.0.0-after-paimon/cpp/src/parquet/properties.h
2026-03-27 01:23:23.643831362 +0800
++++ arrow-apache-arrow-17.0.0/cpp/src/parquet/properties.h 2026-03-27
01:27:47.717897537 +0800
+@@ -980,6 +980,7 @@
+ public:
+ Builder()
+ : write_timestamps_as_int96_(false),
++ force_write_int96_timestamps_(false),
+ coerce_timestamps_enabled_(false),
+ coerce_timestamps_unit_(::arrow::TimeUnit::SECOND),
+ truncated_timestamps_allowed_(false),
+@@ -1005,6 +1006,21 @@
+ return this;
+ }
+
++ /// \brief Force writing legacy int96 timestamps.
++ ///
++ /// This bypasses unit-based guards and writes INT96 whenever timestamp
++ /// metadata is resolved.
++ Builder* enable_force_write_int96_timestamps() {
++ force_write_int96_timestamps_ = true;
++ return this;
++ }
++
++ /// \brief Disable forcing legacy int96 timestamps (default).
++ Builder* disable_force_write_int96_timestamps() {
++ force_write_int96_timestamps_ = false;
++ return this;
++ }
++
+ /// \brief Coerce all timestamps to the specified time unit.
+ /// \param unit time unit to truncate to.
+ /// For Parquet versions 1.0 and 2.4, nanoseconds are casted to
microseconds.
+@@ -1085,7 +1101,8 @@
+ /// Create the final properties.
+ std::shared_ptr<ArrowWriterProperties> build() {
+ return std::shared_ptr<ArrowWriterProperties>(new ArrowWriterProperties(
+- write_timestamps_as_int96_, coerce_timestamps_enabled_,
coerce_timestamps_unit_,
++ write_timestamps_as_int96_, force_write_int96_timestamps_,
++ coerce_timestamps_enabled_, coerce_timestamps_unit_,
+ truncated_timestamps_allowed_, store_schema_,
compliant_nested_types_,
+ engine_version_, use_threads_, executor_));
+ }
+@@ -1093,6 +1110,8 @@
+ private:
+ bool write_timestamps_as_int96_;
+
++ bool force_write_int96_timestamps_;
++
+ bool coerce_timestamps_enabled_;
+ ::arrow::TimeUnit::type coerce_timestamps_unit_;
+ bool truncated_timestamps_allowed_;
+@@ -1107,6 +1126,8 @@
+
+ bool support_deprecated_int96_timestamps() const { return
write_timestamps_as_int96_; }
+
++ bool force_write_int96_timestamps() const { return
force_write_int96_timestamps_; }
++
+ bool coerce_timestamps_enabled() const { return coerce_timestamps_enabled_;
}
+ ::arrow::TimeUnit::type coerce_timestamps_unit() const {
+ return coerce_timestamps_unit_;
+@@ -1138,6 +1159,7 @@
+
+ private:
+ explicit ArrowWriterProperties(bool write_nanos_as_int96,
++ bool force_write_int96_timestamps,
+ bool coerce_timestamps_enabled,
+ ::arrow::TimeUnit::type
coerce_timestamps_unit,
+ bool truncated_timestamps_allowed, bool
store_schema,
+@@ -1145,6 +1167,7 @@
+ EngineVersion engine_version, bool
use_threads,
+ ::arrow::internal::Executor* executor)
+ : write_timestamps_as_int96_(write_nanos_as_int96),
++ force_write_int96_timestamps_(force_write_int96_timestamps),
+ coerce_timestamps_enabled_(coerce_timestamps_enabled),
+ coerce_timestamps_unit_(coerce_timestamps_unit),
+ truncated_timestamps_allowed_(truncated_timestamps_allowed),
+@@ -1155,6 +1178,7 @@
+ executor_(executor) {}
+
+ const bool write_timestamps_as_int96_;
++ const bool force_write_int96_timestamps_;
+ const bool coerce_timestamps_enabled_;
+ const ::arrow::TimeUnit::type coerce_timestamps_unit_;
+ const bool truncated_timestamps_allowed_;
diff --git a/thirdparty/patches/apache-arrow-17.0.0-lzo.patch
b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch
new file mode 100644
index 00000000000..a983818413a
--- /dev/null
+++ b/thirdparty/patches/apache-arrow-17.0.0-lzo.patch
@@ -0,0 +1,84 @@
+--- a/cpp/src/parquet/column_reader.cc
++++ b/cpp/src/parquet/column_reader.cc
+@@ -30,0 +31,2 @@
++
++#include <lzo/lzo1x.h>
+@@ -268,0 +269 @@
++ compression_codec_(codec),
+@@ -279 +282,7 @@
+- decompressor_ = GetCodec(codec);
++ if (compression_codec_ == Compression::LZO) {
++ if (lzo_init() != LZO_E_OK) {
++ throw ParquetException("Failed to initialize LZO codec");
++ }
++ } else {
++ decompressor_ = GetCodec(codec);
++ }
+@@ -315,0 +325 @@
++ Compression::type compression_codec_;
+@@ -585 +595 @@
+- if (decompressor_ == nullptr) {
++ if (decompressor_ == nullptr && compression_codec_ != Compression::LZO) {
+@@ -601,0 +612,61 @@
++ if (compression_codec_ == Compression::LZO) {
++ const uint8_t* input = page_buffer->data() + levels_byte_len;
++ const uint8_t* const input_end = page_buffer->data() + compressed_len;
++ uint8_t* output = decompression_buffer_->mutable_data() + levels_byte_len;
++ uint8_t* const output_end = decompression_buffer_->mutable_data() +
uncompressed_len;
++
++ auto load_big_endian_u32 = [](const uint8_t* data) {
++ return (static_cast<uint32_t>(data[0]) << 24) |
++ (static_cast<uint32_t>(data[1]) << 16) |
++ (static_cast<uint32_t>(data[2]) << 8) |
static_cast<uint32_t>(data[3]);
++ };
++
++ while (input < input_end) {
++ if (input_end - input < 4) {
++ throw ParquetException("LZO page decompression failed: truncated
large block length");
++ }
++
++ uint32_t large_block_uncompressed_len = load_big_endian_u32(input);
++ input += 4;
++ if (static_cast<size_t>(output_end - output) <
large_block_uncompressed_len) {
++ throw ParquetException("LZO page decompression failed: output buffer
too small");
++ }
++
++ while (large_block_uncompressed_len > 0) {
++ if (input_end - input < 4) {
++ throw ParquetException("LZO page decompression failed: truncated
small block length");
++ }
++
++ uint32_t small_block_compressed_len = load_big_endian_u32(input);
++ input += 4;
++ if (static_cast<size_t>(input_end - input) <
small_block_compressed_len) {
++ throw ParquetException("LZO page decompression failed: truncated
small block data");
++ }
++
++ auto small_block_uncompressed_len =
++ static_cast<lzo_uint>(large_block_uncompressed_len);
++ const int result =
++ lzo1x_decompress_safe(input,
static_cast<lzo_uint>(small_block_compressed_len),
++ output, &small_block_uncompressed_len,
nullptr);
++ if (result != LZO_E_OK) {
++ throw ParquetException("LZO page decompression failed, error: " +
++ std::to_string(result));
++ }
++ if (small_block_uncompressed_len > large_block_uncompressed_len) {
++ throw ParquetException("LZO page decompression failed: invalid
small block size");
++ }
++
++ input += small_block_compressed_len;
++ output += small_block_uncompressed_len;
++ large_block_uncompressed_len -= small_block_uncompressed_len;
++ }
++ }
++ if (output != output_end) {
++ throw ParquetException("Page didn't decompress to expected size,
expected: " +
++ std::to_string(uncompressed_len -
levels_byte_len) + ", but got:" +
++ std::to_string(output -
(decompression_buffer_->mutable_data() +
++ levels_byte_len)));
++ }
++
++ return decompression_buffer_;
++ }
++
diff --git a/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
new file mode 100644
index 00000000000..4e53117b79b
--- /dev/null
+++ b/thirdparty/patches/apache-arrow-17.0.0-paimon.patch
@@ -0,0 +1,224 @@
+diff --git a/cpp/src/parquet/arrow/schema.cc b/cpp/src/parquet/arrow/schema.cc
+index ec3890a41f..943f69bb6c 100644
+--- a/cpp/src/parquet/arrow/schema.cc
++++ b/cpp/src/parquet/arrow/schema.cc
+@@ -178,7 +178,7 @@ static Status GetTimestampMetadata(const
::arrow::TimestampType& type,
+
+ // The user is explicitly asking for Impala int96 encoding, there is no
+ // logical type.
+- if (arrow_properties.support_deprecated_int96_timestamps()) {
++ if (arrow_properties.support_deprecated_int96_timestamps() && target_unit
== ::arrow::TimeUnit::NANO) {
+ *physical_type = ParquetType::INT96;
+ return Status::OK();
+ }
+
+diff --git a/cpp/src/parquet/arrow/reader.cc b/cpp/src/parquet/arrow/reader.cc
+index 285e2a5973..aa6f92f077 100644
+--- a/cpp/src/parquet/arrow/reader.cc
++++ b/cpp/src/parquet/arrow/reader.cc
+@@ -1013,25 +1013,32 @@ Status FileReaderImpl::GetRecordBatchReader(const
std::vector<int>& row_groups,
+ return Status::OK();
+ }
+
+- int64_t num_rows = 0;
++ std::vector<int64_t> num_rows;
+ for (int row_group : row_groups) {
+- num_rows += parquet_reader()->metadata()->RowGroup(row_group)->num_rows();
++
num_rows.push_back(parquet_reader()->metadata()->RowGroup(row_group)->num_rows());
+ }
+
+ using ::arrow::RecordBatchIterator;
++ int row_group_idx = 0;
+
+ // NB: This lambda will be invoked outside the scope of this call to
+ // `GetRecordBatchReader()`, so it must capture `readers` and
`batch_schema` by value.
+ // `this` is a non-owning pointer so we are relying on the parent
FileReader outliving
+ // this RecordBatchReader.
+ ::arrow::Iterator<RecordBatchIterator> batches =
::arrow::MakeFunctionIterator(
+- [readers, batch_schema, num_rows,
++ [readers, batch_schema, num_rows, row_group_idx,
+ this]() mutable -> ::arrow::Result<RecordBatchIterator> {
+ ::arrow::ChunkedArrayVector columns(readers.size());
+
+- // don't reserve more rows than necessary
+- int64_t batch_size = std::min(properties().batch_size(), num_rows);
+- num_rows -= batch_size;
++ int64_t batch_size = 0;
++ if (!num_rows.empty()) {
++ // don't reserve more rows than necessary
++ batch_size = std::min(properties().batch_size(),
num_rows[row_group_idx]);
++ num_rows[row_group_idx] -= batch_size;
++ if (num_rows[row_group_idx] == 0 && (num_rows.size() - 1) !=
row_group_idx) {
++ row_group_idx++;
++ }
++ }
+
+ RETURN_NOT_OK(::arrow::internal::OptionalParallelFor(
+ reader_properties_.use_threads(),
static_cast<int>(readers.size()),
+diff --git a/cpp/src/parquet/arrow/writer.cc b/cpp/src/parquet/arrow/writer.cc
+index 4fd7ef1b47..87326a54f1 100644
+--- a/cpp/src/parquet/arrow/writer.cc
++++ b/cpp/src/parquet/arrow/writer.cc
+@@ -314,6 +314,14 @@ class FileWriterImpl : public FileWriter {
+ return Status::OK();
+ }
+
++ int64_t GetBufferedSize() override {
++ if (row_group_writer_ == nullptr) {
++ return 0;
++ }
++ return row_group_writer_->total_compressed_bytes() +
++ row_group_writer_->total_compressed_bytes_written();
++ }
++
+ Status Close() override {
+ if (!closed_) {
+ // Make idempotent
+@@ -418,10 +426,13 @@ class FileWriterImpl : public FileWriter {
+
+ // Max number of rows allowed in a row group.
+ const int64_t max_row_group_length =
this->properties().max_row_group_length();
++ const int64_t max_row_group_size =
this->properties().max_row_group_size();
+
+ // Initialize a new buffered row group writer if necessary.
+ if (row_group_writer_ == nullptr || !row_group_writer_->buffered() ||
+- row_group_writer_->num_rows() >= max_row_group_length) {
++ row_group_writer_->num_rows() >= max_row_group_length ||
++ (row_group_writer_->total_compressed_bytes_written() +
++ row_group_writer_->total_compressed_bytes() >= max_row_group_size)) {
+ RETURN_NOT_OK(NewBufferedRowGroup());
+ }
+
+diff --git a/cpp/src/parquet/arrow/writer.h b/cpp/src/parquet/arrow/writer.h
+index 4a1a033a7b..0f13d05e44 100644
+--- a/cpp/src/parquet/arrow/writer.h
++++ b/cpp/src/parquet/arrow/writer.h
+@@ -138,6 +138,9 @@ class PARQUET_EXPORT FileWriter {
+ /// option in this case.
+ virtual ::arrow::Status WriteRecordBatch(const ::arrow::RecordBatch& batch)
= 0;
+
++ /// \brief Return the buffered size in bytes.
++ virtual int64_t GetBufferedSize() = 0;
++
+ /// \brief Write the footer and close the file.
+ virtual ::arrow::Status Close() = 0;
+ virtual ~FileWriter();
+diff --git a/cpp/src/parquet/properties.h b/cpp/src/parquet/properties.h
+index 4d3acb491e..3906ff3c59 100644
+--- a/cpp/src/parquet/properties.h
++++ b/cpp/src/parquet/properties.h
+@@ -139,6 +139,7 @@ static constexpr bool DEFAULT_IS_DICTIONARY_ENABLED = true;
+ static constexpr int64_t DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT =
kDefaultDataPageSize;
+ static constexpr int64_t DEFAULT_WRITE_BATCH_SIZE = 1024;
+ static constexpr int64_t DEFAULT_MAX_ROW_GROUP_LENGTH = 1024 * 1024;
++static constexpr int64_t DEFAULT_MAX_ROW_GROUP_SIZE = 128 * 1024 * 1024;
+ static constexpr bool DEFAULT_ARE_STATISTICS_ENABLED = true;
+ static constexpr int64_t DEFAULT_MAX_STATISTICS_SIZE = 4096;
+ static constexpr Encoding::type DEFAULT_ENCODING = Encoding::UNKNOWN;
+@@ -232,6 +233,7 @@ class PARQUET_EXPORT WriterProperties {
+ dictionary_pagesize_limit_(DEFAULT_DICTIONARY_PAGE_SIZE_LIMIT),
+ write_batch_size_(DEFAULT_WRITE_BATCH_SIZE),
+ max_row_group_length_(DEFAULT_MAX_ROW_GROUP_LENGTH),
++ max_row_group_size_(DEFAULT_MAX_ROW_GROUP_SIZE),
+ pagesize_(kDefaultDataPageSize),
+ version_(ParquetVersion::PARQUET_2_6),
+ data_page_version_(ParquetDataPageVersion::V1),
+@@ -244,6 +246,7 @@ class PARQUET_EXPORT WriterProperties {
+ dictionary_pagesize_limit_(properties.dictionary_pagesize_limit()),
+ write_batch_size_(properties.write_batch_size()),
+ max_row_group_length_(properties.max_row_group_length()),
++ max_row_group_size_(properties.max_row_group_size()),
+ pagesize_(properties.data_pagesize()),
+ version_(properties.version()),
+ data_page_version_(properties.data_page_version()),
+@@ -321,6 +324,13 @@ class PARQUET_EXPORT WriterProperties {
+ return this;
+ }
+
++ /// Specify the max bytes size to put in a single row group.
++ /// Default 128 M.
++ Builder* max_row_group_size(int64_t max_row_group_size) {
++ max_row_group_size_ = max_row_group_size;
++ return this;
++ }
++
+ /// Specify the data page size.
+ /// Default 1MB.
+ Builder* data_pagesize(int64_t pg_size) {
+@@ -664,7 +674,7 @@ class PARQUET_EXPORT WriterProperties {
+
+ return std::shared_ptr<WriterProperties>(new WriterProperties(
+ pool_, dictionary_pagesize_limit_, write_batch_size_,
max_row_group_length_,
+- pagesize_, version_, created_by_, page_checksum_enabled_,
++ max_row_group_size_, pagesize_, version_, created_by_,
page_checksum_enabled_,
+ std::move(file_encryption_properties_), default_column_properties_,
+ column_properties, data_page_version_, store_decimal_as_integer_,
+ std::move(sorting_columns_)));
+@@ -675,6 +685,7 @@ class PARQUET_EXPORT WriterProperties {
+ int64_t dictionary_pagesize_limit_;
+ int64_t write_batch_size_;
+ int64_t max_row_group_length_;
++ int64_t max_row_group_size_;
+ int64_t pagesize_;
+ ParquetVersion::type version_;
+ ParquetDataPageVersion data_page_version_;
+@@ -705,6 +716,8 @@ class PARQUET_EXPORT WriterProperties {
+
+ inline int64_t max_row_group_length() const { return max_row_group_length_;
}
+
++ inline int64_t max_row_group_size() const { return max_row_group_size_; }
++
+ inline int64_t data_pagesize() const { return pagesize_; }
+
+ inline ParquetDataPageVersion data_page_version() const {
+@@ -810,7 +823,7 @@ class PARQUET_EXPORT WriterProperties {
+ private:
+ explicit WriterProperties(
+ MemoryPool* pool, int64_t dictionary_pagesize_limit, int64_t
write_batch_size,
+- int64_t max_row_group_length, int64_t pagesize, ParquetVersion::type
version,
++ int64_t max_row_group_length, int64_t max_row_group_size, int64_t
pagesize, ParquetVersion::type version,
+ const std::string& created_by, bool page_write_checksum_enabled,
+ std::shared_ptr<FileEncryptionProperties> file_encryption_properties,
+ const ColumnProperties& default_column_properties,
+@@ -821,6 +834,7 @@ class PARQUET_EXPORT WriterProperties {
+ dictionary_pagesize_limit_(dictionary_pagesize_limit),
+ write_batch_size_(write_batch_size),
+ max_row_group_length_(max_row_group_length),
++ max_row_group_size_(max_row_group_size),
+ pagesize_(pagesize),
+ parquet_data_page_version_(data_page_version),
+ parquet_version_(version),
+@@ -836,6 +850,7 @@ class PARQUET_EXPORT WriterProperties {
+ int64_t dictionary_pagesize_limit_;
+ int64_t write_batch_size_;
+ int64_t max_row_group_length_;
++ int64_t max_row_group_size_;
+ int64_t pagesize_;
+ ParquetDataPageVersion parquet_data_page_version_;
+ ParquetVersion::type parquet_version_;
+diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake
b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+index 9df922afa2..5c8b3d4d07 100644
+--- a/cpp/cmake_modules/ThirdpartyToolchain.cmake
++++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake
+@@ -1789,7 +1789,20 @@ if(ARROW_WITH_THRIFT)
+ REQUIRED_VERSION
+ 0.11.0)
+
+- string(REPLACE "." ";" Thrift_VERSION_LIST ${Thrift_VERSION})
++ if(NOT Thrift_VERSION)
++ if(DEFINED thrift_PC_VERSION AND thrift_PC_VERSION)
++ set(Thrift_VERSION "${thrift_PC_VERSION}")
++ elseif(DEFINED ThriftAlt_VERSION AND ThriftAlt_VERSION)
++ set(Thrift_VERSION "${ThriftAlt_VERSION}")
++ elseif(DEFINED THRIFT_VERSION AND THRIFT_VERSION)
++ set(Thrift_VERSION "${THRIFT_VERSION}")
++ endif()
++ endif()
++ if(NOT Thrift_VERSION)
++ message(FATAL_ERROR "Thrift_VERSION is empty after resolving Thrift
dependency")
++ endif()
++
++ string(REPLACE "." ";" Thrift_VERSION_LIST "${Thrift_VERSION}")
+ list(GET Thrift_VERSION_LIST 0 Thrift_VERSION_MAJOR)
+ list(GET Thrift_VERSION_LIST 1 Thrift_VERSION_MINOR)
+ list(GET Thrift_VERSION_LIST 2 Thrift_VERSION_PATCH)
diff --git a/thirdparty/test/arrow-paimon-lifecycle-test.sh
b/thirdparty/test/arrow-paimon-lifecycle-test.sh
index f946be5c763..5ea5ac9cf5a 100755
--- a/thirdparty/test/arrow-paimon-lifecycle-test.sh
+++ b/thirdparty/test/arrow-paimon-lifecycle-test.sh
@@ -80,8 +80,8 @@ exercise_semantic_fingerprints() {
fingerprint_from_fixture "${first_fixture}" paimon first_paimon
[[ "${first_arrow}" == "${ARROW_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}"
]] ||
fail "the Arrow legacy marker migration target is stale"
- [[ "${first_paimon}" == "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}"
]] ||
- fail "the Paimon legacy marker migration target is stale"
+ [[ "${first_paimon}" != "${PAIMON_LEGACY_COMPATIBLE_SEMANTIC_FINGERPRINT}"
]] ||
+ fail "the dual-prefix Paimon build unexpectedly accepts the legacy
root-prefix marker"
fingerprint_from_fixture "${second_fixture}" arrow second_arrow
fingerprint_from_fixture "${second_fixture}" paimon second_paimon
[[ "${first_arrow}" == "${second_arrow}" ]] ||
@@ -155,10 +155,14 @@ create_archive() {
arrow_source="arrow-apache-arrow-24.0.0"
arrow_archive="apache-arrow-24.0.0.tar.gz"
+arrow_17_source="arrow-apache-arrow-17.0.0"
+arrow_17_archive="apache-arrow-17.0.0.tar.gz"
paimon_source="doris-thirdparty-paimon-cpp-0a4f4e2"
+paimon_17_source="${paimon_source}-arrow-17"
paimon_archive="paimon-cpp-0a4f4e2.tar.gz"
create_archive "${arrow_source}" "${arrow_archive}" arrow
+create_archive "${arrow_17_source}" "${arrow_17_archive}" arrow17
create_archive "${paimon_source}" "${paimon_archive}" paimon
arrow_patches=(
@@ -166,6 +170,11 @@ arrow_patches=(
apache-arrow-24.0.0-force-write-int96-timestamps.patch
apache-arrow-24.0.0-lzo.patch
)
+arrow_17_patches=(
+ apache-arrow-17.0.0-paimon.patch
+ apache-arrow-17.0.0-force-write-int96-timestamps.patch
+ apache-arrow-17.0.0-lzo.patch
+)
paimon_patches=(
paimon-cpp-buildutils-static-deps.patch
paimon-cpp-arrow-24-compatibility.patch
@@ -174,16 +183,22 @@ paimon_patches=(
for index in 0 1 2; do
create_patch "${harness}/patches/${arrow_patches[${index}]}"
"arrow-$((index + 1)).txt"
+ create_patch "${harness}/patches/${arrow_17_patches[${index}]}"
"arrow17-$((index + 1)).txt"
create_patch "${harness}/patches/${paimon_patches[${index}]}"
"paimon-$((index + 1)).txt"
done
arrow_md5="$(md5sum "${harness}/src/${arrow_archive}" | awk '{print $1}')"
+arrow_17_md5="$(md5sum "${harness}/src/${arrow_17_archive}" | awk '{print
$1}')"
paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" | awk '{print $1}')"
{
printf 'TP_SOURCE_DIR="%s"\n' "${harness}/src"
printf 'TP_INSTALL_DIR="%s"\n' "${harness}/installed"
printf 'TP_PATCH_DIR="%s"\n' "${harness}/patches"
- printf '%s\n' 'TP_ARCHIVES=(ARROW PAIMON_CPP)'
+ printf '%s\n' 'TP_ARCHIVES=(ARROW_17 ARROW PAIMON_CPP_17 PAIMON_CPP)'
+ printf 'ARROW_17_NAME="%s"\n' "${arrow_17_archive}"
+ printf 'ARROW_17_SOURCE="%s"\n' "${arrow_17_source}"
+ printf 'ARROW_17_MD5SUM="%s"\n' "${arrow_17_md5}"
+ printf '%s\n' 'ARROW_17_DOWNLOAD="unused"'
printf 'ARROW_NAME="%s"\n' "${arrow_archive}"
printf 'ARROW_SOURCE="%s"\n' "${arrow_source}"
printf 'ARROW_MD5SUM="%s"\n' "${arrow_md5}"
@@ -192,7 +207,13 @@ paimon_md5="$(md5sum "${harness}/src/${paimon_archive}" |
awk '{print $1}')"
printf 'PAIMON_CPP_SOURCE="%s"\n' "${paimon_source}"
printf 'PAIMON_CPP_MD5SUM="%s"\n' "${paimon_md5}"
printf '%s\n' 'PAIMON_CPP_DOWNLOAD="unused"'
+ printf 'PAIMON_CPP_17_NAME="%s"\n' "${paimon_archive}"
+ printf 'PAIMON_CPP_17_ARCHIVE_SOURCE="%s"\n' "${paimon_source}"
+ printf 'PAIMON_CPP_17_SOURCE="%s"\n' "${paimon_17_source}"
+ printf 'PAIMON_CPP_17_MD5SUM="%s"\n' "${paimon_md5}"
+ printf '%s\n' 'PAIMON_CPP_17_DOWNLOAD="unused"'
printf '%s\n' 'arrow_paimon_build_fingerprint() { printf "%s\n"
test-fingerprint; }'
+ printf '%s\n' 'arrow_paimon_17_build_fingerprint() { printf "%s\n"
test-17-fingerprint; }'
} >"${harness}/vars.sh"
exercise_interrupted_patch_set() {
@@ -238,6 +259,34 @@ exercise_interrupted_patch_set ARROW "${arrow_source}"
"${arrow_archive}" arrow
exercise_interrupted_patch_set PAIMON_CPP "${paimon_source}"
"${paimon_archive}" paimon \
"${paimon_patches[@]}"
+exercise_legacy_source_isolation() {
+ local index
+
+ touch "${harness}/src/${arrow_source}/current-arrow-sentinel"
+ TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \
+ bash "${harness}/download-thirdparty.sh" ARROW_17 >/dev/null
+ for index in 1 2 3; do
+ [[ "$(<"${harness}/src/${arrow_17_source}/arrow17-${index}.txt")" ==
"patched" ]] ||
+ fail "Arrow 17 patch ${index} was not applied"
+ done
+ [[ -f "${harness}/src/${arrow_source}/current-arrow-sentinel" ]] ||
+ fail "extracting Arrow 17 modified the Arrow 24 source"
+
+ touch "${harness}/src/${paimon_source}/current-paimon-sentinel"
+ TP_DIR="${harness}" DORIS_HOME="${tmpdir}" \
+ bash "${harness}/download-thirdparty.sh" PAIMON_CPP_17 >/dev/null
+ [[ "$(<"${harness}/src/${paimon_17_source}/paimon-1.txt")" == "patched" ]]
||
+ fail "Paimon Arrow 17 static dependency patch was not applied"
+ for index in 2 3; do
+ [[ "$(<"${harness}/src/${paimon_17_source}/paimon-${index}.txt")" ==
"original" ]] ||
+ fail "Paimon Arrow 17 received an Arrow 24-only patch"
+ done
+ [[ -f "${harness}/src/${paimon_source}/current-paimon-sentinel" ]] ||
+ fail "extracting Paimon for Arrow 17 modified the Arrow 24 source"
+}
+
+exercise_legacy_source_isolation
+
exercise_generic_recovery_dispatch() {
local generic="${tmpdir}/generic-recovery"
local thirdparty_dir="${generic}/thirdparty"
@@ -254,6 +303,8 @@ exercise_generic_recovery_dispatch() {
local clean
local package1
local package2
+ local package3
+ local package4
local extra
mkdir -p "${thirdparty_dir}/installed/lib/hadoop_hdfs_3_4/native" \
@@ -307,6 +358,18 @@ exercise_generic_recovery_dispatch() {
fail "${non_native_target} invoked the native Arrow/Paimon builder"
done
+ rm -f "${args_file}"
+ if DORIS_THIRDPARTY="${external_thirdparty_dir}" \
+ bash "${generic}/build.sh" --cloud >"${output_file}" 2>&1; then
+ fail "--cloud did not reach the generated-source sentinel"
+ else
+ status=$?
+ fi
+ [[ "${status}" -eq "${non_native_status}" ]] ||
+ fail "--cloud incorrectly required the BE Arrow/Paimon stack"
+ [[ ! -e "${args_file}" ]] ||
+ fail "--cloud invoked the native Arrow/Paimon builder"
+
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
bash "${generic}/build.sh" --fe --clean >"${output_file}" 2>&1; then
fail "--fe --clean did not reach the generated-source sentinel"
@@ -318,6 +381,32 @@ exercise_generic_recovery_dispatch() {
[[ ! -e "${args_file}" ]] ||
fail "--fe --clean invoked the native Arrow/Paimon builder"
+ rm -f "${args_file}"
+ if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
+ bash "${generic}/build.sh" --compile-bench >"${output_file}" 2>&1; then
+ fail "standalone --compile-bench skipped stale-prebuilt recovery"
+ else
+ status=$?
+ fi
+ [[ "${status}" -eq 73 ]] ||
+ fail "standalone --compile-bench failed before invoking its builder"
+ read -r flag parallel package1 package2 package3 package4 extra
<"${args_file}"
+ [[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ &&
+ "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" &&
+ "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z
"${extra}" ]] ||
+ fail "standalone --compile-bench dispatched the wrong build package
set"
+
+ rm -f "${args_file}"
+ if ARROW_HOME="${generic}/explicit-arrow"
PAIMON_HOME="${generic}/explicit-paimon" \
+ DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}"
\
+ bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then
+ fail "build.sh accepted unsupported explicit Arrow/Paimon prefixes"
+ fi
+ grep -Fq "only supports the Arrow/Paimon stack selected from
DORIS_THIRDPARTY" \
+ "${output_file}" || fail "build.sh did not explain its supported
Arrow/Paimon selection"
+ [[ ! -e "${args_file}" ]] ||
+ fail "an unsupported explicit Arrow/Paimon selection invoked the
builder"
+
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
bash "${generic}/build.sh" --be >"${output_file}" 2>&1; then
fail "generic stale-prebuilt recovery did not invoke the focused
builder"
@@ -325,9 +414,10 @@ exercise_generic_recovery_dispatch() {
status=$?
fi
[[ "${status}" -eq 73 ]] || fail "generic recovery failed before invoking
its builder"
- read -r flag parallel package1 package2 extra <"${args_file}"
+ read -r flag parallel package1 package2 package3 package4 extra
<"${args_file}"
[[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ &&
- "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z
"${extra}" ]] ||
+ "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" &&
+ "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z
"${extra}" ]] ||
fail "generic recovery dispatched the wrong build package set"
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
@@ -337,9 +427,10 @@ exercise_generic_recovery_dispatch() {
status=$?
fi
[[ "${status}" -eq 73 ]] || fail "generic clean recovery failed before
invoking its builder"
- read -r flag parallel clean package1 package2 extra <"${args_file}"
+ read -r flag parallel clean package1 package2 package3 package4 extra
<"${args_file}"
[[ "${flag}" == "-j" && "${parallel}" =~ ^[0-9]+$ && "${clean}" ==
"--clean" &&
- "${package1}" == "arrow" && "${package2}" == "paimon_cpp" && -z
"${extra}" ]] ||
+ "${package1}" == "arrow_17" && "${package2}" == "paimon_cpp_17" &&
+ "${package3}" == "arrow" && "${package4}" == "paimon_cpp" && -z
"${extra}" ]] ||
fail "generic clean recovery dispatched the wrong build package set"
if DORIS_THIRDPARTY="${thirdparty_dir}" RECOVERY_ARGS_FILE="${args_file}" \
@@ -378,18 +469,24 @@ exercise_generic_recovery_dispatch
# A Paimon-only build may publish only its own fingerprint. It must not make a
# stale Arrow installation pass the shared prebuilt validation.
prebuilt="${tmpdir}/prebuilt"
-mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64"
+selected_prebuilt="$(arrow_install_dir "${prebuilt}")"
+mkdir -p "${selected_prebuilt}/include/arrow/util" "${selected_prebuilt}/lib64"
printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_VERSION}" \
- >"${prebuilt}/include/arrow/util/config.h"
+ >"${selected_prebuilt}/include/arrow/util/config.h"
for library in "${ARROW_PAIMON_REQUIRED_LIBRARIES[@]}"; do
- touch "${prebuilt}/lib64/${library}"
+ touch "${selected_prebuilt}/lib64/${library}"
done
prepare_arrow_paimon_download_packages "${ARROW_PAIMON_BUILD_PACKAGES[@]}"
[[ "${ARROW_PAIMON_BUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] ||
fail "focused recovery dispatches a bundled source package as a build
target"
+[[ "${ARROW_PAIMON_SHARED_BUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 arrow
paimon_cpp" ]] ||
+ fail "shared recovery does not cover both installed Arrow/Paimon stacks"
[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow paimon_cpp xsimd brotli"
]] ||
fail "focused recovery does not download the complete Arrow source closure"
+prepare_arrow_paimon_download_packages arrow_17 paimon_cpp_17
+[[ "${ARROW_PAIMON_DOWNLOAD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17 xsimd_17
brotli" ]] ||
+ fail "Arrow 17 build does not download its independent source closure"
# A legacy prebuilt may have the old combined marker but no component markers.
# Generic build.sh consumers must reject it before importing Arrow Compute.
@@ -399,11 +496,20 @@ if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null
2>&1; then
fi
printf '%s\n' "${ARROW_LEGACY_BUILD_FINGERPRINTS[0]}" \
- >"${prebuilt}/arrow-build-fingerprint.txt"
+ >"${selected_prebuilt}/arrow-build-fingerprint.txt"
printf '%s\n' "${PAIMON_LEGACY_BUILD_FINGERPRINTS[0]}" \
- >"${prebuilt}/paimon-build-fingerprint.txt"
+ >"${selected_prebuilt}/paimon-build-fingerprint.txt"
+arrow_prebuilt_valid "${prebuilt}" ||
+ fail "the unchanged Arrow artifacts were rejected during fingerprint
migration"
+if paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
+ fail "the root-prefix Paimon marker certified the dual-prefix Paimon build"
+fi
+if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
+ fail "the shared prebuilt accepted a stale Paimon marker"
+fi
+publish_paimon_prebuilt_marker "${prebuilt}"
arrow_paimon_prebuilt_valid "${prebuilt}" ||
- fail "the complete shared prebuilt was rejected during fingerprint
migration"
+ fail "the migrated Arrow and Paimon component markers were rejected"
if (
ARROW_BUILD_SCHEMA_VERSION="${ARROW_BUILD_SCHEMA_VERSION}-changed"
arrow_prebuilt_valid "${prebuilt}"
@@ -419,13 +525,13 @@ fi
publish_arrow_prebuilt_marker "${prebuilt}"
publish_paimon_prebuilt_marker "${prebuilt}"
-rm "${prebuilt}/lib64/libarrow_compute.a"
+rm "${selected_prebuilt}/lib64/libarrow_compute.a"
if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
fail "prebuilt validation accepted a missing Arrow Compute archive"
fi
-touch "${prebuilt}/lib64/libarrow_compute.a"
+touch "${selected_prebuilt}/lib64/libarrow_compute.a"
-printf '%s\n' stale-arrow >"${prebuilt}/arrow-build-fingerprint.txt"
+printf '%s\n' stale-arrow >"${selected_prebuilt}/arrow-build-fingerprint.txt"
if arrow_paimon_prebuilt_valid "${prebuilt}" >/dev/null 2>&1; then
fail "Paimon-only marker update certified a stale Arrow build"
fi
@@ -451,5 +557,132 @@ fi
publish_arrow_prebuilt_marker "${prebuilt}"
arrow_paimon_prebuilt_valid "${prebuilt}" ||
fail "republished component markers were rejected"
+select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1
+[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] ||
+ fail "recovery did not select only the missing Arrow 17 stack"
+
+# The legacy stack remains at the unversioned prefix and is validated
+# independently. Rebuilding or invalidating either stack must not affect the
+# other branch's artifacts or fingerprints.
+mkdir -p "${prebuilt}/include/arrow/util" "${prebuilt}/lib64"
+printf '#define ARROW_VERSION_STRING "%s"\n' "${ARROW_17_VERSION}" \
+ >"${prebuilt}/include/arrow/util/config.h"
+for library in "${ARROW_17_REQUIRED_LIBRARIES[@]}"
"${PAIMON_REQUIRED_LIBRARIES[@]}"; do
+ touch "${prebuilt}/lib64/${library}"
+done
+publish_arrow_17_prebuilt_marker "${prebuilt}"
+publish_paimon_17_prebuilt_marker "${prebuilt}"
+arrow_paimon_17_prebuilt_valid "${prebuilt}" ||
+ fail "matching Arrow 17 and Paimon artifacts were rejected"
+arrow_paimon_prebuilt_valid "${prebuilt}" ||
+ fail "publishing the Arrow 17 stack invalidated Arrow 24"
+shared_arrow_paimon_prebuilt_valid "${prebuilt}" ||
+ fail "matching shared Arrow/Paimon stacks were rejected"
+select_arrow_paimon_rebuild_packages "${prebuilt}"
+[[ "${#ARROW_PAIMON_REBUILD_PACKAGES[@]}" -eq 0 ]] ||
+ fail "recovery rebuilt an already valid shared stack"
+
+invalidate_arrow_prebuilt_marker "${prebuilt}"
+arrow_paimon_17_prebuilt_valid "${prebuilt}" ||
+ fail "invalidating Arrow 24 affected the Arrow 17 stack"
+select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1
+[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow paimon_cpp" ]] ||
+ fail "recovery did not isolate an invalid Arrow 24 stack"
+publish_arrow_prebuilt_marker "${prebuilt}"
+
+invalidate_arrow_17_prebuilt_marker "${prebuilt}"
+arrow_paimon_prebuilt_valid "${prebuilt}" ||
+ fail "invalidating Arrow 17 affected the Arrow 24 stack"
+select_arrow_paimon_rebuild_packages "${prebuilt}" >/dev/null 2>&1
+[[ "${ARROW_PAIMON_REBUILD_PACKAGES[*]}" == "arrow_17 paimon_cpp_17" ]] ||
+ fail "recovery did not isolate an invalid Arrow 17 stack"
+publish_arrow_17_prebuilt_marker "${prebuilt}"
+arrow_paimon_17_prebuilt_valid "${prebuilt}" ||
+ fail "republished Arrow 17 component markers were rejected"
+
+# Preparing a root-prefix Arrow downgrade removes Paimon first. An interrupted
+# Arrow 17 build therefore cannot expose a root Arrow 17/Paimon 24 mixture to
an
+# unchanged branch-4.1 consumer.
+migration_prefix="${tmpdir}/interrupted-arrow-17-migration"
+migration_selected_prefix="$(arrow_install_dir "${migration_prefix}")"
+mkdir -p \
+ "${migration_prefix}/include/arrow" \
+ "${migration_prefix}/include/parquet" \
+ "${migration_prefix}/include/paimon" \
+ "${migration_prefix}/include/unrelated" \
+ "${migration_prefix}/lib64" \
+ "${migration_selected_prefix}"
+for library in "${ARROW_REQUIRED_LIBRARIES[@]}"
"${PAIMON_REQUIRED_LIBRARIES[@]}"; do
+ touch "${migration_prefix}/lib64/${library}"
+done
+touch \
+ "${migration_prefix}/arrow-build-fingerprint.txt" \
+ "${migration_prefix}/paimon-build-fingerprint.txt" \
+ "${migration_prefix}/arrow-paimon-build-fingerprint.txt" \
+ "${migration_prefix}/arrow-17-build-fingerprint.txt" \
+ "${migration_prefix}/paimon-arrow-17-build-fingerprint.txt" \
+ "${migration_prefix}/arrow-paimon-17-build-fingerprint.txt" \
+ "${migration_prefix}/include/unrelated/sentinel" \
+ "${migration_selected_prefix}/sentinel"
+
+prepare_arrow_17_install_prefix "${migration_prefix}"
+[[ ! -e "${migration_prefix}/include/arrow" &&
+ ! -e "${migration_prefix}/include/parquet" &&
+ ! -e "${migration_prefix}/include/paimon" &&
+ ! -e "${migration_prefix}/lib64/libarrow.a" &&
+ ! -e "${migration_prefix}/lib64/libpaimon.a" &&
+ ! -e "${migration_prefix}/arrow-build-fingerprint.txt" &&
+ ! -e "${migration_prefix}/paimon-build-fingerprint.txt" &&
+ ! -e "${migration_prefix}/arrow-17-build-fingerprint.txt" &&
+ ! -e "${migration_prefix}/paimon-arrow-17-build-fingerprint.txt" ]] ||
+ fail "an interrupted Arrow 17 migration left a mixed root-prefix stack"
+[[ -e "${migration_prefix}/include/unrelated/sentinel" &&
+ -e "${migration_selected_prefix}/sentinel" ]] ||
+ fail "preparing the Arrow 17 migration removed another stack's artifacts"
+
+# Reinstalling one stack cleans only files owned by that stack. In particular,
+# a downgrade of the legacy prefix must remove Arrow 24-only artifacts without
+# deleting Paimon or unrelated thirdparty files.
+cleanup_prefix="${tmpdir}/cleanup-prefix"
+mkdir -p \
+ "${cleanup_prefix}/include/arrow" \
+ "${cleanup_prefix}/include/parquet" \
+ "${cleanup_prefix}/include/paimon" \
+ "${cleanup_prefix}/include/unrelated" \
+ "${cleanup_prefix}/lib64/cmake/ArrowCompute" \
+ "${cleanup_prefix}/lib64/cmake/Paimon" \
+ "${cleanup_prefix}/lib64/pkgconfig" \
+ "${cleanup_prefix}/share/arrow" \
+ "${cleanup_prefix}/share/doc/arrow"
+touch \
+ "${cleanup_prefix}/lib64/libarrow_compute.a" \
+ "${cleanup_prefix}/lib64/libparquet.a" \
+ "${cleanup_prefix}/lib64/libpaimon.a" \
+ "${cleanup_prefix}/lib64/libfmt_paimon.a" \
+ "${cleanup_prefix}/lib64/libunrelated.a" \
+ "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" \
+ "${cleanup_prefix}/include/unrelated/sentinel"
+
+clean_arrow_artifacts_in "${cleanup_prefix}"
+[[ ! -e "${cleanup_prefix}/include/arrow" &&
+ ! -e "${cleanup_prefix}/include/parquet" &&
+ ! -e "${cleanup_prefix}/lib64/libarrow_compute.a" &&
+ ! -e "${cleanup_prefix}/lib64/cmake/ArrowCompute" &&
+ ! -e "${cleanup_prefix}/lib64/pkgconfig/arrow-compute.pc" ]] ||
+ fail "Arrow cleanup left stale artifacts in the selected prefix"
+[[ -e "${cleanup_prefix}/include/paimon" &&
+ -e "${cleanup_prefix}/lib64/libpaimon.a" &&
+ -e "${cleanup_prefix}/include/unrelated/sentinel" ]] ||
+ fail "Arrow cleanup removed another package's artifacts"
+
+clean_paimon_artifacts_in "${cleanup_prefix}"
+[[ ! -e "${cleanup_prefix}/include/paimon" &&
+ ! -e "${cleanup_prefix}/lib64/libpaimon.a" &&
+ ! -e "${cleanup_prefix}/lib64/libfmt_paimon.a" &&
+ ! -e "${cleanup_prefix}/lib64/cmake/Paimon" ]] ||
+ fail "Paimon cleanup left stale artifacts in the selected prefix"
+[[ -e "${cleanup_prefix}/lib64/libunrelated.a" &&
+ -e "${cleanup_prefix}/include/unrelated/sentinel" ]] ||
+ fail "Paimon cleanup removed another package's artifacts"
echo "PASS"
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 27fca98f3f9..13a3a04e6a6 100644
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -618,6 +618,7 @@ export TP_ARCHIVES=(
'CYRUS_SASL'
'LIBRDKAFKA'
'FLATBUFFERS'
+ 'ARROW_17'
'ARROW'
'ARROW_ADBC'
'BROTLI'
@@ -650,6 +651,7 @@ export TP_ARCHIVES=(
'PDQSORT'
'TIMSORT'
'BENCHMARK'
+ 'XSIMD_17'
'XSIMD'
'SIMDJSON'
'NLOHMANN_JSON'
@@ -672,6 +674,7 @@ export TP_ARCHIVES=(
'JINDOFS'
'JUICEFS'
'PUGIXML'
+ 'PAIMON_CPP_17'
'PAIMON_CPP'
'LANCE_C'
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
