This is an automated email from the ASF dual-hosted git repository. philo-he pushed a commit to branch branch-1.7 in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 84ef898f17825041d050c43e9bcf5061406692d2 Author: Philo He <[email protected]> AuthorDate: Tue Jul 21 10:57:39 2026 +0800 [VL] Fix build issues reported by weekly jobs (#12321) Co-authored-by: Yuan <[email protected]> --- .github/workflows/velox_weekly.yml | 126 ++++++++++++-------------------- cpp/CMake/ConfigArrow.cmake | 34 ++++++--- dev/build-thirdparty.sh | 7 +- dev/vcpkg/setup-build-depends.sh | 2 +- ep/build-velox/src/get-velox.sh | 1 + ep/build-velox/src/setup-centos8.sh | 18 +++-- ep/build-velox/src/setup-openeuler24.sh | 30 ++++++-- 7 files changed, 110 insertions(+), 108 deletions(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 1e0cd10155..b86e50d287 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -35,40 +35,8 @@ concurrency: cancel-in-progress: true jobs: - build-on-centos-7: - strategy: - fail-fast: false - if: ${{ startsWith(github.repository, 'apache/') }} - runs-on: ubuntu-22.04 - container: centos:7 - steps: - - name: Build Gluten native libraries and package - run: | - sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true - sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true - yum update -y - yum install -y epel-release sudo dnf - yum install -y centos-release-scl - rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f - sed -i \ - -e 's/^mirrorlist/#mirrorlist/' \ - -e 's/^#baseurl/baseurl/' \ - -e 's/mirror\.centos\.org/vault.centos.org/' \ - /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo - yum install -y devtoolset-11 - source /opt/rh/devtoolset-11/enable - yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip - export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ - export PATH=$JAVA_HOME/bin:$PATH - - # action/checkout does not work centos7 anymore, so we clone the branch instead. - git clone -b main --depth=1 https://github.com/apache/gluten.git && cd gluten/ - if [ ${{ github.event_name }} = "pull_request" ]; then - git fetch origin ${{ github.ref }}:pr_branch && git checkout pr_branch - fi - ./dev/package.sh --spark_version=3.5 - build-on-centos: + weekly-build-on-centos: strategy: fail-fast: false matrix: @@ -79,42 +47,33 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Update mirror list - run: | - sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true - sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true - name: build run: | - yum update -y - yum install -y epel-release sudo dnf - if [ "${{ matrix.os }}" = "centos:7" ]; then - yum install -y centos-release-scl - rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f - sed -i \ - -e 's/^mirrorlist/#mirrorlist/' \ - -e 's/^#baseurl/baseurl/' \ - -e 's/mirror\.centos\.org/vault.centos.org/' \ - /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo - yum install -y devtoolset-11 - source /opt/rh/devtoolset-11/enable - elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then - dnf install -y --setopt=install_weak_deps=False gcc-toolset-12 - source /opt/rh/gcc-toolset-12/enable || exit 1 - else - dnf install -y --setopt=install_weak_deps=False gcc-toolset-11 - source /opt/rh/gcc-toolset-11/enable || exit 1 + if [ "${{ matrix.os }}" = "centos:8" ]; then + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true + dnf update -y + dnf install -y centos-release-stream + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true + dnf swap -y --allowerasing centos-{linux,stream}-repos + sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true + sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true fi + dnf install -y epel-release sudo dnf + dnf install -y --setopt=install_weak_deps=False gcc-toolset-12 + source /opt/rh/gcc-toolset-12/enable || exit 1 yum install -y java-1.8.0-openjdk-devel patch wget git perl export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ export PATH=$JAVA_HOME/bin:$PATH - + export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 - build-on-ubuntu: + weekly-build-on-ubuntu: strategy: fail-fast: false matrix: - os: [ "ubuntu:22.04" ] # skip "ubuntu:20.04" due to not enough disk space + os: [ "ubuntu:22.04" ] if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ubuntu-22.04 container: ${{ matrix.os }} @@ -125,30 +84,35 @@ jobs: # sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list # To avoid the prompt for region selection during installing tzdata. export DEBIAN_FRONTEND=noninteractive - apt-get update && apt-get install -y sudo maven wget git curl - if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then - sudo apt install -y software-properties-common - sudo add-apt-repository ppa:ubuntu-toolchain-r/test - sudo apt update && sudo apt install -y gcc-11 g++-11 build-essential cmake - export CC=/usr/bin/gcc-11 - export CXX=/usr/bin/g++-11 - wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz - tar xf v0.8.1.tar.gz - cd xxHash-0.8.1/ - make && make install - cd /tmp - wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz - tar xf v1.13.0.tar.gz && cd googletest-1.13.0/ - export CXXFLAGS=-fPIC - cmake . && make install -j - apt autoremove -y - rm -rf /tmp/* - fi - sudo apt-get install -y openjdk-8-jdk python3-pip cmake + export INSTALL_PREFIX=/usr/ + apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake openjdk-8-jdk python3-pip + pip3 install cmake==3.31.4 + cd /tmp + wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz + tar -xzf gflags.tar.gz && cd gflags-2.2.2 + cmake -B build \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_STATIC_LIBS=ON \ + -DBUILD_gflags_LIB=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=Release + cmake --build build -j4 --target install + wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz + tar xf v0.8.1.tar.gz + cd xxHash-0.8.1/ + make && make install + cd /tmp + wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz + tar xf v1.13.0.tar.gz && cd googletest-1.13.0/ + export CXXFLAGS=-fPIC + cmake . && make install -j + apt autoremove -y export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + export VELOX_BUILD_SHARED=ON cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 - build-on-openeuler: + weekly-build-on-openeuler: strategy: fail-fast: false matrix: @@ -177,7 +141,7 @@ jobs: $TPCH_TEST --extra-conf=spark.gluten.loadLibFromJar=true \ --extra-conf=spark.gluten.loadLibOS=openEuler --extra-conf=spark.gluten.loadLibOSVersion=24.03 - build-on-openeuler-vcpkg: + weekly-build-on-openeuler-vcpkg: strategy: fail-fast: false matrix: @@ -205,7 +169,7 @@ jobs: cd $GITHUB_WORKSPACE/tools/gluten-it $TPCH_TEST - build-on-macos: + weekly-build-on-macos: strategy: fail-fast: false if: ${{ startsWith(github.repository, 'apache/') }} diff --git a/cpp/CMake/ConfigArrow.cmake b/cpp/CMake/ConfigArrow.cmake index ee7eb6fcca..0d604a91e6 100644 --- a/cpp/CMake/ConfigArrow.cmake +++ b/cpp/CMake/ConfigArrow.cmake @@ -44,20 +44,32 @@ function(FIND_ARROW_LIB LIB_NAME) endif() message(STATUS "Found Arrow library: ${ARROW_LIB_${LIB_NAME}}") - # Get the parent-parent directory of the lib file. For example: - # - # * ${ARROW_LIB_${LIB_NAME}}: /usr/local/lib/libarrow.a - # * ${ARROW_LIB_INCLUDE_DIR}: /usr/local - # - # Then we can get our include directory: /usr/local/include - get_filename_component(ARROW_LIB_INCLUDE_DIR "${ARROW_LIB_${LIB_NAME}}" - PATH) - get_filename_component(ARROW_LIB_INCLUDE_DIR "${ARROW_LIB_INCLUDE_DIR}" - PATH) + # Walk up from the library directory until we find a parent that contains an + # "include/arrow/api.h" anchor file. This handles both conventional layouts + # (/usr/local/lib/libarrow.a → /usr/local/include) and multi-arch ones + # (/usr/lib/x86_64-linux-gnu/libarrow.a → /usr/include) without hard-coding + # the number of directory levels to strip. + get_filename_component(_arrow_search_dir "${ARROW_LIB_${LIB_NAME}}" PATH) + set(ARROW_LIB_INCLUDE_DIR "") + foreach(_level RANGE 5) + if(EXISTS "${_arrow_search_dir}/include/arrow/api.h") + set(ARROW_LIB_INCLUDE_DIR "${_arrow_search_dir}/include") + break() + endif() + get_filename_component(_arrow_search_dir "${_arrow_search_dir}" PATH) + endforeach() + if(NOT ARROW_LIB_INCLUDE_DIR) + message( + FATAL_ERROR + "Could not locate Arrow headers near ${ARROW_LIB_${LIB_NAME}}. " + "Set ARROW_HOME to the Arrow installation prefix (the directory " + "that contains lib/ and include/).") + endif() + message(STATUS "Found Arrow include: ${ARROW_LIB_INCLUDE_DIR}") set_target_properties( Arrow::${LIB_NAME} PROPERTIES IMPORTED_LOCATION ${ARROW_LIB_${LIB_NAME}} - INTERFACE_INCLUDE_DIRECTORIES ${ARROW_LIB_INCLUDE_DIR}/include) + INTERFACE_INCLUDE_DIRECTORIES ${ARROW_LIB_INCLUDE_DIR}) endif() endfunction() diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh index e0968b316d..077999ccce 100755 --- a/dev/build-thirdparty.sh +++ b/dev/build-thirdparty.sh @@ -32,13 +32,16 @@ function process_setup_ubuntu_2004 { function process_setup_ubuntu_2204 { cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.2,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/ - cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_regex.so.1.84.0} $THIRDPARTY_LIB/ + find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null + find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } function process_setup_centos_9 { cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/ cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/ - cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/ + cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/ + find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null + find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null } function process_setup_centos_8 { diff --git a/dev/vcpkg/setup-build-depends.sh b/dev/vcpkg/setup-build-depends.sh index 1a5ed49013..f92df8b479 100755 --- a/dev/vcpkg/setup-build-depends.sh +++ b/dev/vcpkg/setup-build-depends.sh @@ -207,7 +207,7 @@ install_openeuler_24.03() { gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \ java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static - pip install cmake==3.28.3 + pip install cmake==3.31.4 install_maven_from_source } diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index c9a704cd7f..dd357dbbf7 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -65,6 +65,7 @@ if [ "$VELOX_HOME" == "" ]; then fi function process_setup_ubuntu { + sed -i "s|run_and_time install_arrow||g" scripts/setup-centos9.sh echo "Using setup script from Velox" } diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 9841157539..3aa6f922a9 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -39,8 +39,8 @@ export CPPFLAGS=$CFLAGS # Used by LZO. CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" BUILD_DUCKDB="${BUILD_DUCKDB:-true}" BUILD_GEOS="${BUILD_GEOS:-true}" -export CC=/opt/rh/gcc-toolset-11/root/bin/gcc -export CXX=/opt/rh/gcc-toolset-11/root/bin/g++ +export CC=/opt/rh/gcc-toolset-12/root/bin/gcc +export CXX=/opt/rh/gcc-toolset-12/root/bin/g++ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} FB_OS_VERSION="v2026.01.05.00" @@ -57,9 +57,10 @@ function dnf_install { function install_build_prerequisites { dnf update -y dnf_install epel-release dnf-plugins-core # For ccache, ninja - dnf config-manager --set-enabled powertools + dnf config-manager --set-enabled powertools || true # For centos8, powertools is needed for some dependencies. + dnf config-manager --set-enabled Stream-PowerTools || true dnf update -y - dnf_install ninja-build curl ccache gcc-toolset-11 git wget which expat-devel gettext-devel + dnf_install ninja-build curl ccache gcc-toolset-12 git wget which expat-devel gettext-devel dnf_install yasm dnf_install autoconf automake python39 python39-devel python39-pip libtool pip3.9 install cmake==3.28.3 @@ -172,7 +173,10 @@ function install_wangle { function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF + cd ${DEPENDENCY_DIR}/fbthrift + git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true + cd ${DEPENDENCY_DIR} + cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } function install_mvfst { @@ -228,7 +232,7 @@ function install_velox_deps { ( if [[ $# -ne 0 ]]; then # Activate gcc11; enable errors on unset variables afterwards. - source /opt/rh/gcc-toolset-11/enable || exit 1 + source /opt/rh/gcc-toolset-12/enable || exit 1 set -u for cmd in "$@"; do run_and_time "${cmd}" @@ -242,7 +246,7 @@ function install_velox_deps { echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" fi # Activate gcc11; enable errors on unset variables afterwards. - source /opt/rh/gcc-toolset-11/enable || exit 1 + source /opt/rh/gcc-toolset-12/enable || exit 1 set -u install_velox_deps echo "All dependencies for Velox installed!" diff --git a/ep/build-velox/src/setup-openeuler24.sh b/ep/build-velox/src/setup-openeuler24.sh index 29204bbbf8..80a66917b2 100755 --- a/ep/build-velox/src/setup-openeuler24.sh +++ b/ep/build-velox/src/setup-openeuler24.sh @@ -36,6 +36,7 @@ NPROC=$(getconf _NPROCESSORS_ONLN) export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO. export CXXFLAGS=$CFLAGS # Used by boost. export CPPFLAGS=$CFLAGS # Used by LZO. +EXTRA_PKG_CXXFLAGS=" -isystem ${INSTALL_PREFIX}/include" CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so. BUILD_DUCKDB="${BUILD_DUCKDB:-true}" @@ -44,8 +45,8 @@ VERSION=$(cat /etc/os-release | grep VERSION_ID) export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"} DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} -FB_OS_VERSION="v2024.07.01.00" -FMT_VERSION="10.1.1" +FB_OS_VERSION="v2026.01.05.00" +FMT_VERSION="11.2.0" BOOST_VERSION="boost-1.84.0" DUCKDB_VERSION="v0.8.1" GEOS_VERSION="3.10.7" @@ -65,7 +66,7 @@ function install_build_prerequisites { dnf_install autoconf automake python3-devel python3-pip libtool dnf_install libxml2-devel libgsasl-devel libuuid-devel - pip install cmake==3.28.3 + pip install cmake==3.31.4 } # Install dependencies from the package managers. @@ -110,11 +111,23 @@ function install_boost { ) } +function install_xxhash { + wget_and_untar https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.2.tar.gz xxHash + cd ${DEPENDENCY_DIR}/xxHash + make "-j${NPROC}" + make install PREFIX=${INSTALL_PREFIX} +} + function install_snappy { wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy cmake_install_dir snappy -DSNAPPY_BUILD_TESTS=OFF } +function install_fast_float { + wget_and_untar https://github.com/fastfloat/fast_float/archive/refs/tags/v8.0.2.tar.gz fast_float + cmake_install_dir fast_float -DBUILD_TESTS=OFF +} + function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt cmake_install_dir fmt -DFMT_TEST=OFF @@ -133,7 +146,7 @@ function install_protobuf { function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF + cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF } function install_folly { @@ -143,11 +156,14 @@ function install_folly { function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF + cmake_install_dir wangle/wangle -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF } function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift + cd ${DEPENDENCY_DIR}/fbthrift + git apply ${VELOX_HOME:-${DEPENDENCY_DIR}/../}/CMake/resolve_dependency_modules/fbthrift/compactv1-protocol-refiller.patch 2>/dev/null || true + cd ${DEPENDENCY_DIR} cmake_install_dir fbthrift -Denable_tests=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } @@ -198,8 +214,10 @@ function install_velox_deps { run_and_time install_gflags run_and_time install_glog run_and_time install_lzo + run_and_time install_xxhash run_and_time install_snappy run_and_time install_boost + run_and_time install_fast_float run_and_time install_protobuf run_and_time install_fmt run_and_time install_folly @@ -233,4 +251,4 @@ function install_velox_deps { echo "All dependencies for Velox installed!" dnf clean all fi -) +) \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
