This is an automated email from the ASF dual-hosted git repository.
philo-he pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1986efcb38 [VL] Add a basic TPC test for weekly job (#12619)
1986efcb38 is described below
commit 1986efcb387803d393f4e7453d0f1e48a59445ad
Author: Yuan <[email protected]>
AuthorDate: Tue Jul 28 15:52:23 2026 -0700
[VL] Add a basic TPC test for weekly job (#12619)
---
.github/workflows/velox_weekly.yml | 187 +++++++++++++++++++--
.../gluten/spi/SharedLibraryLoaderCentos7.scala | 2 +-
.../gluten/spi/SharedLibraryLoaderCentos8.scala | 2 +-
.../gluten/spi/SharedLibraryLoaderCentos9.scala | 2 +-
.../gluten/spi/SharedLibraryLoaderDebian11.scala | 2 +-
.../gluten/spi/SharedLibraryLoaderDebian12.scala | 2 +-
.../spi/SharedLibraryLoaderOpenEuler2403.scala | 2 +-
.../gluten/spi/SharedLibraryLoaderUbuntu2204.scala | 2 +-
dev/build-thirdparty.sh | 27 +--
.../cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf | 2 +-
ep/build-velox/src/build-velox.sh | 4 +
ep/build-velox/src/get-velox.sh | 2 +-
ep/build-velox/src/setup-centos8.sh | 2 +-
ep/build-velox/src/setup-openeuler24.sh | 2 +-
14 files changed, 206 insertions(+), 34 deletions(-)
diff --git a/.github/workflows/velox_weekly.yml
b/.github/workflows/velox_weekly.yml
index 43bac7fc6b..af8a17530c 100644
--- a/.github/workflows/velox_weekly.yml
+++ b/.github/workflows/velox_weekly.yml
@@ -25,9 +25,10 @@ on:
env:
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
- TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare
+ TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare
--local --preset=velox --benchmark-type=h --error-on-memleak
--off-heap-size=10g -s=1.0 --threads=16 --iterations=1"
INSTALL_PREFIX: /usr/local
+ CCACHE_DIR: "${{ github.workspace }}/.ccache"
concurrency:
@@ -47,6 +48,30 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
+ - name: Install Stash restore dependencies # the stash action needs gh,
jq, and python3 (>=3.7), absent from this container
+ run: |
+ if [ "${{ matrix.os }}" = "centos:8" ]; then
+ sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-*
|| true
+ sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true
+ # centos:8 ships python3.6 by default; install python39 from
AppStream and override the symlink
+ dnf install -y python39 && ln -sf /usr/bin/python3.9
/usr/bin/python3
+ else
+ dnf install -y python3
+ fi
+ case "$(uname -m)" in
+ x86_64) ARCH=amd64 ;;
+ aarch64) ARCH=arm64 ;;
+ esac
+ curl -fsSL
"https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz";
\
+ | tar -xz -C /usr/local --strip-components=1
"gh_2.63.2_linux_${ARCH}/bin/gh"
+ curl -fsSL -o /usr/local/bin/jq
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}";
+ chmod +x /usr/local/bin/jq
+ gh --version && jq --version
+ - name: Get Ccache from Apache Stash
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-centos-${{ matrix.os }}-${{
hashFiles('ep/build-velox/src/**') }}
- name: build
run: |
if [ "${{ matrix.os }}" = "centos:8" ]; then
@@ -63,33 +88,74 @@ jobs:
dnf install -y epel-release sudo dnf
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
source /opt/rh/gcc-toolset-12/enable || exit 1
- yum install -y java-1.8.0-openjdk-devel patch wget git perl
+ yum install -y java-1.8.0-openjdk-devel patch wget git perl ccache
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
export PATH=$JAVA_HOME/bin:$PATH
+ export CCACHE_MAXSIZE=1G
+ mkdir -p $CCACHE_DIR
export VELOX_BUILD_SHARED=ON
cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
+ - name: Clean Ccache tmp before save
+ if: always()
+ run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+ - name: Save Ccache to Apache Stash
+ if: always()
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-centos-${{ matrix.os }}-${{
hashFiles('ep/build-velox/src/**') }}
+ include-hidden-files: true
+ retention-days: '7'
+ - name: Run TPC-H
+ run: |
+ export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $TPCH_TEST
weekly-build-on-ubuntu:
strategy:
fail-fast: false
matrix:
- os: [ "ubuntu:22.04" ]
+ os: [ "ubuntu:22.04", "ubuntu:24.04" ]
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
container: ${{ matrix.os }}
steps:
- uses: actions/checkout@v4
+ - name: Install Stash restore dependencies # the stash action needs gh,
jq, and python3, absent from this container
+ run: |
+ case "$(uname -m)" in
+ x86_64) ARCH=amd64 ;;
+ aarch64) ARCH=arm64 ;;
+ esac
+ apt-get update -y && apt-get install -y curl python3
+ curl -fsSL
"https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz";
\
+ | tar -xz -C /usr/local --strip-components=1
"gh_2.63.2_linux_${ARCH}/bin/gh"
+ curl -fsSL -o /usr/local/bin/jq
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}";
+ chmod +x /usr/local/bin/jq
+ gh --version && jq --version
+ - name: Get Ccache from Apache Stash
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-ubuntu-${{ matrix.os }}-${{
hashFiles('ep/build-velox/src/**') }}
- name: build
run: |
# sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list
# To avoid the prompt for region selection during installing tzdata.
export DEBIAN_FRONTEND=noninteractive
export INSTALL_PREFIX=/usr/
- apt-get update && apt-get install -y sudo maven wget git curl gcc-11
g++-11 build-essential cmake openjdk-8-jdk python3-pip
- pip3 install cmake==3.31.4
+ apt-get update && apt-get install -y sudo maven wget git curl gcc-11
g++-11 build-essential openjdk-8-jdk python3-pip ccache
cd /tmp
- wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O
gflags.tar.gz
- tar -xzf gflags.tar.gz && cd gflags-2.2.2
+ wget
https://github.com/Kitware/CMake/releases/download/v3.31.12/cmake-3.31.12-linux-$(uname
-m).sh; \
+ chmod +x cmake-3.31.12-linux-$(uname -m).sh; \
+ ./cmake-3.31.12-linux-$(uname -m).sh --skip-license
--prefix=/usr/local; \
+ rm -f cmake-3.31.12-linux-$(uname -m).sh;
+ wget https://github.com/gflags/gflags/archive/v2.3.0.tar.gz -O
gflags.tar.gz
+ tar -xzf gflags.tar.gz && cd gflags-2.3.0
cmake -B build \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DBUILD_SHARED_LIBS=ON \
@@ -109,8 +175,30 @@ jobs:
cmake . && make install -j
apt autoremove -y
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ export CCACHE_MAXSIZE=1G
+ mkdir -p $CCACHE_DIR
export VELOX_BUILD_SHARED=ON
cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
+ - name: Clean Ccache tmp before save
+ if: always()
+ run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+ - name: Save Ccache to Apache Stash
+ if: always()
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-ubuntu-${{ matrix.os }}-${{
hashFiles('ep/build-velox/src/**') }}
+ include-hidden-files: true
+ retention-days: '7'
+ - name: Run TPC-H
+ run: |
+ export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
+ echo "JAVA_HOME: $JAVA_HOME"
+ cd $GITHUB_WORKSPACE/tools/gluten-it
+ $TPCH_TEST
+
weekly-build-on-openeuler:
strategy:
@@ -124,16 +212,45 @@ jobs:
- uses: actions/checkout@v4
- name: Install dependencies
run: |
- dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
git perl
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
git perl ccache curl python3
echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+ - name: Install Stash restore dependencies # the stash action needs gh
and jq, absent from this container
+ run: |
+ case "$(uname -m)" in
+ x86_64) ARCH=amd64 ;;
+ aarch64) ARCH=arm64 ;;
+ esac
+ curl -fsSL
"https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz";
\
+ | tar -xz -C /usr/local --strip-components=1
"gh_2.63.2_linux_${ARCH}/bin/gh"
+ curl -fsSL -o /usr/local/bin/jq
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}";
+ chmod +x /usr/local/bin/jq
+ gh --version && jq --version
+ - name: Get Ccache from Apache Stash
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-openeuler-${{runner.arch}}-${{
hashFiles('ep/build-velox/src/**') }}
- name: Build
run: |
echo "JAVA_HOME: $JAVA_HOME"
+ export CCACHE_MAXSIZE=1G
+ mkdir -p $CCACHE_DIR
cd $GITHUB_WORKSPACE/
./dev/package.sh --spark_version=3.5
cd $GITHUB_WORKSPACE/tools/gluten-it
$GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
cp
$GITHUB_WORKSPACE/package/target/thirdparty-lib/gluten-thirdparty-lib-*.jar
package/target/lib/
+ - name: Clean Ccache tmp before save
+ if: always()
+ run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+ - name: Save Ccache to Apache Stash
+ if: always()
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-openeuler-${{runner.arch}}-${{
hashFiles('ep/build-velox/src/**') }}
+ include-hidden-files: true
+ retention-days: '7'
- name: Run TPC-H
run: |
echo "JAVA_HOME: $JAVA_HOME"
@@ -153,16 +270,45 @@ jobs:
- uses: actions/checkout@v4
- name: Install dependencies
run: |
- dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
git perl
+ dnf update -y && dnf install -y java-1.8.0-openjdk-devel sudo patch
git perl curl python3
echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
+ - name: Install Stash restore dependencies # the stash action needs gh
and jq, absent from this container
+ run: |
+ case "$(uname -m)" in
+ x86_64) ARCH=amd64 ;;
+ aarch64) ARCH=arm64 ;;
+ esac
+ curl -fsSL
"https://github.com/cli/cli/releases/download/v2.63.2/gh_2.63.2_linux_${ARCH}.tar.gz";
\
+ | tar -xz -C /usr/local --strip-components=1
"gh_2.63.2_linux_${ARCH}/bin/gh"
+ curl -fsSL -o /usr/local/bin/jq
"https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${ARCH}";
+ chmod +x /usr/local/bin/jq
+ gh --version && jq --version
+ - name: Get Ccache from Apache Stash
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-openeuler-vcpkg-${{runner.arch}}-${{
hashFiles('ep/build-velox/src/**') }}
- name: Build
run: |
echo "JAVA_HOME: $JAVA_HOME"
+ export CCACHE_MAXSIZE=1G
+ mkdir -p $CCACHE_DIR
cd $GITHUB_WORKSPACE/
./dev/vcpkg/setup-build-depends.sh
./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5
cd $GITHUB_WORKSPACE/tools/gluten-it
$GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
+ - name: Clean Ccache tmp before save
+ if: always()
+ run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+ - name: Save Ccache to Apache Stash
+ if: always()
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-openeuler-vcpkg-${{runner.arch}}-${{
hashFiles('ep/build-velox/src/**') }}
+ include-hidden-files: true
+ retention-days: '7'
- name: Run TPC-H
run: |
echo "JAVA_HOME: $JAVA_HOME"
@@ -184,6 +330,25 @@ jobs:
distribution: temurin
java-version: 17
- name: Install Dependencies
- run: brew install wget pkgconfig abseil ccache
+ run: brew install wget pkgconfig ccache
+ - name: Get Ccache from Apache Stash
+ uses:
apache/infrastructure-actions/stash/restore@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-macos-${{runner.arch}}-${{
hashFiles('ep/build-velox/src/**') }}
- name: Build
- run: ./dev/buildbundle-veloxbe.sh --run_setup_script=ON
--build_arrow=ON --build_tests=ON --build_benchmarks=ON --spark_version=3.5
+ run: |
+ export CCACHE_MAXSIZE=1G
+ mkdir -p $CCACHE_DIR
+ ./dev/buildbundle-veloxbe.sh --run_setup_script=ON --build_arrow=ON
--build_tests=ON --build_benchmarks=ON --spark_version=3.5
+ - name: Clean Ccache tmp before save
+ if: always()
+ run: rm -rf '${{ env.CCACHE_DIR }}/tmp'
+ - name: Save Ccache to Apache Stash
+ if: always()
+ uses:
apache/infrastructure-actions/stash/save@0ba14156c9f4c3cfbe4b0c9f36339ab0f8d81e53
+ with:
+ path: '${{ env.CCACHE_DIR }}'
+ key: ccache-weekly-macos-${{runner.arch}}-${{
hashFiles('ep/build-velox/src/**') }}
+ include-hidden-files: true
+ retention-days: '7'
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
index 64db4f23f8..a7d192b1f4 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos7.scala
@@ -38,7 +38,7 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader {
loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
loader.loadAndCreateLink("libdouble-conversion.so.1",
"libdouble-conversion.so")
loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so")
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
loader.loadAndCreateLink("libglog.so.0", "libglog.so")
loader.loadAndCreateLink("libre2.so.10", "libre2.so")
loader.loadAndCreateLink("libzstd.so.1", "libzstd.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
index 02faf8649f..d6b4b77ce8 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos8.scala
@@ -42,7 +42,7 @@ class SharedLibraryLoaderCentos8 extends SharedLibraryLoader {
loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so")
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
loader.loadAndCreateLink("libglog.so.1", "libglog.so")
loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so")
loader.loadAndCreateLink("libre2.so.0", "libre2.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
index cfb9ff17ef..3862ece676 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderCentos9.scala
@@ -38,7 +38,7 @@ class SharedLibraryLoaderCentos9 extends SharedLibraryLoader {
loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
loader.loadAndCreateLink("libglog.so.1", "libglog.so")
loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so")
loader.loadAndCreateLink("libre2.so.9", "libre2.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
index f06589eeed..f2d8a224eb 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian11.scala
@@ -33,7 +33,7 @@ class SharedLibraryLoaderDebian11 extends SharedLibraryLoader
{
loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so")
loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")
loader.loadAndCreateLink("libssh2.so.1", "libssh2.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
index 2ce752adf7..642402392a 100644
---
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderDebian12.scala
@@ -39,7 +39,7 @@ class SharedLibraryLoaderDebian12 extends SharedLibraryLoader
{
loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so")
loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so")
loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so")
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so")
loader.loadAndCreateLink("librtmp.so.1", "librtmp.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
index 0705539c40..a159994548 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderOpenEuler2403.scala
@@ -36,7 +36,7 @@ class SharedLibraryLoaderOpenEuler2403 extends
SharedLibraryLoader {
loader.loadAndCreateLink("libboost_context.so.1.84.0",
"libboost_context.so")
loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
loader.loadAndCreateLink("libglog.so.1", "libglog.so")
loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so")
loader.loadAndCreateLink("libidn.so.12", "libidn.so")
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
index 4786ee6710..14976c4913 100755
---
a/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
+++
b/backends-velox/src/main/scala/org/apache/gluten/spi/SharedLibraryLoaderUbuntu2204.scala
@@ -38,7 +38,7 @@ class SharedLibraryLoaderUbuntu2204 extends
SharedLibraryLoader {
loader.loadAndCreateLink("libcurl.so.4", "libcurl.so")
loader.loadAndCreateLink("libdouble-conversion.so.3",
"libdouble-conversion.so")
loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so")
- loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so")
+ loader.loadAndCreateLink("libgflags.so.2.3", "libgflags.so")
loader.loadAndCreateLink("libunwind.so.8", "libunwind.so")
loader.loadAndCreateLink("libglog.so.0", "libglog.so")
loader.loadAndCreateLink("libidn.so.12", "libidn.so")
diff --git a/dev/build-thirdparty.sh b/dev/build-thirdparty.sh
index 077999ccce..a3503e7645 100755
--- a/dev/build-thirdparty.sh
+++ b/dev/build-thirdparty.sh
@@ -31,7 +31,14 @@ function process_setup_ubuntu_2004 {
}
function process_setup_ubuntu_2204 {
- cp
/usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.2,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70}
$THIRDPARTY_LIB/
+ cp
/usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70}
$THIRDPARTY_LIB/
+ cp /usr/local/lib/libgflags.so.2.3 $THIRDPARTY_LIB/
+ find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/
\; 2>/dev/null
+ find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \;
2>/dev/null
+}
+function process_setup_ubuntu_2404 {
+ cp
/usr/lib/${ARCH}-linux-gnu/{libre2.so.10,libdouble-conversion.so.3,libglog.so.1,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libnghttp2.so.14,libldap.so.2,liblber.so.2,librtmp.so.1,libsasl2.so.2,libssh.so.4,libsodium.so.23,libssl.so.3,libcrypto.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1,libbrotlidec.so.1,libbrotlicommon.so.1,libidn2.so.0}
$THIRDPARTY_LIB/
+ cp /usr/local/lib/libgflags.so.2.3 $THIRDPARTY_LIB/
find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/
\; 2>/dev/null
find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \;
2>/dev/null
}
@@ -39,7 +46,7 @@ function process_setup_ubuntu_2204 {
function process_setup_centos_9 {
cp
/lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23}
$THIRDPARTY_LIB/
cp
/usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0}
$THIRDPARTY_LIB/
- cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/
+ cp /usr/local/lib/libgflags.so.2.3 $THIRDPARTY_LIB/
find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {}
$THIRDPARTY_LIB/ \; 2>/dev/null
find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {}
$THIRDPARTY_LIB/ \; 2>/dev/null
}
@@ -47,29 +54,23 @@ function process_setup_centos_9 {
function process_setup_centos_8 {
cp
/usr/lib64/{libre2.so.0,libdouble-conversion.so.3,libevent-2.1.so.6,libdwarf.so.1,libicudata.so.60,libicui18n.so.60,libicuuc.so.60,libsodium.so.23}
$THIRDPARTY_LIB/
cp
/usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0}
$THIRDPARTY_LIB/
- cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7}
$THIRDPARTY_LIB/
-}
-
-function process_setup_centos_7 {
- cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.0,libgeos.so.3.10.7}
$THIRDPARTY_LIB/
- cp
/usr/lib64/{libdouble-conversion.so.1,libevent-2.0.so.5,libzstd.so.1,libntlm.so.0,libgsasl.so.7,liblz4.so.1}
$THIRDPARTY_LIB/
- cp
/usr/local/lib/{libre2.so.10,libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_regex.so.1.84.0,libboost_atomic.so.1.84.0}
$THIRDPARTY_LIB/
+ cp /usr/local/lib64/{libgflags.so.2.3,libglog.so.1,libgeos.so.3.10.7}
$THIRDPARTY_LIB/
}
function process_setup_debian_11 {
- cp
/usr/lib/x86_64-linux-gnu/{libre2.so.9,libthrift-0.13.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libglog.so.0,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.67,libicuuc.so.67,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap_r-2.4.so.2,liblber-2.4.so.2,libbrotlidec.so.1,libicudata.so.67,libsasl2.so.2,libbrotlicommon.so.1}
$THIRDPARTY_LIB/
+ cp
/usr/lib/x86_64-linux-gnu/{libre2.so.9,libthrift-0.13.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.3,libglog.so.0,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.67,libicuuc.so.67,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap_r-2.4.so.2,liblber-2.4.so.2,libbrotlidec.so.1,libicudata.so.67,libsasl2.so.2,libbrotlicommon.so.1}
$THIRDPARTY_LIB/
cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
}
function process_setup_debian_12 {
- cp
/usr/lib/x86_64-linux-gnu/{libthrift-0.17.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.2,libglog.so.1,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.72,libicuuc.so.72,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap-2.5.so.0,liblber-2.5.so.0,libbrotlidec.so.1,libicudata.so.72,libsasl2.so.2,libbrotlicommon.so.1,libcrypto.so.3,libssl.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1}
$THIRDPARTY_LIB/
+ cp
/usr/lib/x86_64-linux-gnu/{libthrift-0.17.0.so,libdouble-conversion.so.3,libevent-2.1.so.7,libgflags.so.2.3,libglog.so.1,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libicui18n.so.72,libicuuc.so.72,libnghttp2.so.14,librtmp.so.1,libssh2.so.1,libpsl.so.5,libldap-2.5.so.0,liblber-2.5.so.0,libbrotlidec.so.1,libicudata.so.72,libsasl2.so.2,libbrotlicommon.so.1,libcrypto.so.3,libssl.so.3,libgssapi_krb5.so.2,libkrb5.so.3,libk5crypto.so.3,libkrb5support.so.0,libkeyutils.so.1}
$THIRDPARTY_LIB/
cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_regex.so.1.84.0}
$THIRDPARTY_LIB/
}
function process_setup_openeuler_24 {
cp
/usr/lib64/{libre2.so.11,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libgsasl.so.7,libicudata.so.74,libicui18n.so.74,libicuuc.so.74,libidn.so.12,libntlm.so.0,libsodium.so.26}
$THIRDPARTY_LIB/
cp
/usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0}
$THIRDPARTY_LIB/
- cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1} $THIRDPARTY_LIB/
+ cp /usr/local/lib64/{libgflags.so.2.3,libglog.so.1} $THIRDPARTY_LIB/
}
if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then
@@ -77,6 +78,8 @@ if [[ "$LINUX_OS" == "ubuntu" || "$LINUX_OS" == "pop" ]]; then
process_setup_ubuntu_2004
elif [ "$VERSION" == "22.04" ]; then
process_setup_ubuntu_2204
+ elif [ "$VERSION" == "24.04" ]; then
+ process_setup_ubuntu_2404
fi
elif [ "$LINUX_OS" == "centos" ]; then
if [ "$VERSION" == "9" ]; then
diff --git a/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf
b/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf
index 675a601c19..913f9ac7ee 100644
--- a/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf
+++ b/dev/docker/cudf/Dockerfile.centos-9-jdk17-cuda12.9-cudf
@@ -31,7 +31,7 @@ ENV CXX=/opt/rh/gcc-toolset-14/root/bin/g++
ENV
LD_LIBRARY_PATH=/opt/gluten/ep/build-velox/build/velox_ep/_build/release/_deps/curl-build/lib:$LD_LIBRARY_PATH
# Install dependencies and cudf related libraries to /usr
-RUN yum install -y sudo patch perl java-${JAVA_VERSION}-openjdk-devel && \
+RUN yum install -y sudo patch perl java-${JAVA_VERSION}-openjdk-devel
cuda-nvtx-12-9 && \
ln -sf /usr/local/bin/cmake /usr/bin && \
git clone --depth=1 https://github.com/apache/gluten /opt/gluten && \
cd /opt/gluten && \
diff --git a/ep/build-velox/src/build-velox.sh
b/ep/build-velox/src/build-velox.sh
index 3e279bf680..0c9e75b9ef 100755
--- a/ep/build-velox/src/build-velox.sh
+++ b/ep/build-velox/src/build-velox.sh
@@ -134,6 +134,10 @@ function compile {
COMPILE_OPTION="$COMPILE_OPTION -DCMAKE_IGNORE_PREFIX_PATH=/usr/local"
COMPILE_OPTION="$COMPILE_OPTION
-DCMAKE_IGNORE_PATH=/usr/local\;/usr/local/include\;/usr/local/lib\;/usr/local/lib/cmake"
COMPILE_OPTION="$COMPILE_OPTION
-DCMAKE_SYSTEM_IGNORE_PATH=/usr/local\;/usr/local/include\;/usr/local/lib\;/usr/local/lib/cmake"
+ # Force fmt to build from source via FetchContent: Homebrew's fmt config
file
+ # at /opt/homebrew/lib/cmake/fmt/ wins the AUTO find_package probe ahead of
+ # INSTALL_PREFIX, producing a version mismatch. BUNDLED skips find_package.
+ COMPILE_OPTION="$COMPILE_OPTION -Dfmt_SOURCE=BUNDLED"
fi
if [ $BUILD_TEST_UTILS == "ON" ]; then
COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TEST_UTILS=ON"
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index 72865dc94a..5afc1af735 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -65,7 +65,7 @@ if [ "$VELOX_HOME" == "" ]; then
fi
function process_setup_ubuntu {
- sed -i "s|run_and_time install_arrow||g" scripts/setup-centos9.sh
+ sed -i "s|run_and_time install_arrow||g" scripts/setup-ubuntu.sh
echo "Using setup script from Velox"
}
diff --git a/ep/build-velox/src/setup-centos8.sh
b/ep/build-velox/src/setup-centos8.sh
index 3aa6f922a9..95b0e5586f 100755
--- a/ep/build-velox/src/setup-centos8.sh
+++ b/ep/build-velox/src/setup-centos8.sh
@@ -102,7 +102,7 @@ function install_xxhash {
function install_gflags {
# Remove an older version if present.
dnf remove -y gflags
- wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+ wget_and_untar https://github.com/gflags/gflags/archive/v2.3.0.tar.gz gflags
cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
}
diff --git a/ep/build-velox/src/setup-openeuler24.sh
b/ep/build-velox/src/setup-openeuler24.sh
index 80a66917b2..39fd196e73 100755
--- a/ep/build-velox/src/setup-openeuler24.sh
+++ b/ep/build-velox/src/setup-openeuler24.sh
@@ -83,7 +83,7 @@ function install_velox_deps_from_dnf {
function install_gflags {
# Remove an older version if present.
dnf remove -y gflags
- wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
+ wget_and_untar https://github.com/gflags/gflags/archive/v2.3.0.tar.gz gflags
cmake_install_dir gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
