This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new eb2f5ac006 [GLUTEN-6887][VL] Daily Update Velox Version (2026_10_15)
(#13289)
eb2f5ac006 is described below
commit eb2f5ac006730d41089f3c6556f58fe0c31d025a
Author: Gluten Performance Bot
<[email protected]>
AuthorDate: Fri Oct 16 09:20:59 2026 +0100
[GLUTEN-6887][VL] Daily Update Velox Version (2026_10_15) (#13289)
* [GLUTEN-6887][VL] Daily Update Velox Version (dft-2026_10_15)
Upstream Velox's New Commits:
41498076c by Andrii Rosa, feat(config): Register
task_partitioned_writer_count session property
1ae10f98c by Karthikeyan, refactor(functions): Move calendar functions to a
device-compatible header
4d0721399 by RindsSchei225e, test(nimble): Let the writer fuzzer force
Huffman (#18929)
f0276a8b6 by Andrii Rosa, fix: Paste test name in disabled
DEBUG_ONLY_CO_TEST_F
e162d0b21 by Muhammad Haseeb, feat(cudf): Velox-cuDF table scan ignores
missing files when specified
9e111b510 by xiaodouchen, fix(type): Stop integer filter loops at INT64_MAX
3e035c893 by Karthikeyan, fix(type): Reject out-of-range time zone ids in
locateZone
409cf8d63 by Wechar Yu, fix: Unify Presto serialized-size estimation
ff8dfcdfe by adheer-araokar, fix(memory): Fix free capacity underflow in
shrink
a2ff335a3 by Ke, refactor(fuzzer): Remove Nimble stats from
TableEvolutionFuzzer (#19559)
9db3317a9 by Pedro Pedreira, feat(vector): Add
`FlatMapVector::getKeyChannels()` (#19537)
53ece251d by adheer-araokar, fix(memory): Fix stray placeholder in free
capacity error
b0706264b by Karthikeyan, feat(decimal): Make decimal call() bodies
device-compatible
3286f4d29 by kevinwilfong, test: Reduce io_uring memory in DataInput tests
05cc69aa0 by Karthikeyan, perf(functions): Compute round()'s scale factor
once for constant digits
Signed-off-by: glutenperfbot <[email protected]>
* misc fix rhel setup
Signed-off-by: Yuan <[email protected]>
* remove centos7 scripts
Signed-off-by: Yuan <[email protected]>
* remove centos7 script
Signed-off-by: Yuan <[email protected]>
* fix java setup on kafka
Signed-off-by: Yuan <[email protected]>
---------
Signed-off-by: glutenperfbot <[email protected]>
Signed-off-by: Yuan <[email protected]>
Co-authored-by: glutenperfbot <[email protected]>
Co-authored-by: Yuan <[email protected]>
---
.github/workflows/docker_image.yml | 25 ---
.github/workflows/velox_kafka.yml | 17 +-
dev/build-helper-functions.sh | 1 +
dev/docker/Dockerfile.centos7-gcc13 | 194 -----------------
dev/docker/Dockerfile.centos7-gcc13-static-build | 56 -----
dev/docker/Dockerfile.centos7-static-build | 54 -----
ep/build-velox/src/get-velox.sh | 4 +-
ep/build-velox/src/setup-centos7.sh | 263 -----------------------
ep/build-velox/src/setup-rhel.sh | 8 +-
9 files changed, 18 insertions(+), 604 deletions(-)
diff --git a/.github/workflows/docker_image.yml
b/.github/workflows/docker_image.yml
index 8f7da4bc2a..8d5e6271e0 100644
--- a/.github/workflows/docker_image.yml
+++ b/.github/workflows/docker_image.yml
@@ -33,31 +33,6 @@ env:
DOCKERHUB_REPO: apache/gluten
jobs:
- build-vcpkg-centos-7-gcc13:
- if: ${{ startsWith(github.repository, 'apache/') }}
- runs-on: ubuntu-latest
-
- steps:
- - name: Checkout repository
- uses: actions/checkout@v7
-
- - name: Set up Docker Buildx
- uses:
docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c
-
- - name: Login to Docker Hub
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f
- with:
- username: ${{ secrets.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
- - name: Build and push Docker image
- uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
- with:
- context: .
- file: dev/docker/Dockerfile.centos7-gcc13-static-build
- push: true
- tags: ${{ env.DOCKERHUB_REPO }}:vcpkg-centos-7-gcc13
-
build-ubuntu22-m2-cache:
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
diff --git a/.github/workflows/velox_kafka.yml
b/.github/workflows/velox_kafka.yml
index 443f1e7069..33a5303dbe 100644
--- a/.github/workflows/velox_kafka.yml
+++ b/.github/workflows/velox_kafka.yml
@@ -127,11 +127,12 @@ jobs:
--health-retries 12
steps:
- uses: actions/checkout@v7
- - uses: actions/setup-java@v4
- with:
- distribution: temurin
- java-version: '11'
- cache: maven
+ - name: Setup java
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y openjdk-17-jdk
+ echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64" >> $GITHUB_ENV
+ echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
- uses: actions/download-artifact@v8
with:
name: velox-kafka-native-${{ github.sha }}
@@ -139,18 +140,18 @@ jobs:
- name: Install Spark 3.5 modules with Kafka support
run: |
./build/mvn -B -ntp install -Pbackends-velox -Pspark-3.5
-Pscala-2.12 \
- -Pjava-11 -Pkafka -Pspark-ut -Pfast-build -DskipTests
+ -Pjava-17 -Pkafka -Pspark-ut -Pfast-build -DskipTests
# TODO: Re-enable when VeloxKafkaScanValidationSuite is added.
# - name: Check Kafka schema validation without a broker
# run: |
# ./build/mvn -B -ntp test -pl gluten-ut/spark35 -Pbackends-velox
-Pspark-3.5 \
- # -Pscala-2.12 -Pjava-11 -Pspark-ut -Pfast-build \
+ # -Pscala-2.12 -Pjava-17 -Pspark-ut -Pfast-build \
#
-Dsuites=org.apache.gluten.execution.VeloxKafkaScanValidationSuite
# TODO: Re-enable when VeloxGlutenKafkaScanSuite is added.
# - name: Check native micro-batch reads and headers fallback
# run: |
# ./build/mvn -B -ntp test -pl backends-velox -Pbackends-velox
-Pspark-3.5 \
- # -Pscala-2.12 -Pjava-11 -Pkafka -Pfast-build \
+ # -Pscala-2.12 -Pjava-17 -Pkafka -Pfast-build \
#
-Dsuites=org.apache.gluten.execution.kafka.VeloxGlutenKafkaScanSuite
- name: Upload test results
if: always()
diff --git a/dev/build-helper-functions.sh b/dev/build-helper-functions.sh
index 2f80731d60..565fba7447 100644
--- a/dev/build-helper-functions.sh
+++ b/dev/build-helper-functions.sh
@@ -312,6 +312,7 @@ function setup_linux {
"$LINUX_DISTRIBUTION" == "almalinux" ]]; then
case "${LINUX_VERSION_ID%%.*}" in
9)
+ source $GLUTEN_VELOX_SCRIPT_HOME/setup-rhel.sh
$GLUTEN_VELOX_SCRIPT_HOME/setup-rhel.sh ;;
*)
echo "Unsupported ${LINUX_DISTRIBUTION} version: $LINUX_VERSION_ID"
diff --git a/dev/docker/Dockerfile.centos7-gcc13
b/dev/docker/Dockerfile.centos7-gcc13
deleted file mode 100644
index 0eeb9545f7..0000000000
--- a/dev/docker/Dockerfile.centos7-gcc13
+++ /dev/null
@@ -1,194 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# copied from https://github.com/jeromerobert/centos7-gcc13
-FROM centos:7 as base0
-
-FROM base0 as base
-RUN /usr/bin/sed -i -e "s|mirrorlist=|#mirrorlist=|g"
/etc/yum.repos.d/CentOS-*; sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-*;
-
-
-# Git
-FROM base as git
-RUN curl -kLO
https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.42.0.tar.xz
-RUN sha256sum *.tar.xz
-RUN echo "3278210e9fd2994b8484dd7e3ddd9ea8b940ef52170cdb606daa94d887c93b0d "
*.tar.* | sha256sum -c -
-RUN tar xf *.tar.xz
-WORKDIR git-2.42.0
-RUN yum -y install gcc make zlib-devel gettext libcurl-devel
-RUN ./configure --prefix=/usr/local
-RUN make -j $(nproc)
-RUN make NO_INSTALL_HARDLINKS=YesPlease install
-
-# Binutils
-FROM base as binutils
-RUN curl -LO https://ftp.gnu.org/gnu/binutils/binutils-2.41.tar.xz
-RUN sha256sum binutils*.tar.*
-RUN echo "ae9a5789e23459e59606e6714723f2d3ffc31c03174191ef0d015bdf06007450 "
binutils*.tar.* | sha256sum -c -
-RUN tar xf binutils*.tar.*
-WORKDIR build-binutils
-RUN yum -y install gcc-c++ make zlib-devel gettext bison texinfo
-RUN ../binutils*/configure
-RUN make -j$(nproc)
-RUN make install
-
-# GMP
-FROM base as gmp
-RUN curl -LO https://ftp.gnu.org/gnu/gmp/gmp-6.2.1.tar.xz
-RUN sha256sum gmp*.tar.*
-RUN echo "fd4829912cddd12f84181c3451cc752be224643e87fac497b69edddadc49b4f2 "
gmp*.tar.* | sha256sum -c -
-RUN tar xf gmp*.tar.*
-WORKDIR build-gmp
-RUN yum -y install gcc make m4
-RUN ../gmp*/configure
-RUN make -j$(nproc)
-RUN make install
-
-# mpfr
-FROM base as mpfr
-RUN curl -LO https://ftp.gnu.org/gnu/mpfr/mpfr-4.2.0.tar.xz
-RUN sha256sum mpfr*.tar.*
-RUN echo "06a378df13501248c1b2db5aa977a2c8126ae849a9d9b7be2546fb4a9c26d993 "
mpfr*.tar.* | sha256sum -c -
-RUN tar xf mpfr*.tar.*
-WORKDIR build-mpfr
-RUN yum -y install gcc make
-COPY --from=gmp /usr/local/ /usr/local/
-RUN ../mpfr*/configure
-RUN make -j$(nproc)
-RUN before=$(find /usr/local -type f ; find /usr/local -type l ; find
/usr/local -type d -empty) ; \
- make install ; rm -rf $before
-
-# mpc
-FROM base as mpc
-RUN curl -LO https://ftp.gnu.org/gnu/mpc/mpc-1.3.1.tar.gz
-RUN sha256sum mpc*.tar.*
-RUN echo "ab642492f5cf882b74aa0cb730cd410a81edcdbec895183ce930e706c1c759b8 "
mpc*.tar.* | sha256sum -c -
-RUN tar xf mpc*.tar.*
-WORKDIR build-mpc
-RUN yum -y install gcc make
-COPY --from=mpfr /usr/local/ /usr/local/
-COPY --from=gmp /usr/local/ /usr/local/
-RUN ../mpc*/configure
-RUN make -j$(nproc)
-RUN before=$(find /usr/local -type f ; find /usr/local -type l ; find
/usr/local -type d -empty) ; \
- make install ; rm -rf $before
-
-# OpenSSL
-FROM base as openssl
-RUN curl -LO https://www.openssl.org/source/openssl-3.0.10.tar.gz
-RUN sha256sum openssl*.tar.gz
-RUN echo "1761d4f5b13a1028b9b6f3d4b8e17feb0cedc9370f6afe61d7193d2cdce83323 "
openssl*.tar.gz | sha256sum -c -
-RUN tar xf openssl*.tar.gz
-WORKDIR openssl-3.0.10
-RUN yum -y install gcc make perl-IPC-Cmd
-RUN ./config --prefix=/usr/local --openssldir=/usr/local shared
-RUN make -j$(nproc)
-RUN make install_sw install_ssldirs
-
-# cmake
-FROM base as cmake
-RUN curl -kLO
https://github.com/Kitware/CMake/releases/download/v3.27.4/cmake-3.27.4.tar.gz
-RUN sha256sum cmake-*.tar.gz
-RUN echo "0a905ca8635ca81aa152e123bdde7e54cbe764fdd9a70d62af44cad8b92967af "
cmake-*.tar.gz | sha256sum -c -
-RUN tar xf *.tar.gz
-WORKDIR cmake-3.27.4
-RUN yum -y install gcc-c++ make
-COPY --from=openssl /usr/local/ /usr/local/
-RUN ./bootstrap --parallel=$(nproc)
-RUN make -j$(nproc)
-RUN before=$(find /usr/local -type f ; find /usr/local -type l ; find
/usr/local -type d -empty) ; \
- make install ; rm -rf $before
-
-# Autotools
-FROM base as autotools
-RUN curl -kLO https://ftp.gnu.org/gnu/autoconf/autoconf-2.71.tar.xz
-RUN curl -kLO https://ftp.gnu.org/gnu/automake/automake-1.16.5.tar.xz
-RUN curl -kLO https://ftp.gnu.org/gnu/libtool/libtool-2.4.7.tar.xz
-RUN sha256sum *.tar.*
-RUN echo "f14c83cfebcc9427f2c3cea7258bd90df972d92eb26752da4ddad81c87a0faa4 "
autoconf*.tar.* | sha256sum -c -
-RUN echo "f01d58cd6d9d77fbdca9eb4bbd5ead1988228fdb73d6f7a201f5f8d6b118b469 "
automake*.tar.* | sha256sum -c -
-RUN echo "4f7f217f057ce655ff22559ad221a0fd8ef84ad1fc5fcb6990cecc333aa1635d "
libtool*.tar.* | sha256sum -c -
-RUN tar xf autoconf*.tar.*
-RUN tar xf automake*.tar.*
-RUN tar xf libtool*.tar.*
-RUN yum -y install m4 make perl-Data-Dumper perl-Thread-Queue gcc
-WORKDIR /autoconf-2.71
-RUN ./configure
-RUN make -j$(nproc)
-RUN make install
-WORKDIR /automake-1.16.5
-RUN ./configure
-RUN make -j$(nproc)
-RUN make install
-WORKDIR /libtool-2.4.7
-RUN ./configure
-RUN make -j$(nproc)
-RUN make install
-
-#hwloc
-FROM base as hwloc
-RUN curl -kLO
https://download.open-mpi.org/release/hwloc/v2.9/hwloc-2.9.2.tar.gz
-RUN sha256sum *.tar.*
-RUN echo "ffb554d5735e0e0a19d1fd4b2b86e771d3b58b2d97f257eedacae67ade5054b3 "
*.tar.* | sha256sum -c -
-RUN tar xf *.tar.*
-WORKDIR hwloc-2.9.2
-RUN yum -y install gcc make
-RUN ./configure
-RUN make -j$(nproc)
-RUN make install
-
-# gcc
-FROM base as gcc
-RUN curl -kLO
https://sourceware.org/pub/gcc/releases/gcc-13.2.0/gcc-13.2.0.tar.xz
-RUN sha256sum *.tar.*
-RUN echo "e275e76442a6067341a27f04c5c6b83d8613144004c0413528863dc6b5c743da "
*.tar.* | sha256sum -c -
-RUN tar xf *.tar.*
-WORKDIR build-gcc
-RUN yum -y install gcc-c++ make
-COPY --from=binutils /usr/local/ /usr/local/
-COPY --from=gmp /usr/local/ /usr/local/
-COPY --from=mpfr /usr/local/ /usr/local/
-COPY --from=mpc /usr/local/ /usr/local/
-RUN echo -e '/usr/local/lib\n/usr/local/lib64' > /etc/ld.so.conf.d/local.conf
&& ldconfig
-# Use symlinks instead of hardlinks because hardlinks get duplicated in the
stripping stage
-RUN sed -i 's/LN=@LN@/LN=@LN_S@/g' /gcc-*/gcc/Makefile.in
-RUN ../gcc-*/configure --enable-languages=c,c++,fortran --disable-multilib
-RUN make -j$(nproc)
-RUN before=$(find /usr/local -type f ; find /usr/local -type l) ; \
- make install ; rm -rf $before $(find /usr/local -type d -empty)
-
-# Intermediate stage for stripping
-FROM base as stripped
-RUN yum -y install file
-COPY --from=binutils /usr/local/ /usr/local/
-COPY --from=gmp /usr/local/ /usr/local/
-COPY --from=mpfr /usr/local/ /usr/local/
-COPY --from=mpc /usr/local/ /usr/local/
-COPY --from=git /usr/local/ /usr/local/
-COPY --from=openssl /usr/local/ /usr/local/
-COPY --from=cmake /usr/local/ /usr/local/
-COPY --from=autotools /usr/local/ /usr/local/
-COPY --from=hwloc /usr/local/ /usr/local/
-COPY --from=gcc /usr/local/ /usr/local/
-RUN find /usr/local ! -name '*.o' -type f -exec sh -c "file -b {} | grep -Eq
'^ELF.*, not stripped' && strip {}" \;
-
-# Final stage
-FROM base as final
-RUN yum install -y glibc-devel make zlib-devel swig chrpath libffi-devel
perl-Data-Dumper bzip2 m4 \
- perl-Thread-Queue patch mesa-libGLU-devel libXt-devel unzip libXtst
libXrender libXi \
- libmount-devel gettext pcre-devel
-COPY --from=stripped /usr/local /usr/local
-RUN echo -e '/usr/local/lib\n/usr/local/lib64' > /etc/ld.so.conf.d/local.conf
&& ldconfig
-ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig
diff --git a/dev/docker/Dockerfile.centos7-gcc13-static-build
b/dev/docker/Dockerfile.centos7-gcc13-static-build
deleted file mode 100644
index f3292c1c81..0000000000
--- a/dev/docker/Dockerfile.centos7-gcc13-static-build
+++ /dev/null
@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM inteldpo/gluten:centos7_gcc13
-
-ARG GLUTEN_REPO=https://github.com/apache/gluten
-ARG GLUTEN_BRANCH=main
-
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
-ENV PATH=$JAVA_HOME/bin:$PATH
-
-# An actual path used for vcpkg cache.
-ENV VCPKG_PATH=/var/cache/vcpkg
-# Set vcpkg cache path.
-ENV VCPKG_BINARY_SOURCES=clear;files,${VCPKG_PATH},readwrite
-
-
-RUN set -ex; \
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* ||
true; \
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true; \
- yum install -y epel-release centos-release-scl; \
- rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f; \
- sed -i \
- -e 's/^mirrorlist/#mirrorlist/' \
- -e 's/^#baseurl/baseurl/' \
- -e 's/mirror\.centos\.org/vault.centos.org/' \
- /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo; \
- yum install -y git patch sudo java-1.8.0-openjdk-devel ccache curl; \
- git clone --depth=1 --branch "${GLUTEN_BRANCH}" "${GLUTEN_REPO}"
/opt/gluten; \
- echo "check_certificate = off" >> ~/.wgetrc; \
- cd /opt/gluten; \
- bash ./dev/install-linux-socket-header.sh; \
- bash ./dev/vcpkg/setup-build-depends.sh; \
- yum remove gcc -y && yum clean all; \
- yes | cp -rf /usr/share/aclocal/* /usr/local/share/aclocal/; \
- mkdir -p ${VCPKG_PATH}; \
- echo "vcpkg installs the native dependencies and Maven resolves Java
dependencies."; \
- cd /opt/gluten; \
- bash ./dev/vcpkg/populate-static-image-cache.sh; \
- ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
- dnf clean all; \
- rm -rf /opt/gluten; \
- rm -rf /root/.cache/ccache;
-
diff --git a/dev/docker/Dockerfile.centos7-static-build
b/dev/docker/Dockerfile.centos7-static-build
deleted file mode 100644
index 25b7d798e8..0000000000
--- a/dev/docker/Dockerfile.centos7-static-build
+++ /dev/null
@@ -1,54 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM centos:7
-
-ARG GLUTEN_REPO=https://github.com/apache/gluten
-ARG GLUTEN_BRANCH=main
-
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk
-ENV PATH=$JAVA_HOME/bin:$PATH
-
-# An actual path used for vcpkg cache.
-ENV VCPKG_PATH=/var/cache/vcpkg
-# Set vcpkg cache path.
-ENV VCPKG_BINARY_SOURCES=clear;files,${VCPKG_PATH},readwrite
-
-
-RUN set -ex; \
- sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* ||
true; \
- sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-* || true; \
- yum install -y epel-release centos-release-scl; \
- rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f; \
- sed -i \
- -e 's/^mirrorlist/#mirrorlist/' \
- -e 's/^#baseurl/baseurl/' \
- -e 's/mirror\.centos\.org/vault.centos.org/' \
- /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo; \
- yum install -y git patch sudo java-1.8.0-openjdk-devel ccache; \
- git clone --depth=1 --branch "${GLUTEN_BRANCH}" "${GLUTEN_REPO}"
/opt/gluten; \
- echo "check_certificate = off" >> ~/.wgetrc; \
- cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
- mkdir -p ${VCPKG_PATH}; \
- echo "vcpkg installs the native dependencies and Maven resolves Java
dependencies."; \
- cd /opt/gluten; \
- source /opt/rh/devtoolset-11/enable; \
- source /opt/rh/rh-git227/enable; \
- bash ./dev/vcpkg/populate-static-image-cache.sh; \
- ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta
-Pspark-3.5 -DskipTests; \
- dnf clean all; \
- rm -rf /opt/gluten; \
- rm -rf /root/.cache/ccache;
-
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index d4bbf04efd..63671d7643 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -18,8 +18,8 @@ set -exu
CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd)
VELOX_REPO=https://github.com/IBM/velox.git
-VELOX_BRANCH=dft-2026_10_14
-VELOX_ENHANCED_BRANCH=ibm-2026_10_14
+VELOX_BRANCH=dft-2026_10_15
+VELOX_ENHANCED_BRANCH=ibm-2026_10_15
VELOX_HOME=""
RUN_SETUP_SCRIPT=ON
ENABLE_ENHANCED_FEATURES=OFF
diff --git a/ep/build-velox/src/setup-centos7.sh
b/ep/build-velox/src/setup-centos7.sh
deleted file mode 100755
index 3fe457b326..0000000000
--- a/ep/build-velox/src/setup-centos7.sh
+++ /dev/null
@@ -1,263 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -efx -o pipefail
-# Some of the packages must be build with the same compiler flags
-# so that some low level types are the same size. Also, disable warnings.
-SCRIPTDIR=./scripts
-source $SCRIPTDIR/setup-helper-functions.sh
-DEPENDENCY_DIR=${DEPENDENCY_DIR:-/tmp/velox-deps}
-CPU_TARGET="${CPU_TARGET:-avx}"
-NPROC=$(getconf _NPROCESSORS_ONLN)
-FMT_VERSION=10.1.1
-BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
-BUILD_GEOS="${BUILD_GEOS:-true}"
-export CFLAGS=$(get_cxx_flags $CPU_TARGET) # Used by LZO.
-export CXXFLAGS=$CFLAGS # Used by boost.
-export CPPFLAGS=$CFLAGS # Used by LZO.
-export
PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
-FB_OS_VERSION="v2024.07.01.00"
-GEOS_VERSION="3.10.7"
-
-# shellcheck disable=SC2037
-SUDO="sudo -E"
-
-function run_and_time {
- time "$@"
- { echo "+ Finished running $*"; } 2> /dev/null
-}
-
-function dnf_install {
- $SUDO dnf install -y -q --setopt=install_weak_deps=False "$@"
-}
-
-function yum_install {
- $SUDO yum install -y "$@"
-}
-
-function install_cmake {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar https://cmake.org/files/v3.28/cmake-3.28.3.tar.gz cmake-3
- cd cmake-3
- ./bootstrap --prefix=/usr/local
- make -j$(nproc)
- $SUDO make install
- cmake --version
-}
-
-function install_ninja {
- cd "${DEPENDENCY_DIR}"
- github_checkout ninja-build/ninja v1.11.1 --depth 1
- cd ninja
- ./configure.py --bootstrap
- cmake -Bbuild-cmake
- cmake --build build-cmake
- $SUDO cp ninja /usr/local/bin/
-}
-
-function install_folly {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz
folly
- cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
-DFOLLY_NO_EXCEPTION_TRACER=ON
-}
-
-# not used
-function install_conda {
- cd "${DEPENDENCY_DIR}"
- mkdir -p conda && pushd conda
- wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
- MINICONDA_PATH=/opt/miniconda-for-velox
- bash Miniconda3-latest-Linux-x86_64.sh -b -u $MINICONDA_PATH
- popd
-}
-
-function install_openssl {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz
openssl
- cd openssl
- ./config no-shared
- make depend
- make
- $SUDO make install
-}
-
-function install_gflags {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
- cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
-}
-
-function install_glog {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar https://github.com/google/glog/archive/v0.5.0.tar.gz glog
- cmake_install glog -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON
-DCMAKE_INSTALL_PREFIX:PATH=/usr/local
-}
-
-function install_snappy {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
- cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
-}
-
-function install_dwarf {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/davea42/libdwarf-code/archive/refs/tags/20210528.tar.gz dwarf
- pushd dwarf
- #local
URL=https://github.com/davea42/libdwarf-code/releases/download/v0.5.0/libdwarf-0.5.0.tar.xz
- #local DIR=dwarf
- #mkdir -p "${DIR}"
- #wget -q --max-redirect 3 "${URL}"
- #tar -xf libdwarf-0.5.0.tar.xz -C "${DIR}"
- #cd dwarf/libdwarf-0.5.0
- ./configure --enable-shared=no
- make
- make check
- $SUDO make install
- popd
-}
-
-function install_re2 {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/google/re2/archive/refs/tags/2023-03-01.tar.gz re2
- cd re2
- $SUDO make install
-}
-
-function install_flex {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz flex
- cd flex
- ./autogen.sh
- ./configure
- $SUDO make install
-}
-
-function install_lzo {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
http://www.oberhumer.com/opensource/lzo/download/lzo-2.10.tar.gz lzo
- cd lzo
- ./configure --prefix=/usr/local --enable-shared --disable-static
--docdir=/usr/local/share/doc/lzo-2.10
- make "-j$(nproc)"
- $SUDO make install
-}
-
-function install_boost {
- # Remove old version.
- sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_*
/opt/rh/devtoolset-11/root/usr/lib64/dyninst/libboost_*
- sudo rm -rf "${DEPENDENCY_DIR}"/boost/ /usr/local/include/boost/
/usr/local/lib/cmake/Boost-1.72.0/
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz
boost
- cd boost
- ./bootstrap.sh --prefix=/usr/local --with-python=/usr/bin/python3
--with-python-root=/usr/lib/python3.6 --without-libraries=python
- $SUDO ./b2 "-j$(nproc)" -d0 install threading=multi
-}
-
-function install_protobuf {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz
protobuf
- pushd protobuf
- ./configure CXXFLAGS="-fPIC" --prefix=/usr/local
- make "-j$(nproc)"
- $SUDO make install
- popd
-}
-
-function install_gtest {
- cd "${DEPENDENCY_DIR}"
- wget_and_untar
https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz
googletest
- pushd googletest
- mkdir -p build && cd build && cmake -DBUILD_GTEST=ON -DBUILD_GMOCK=ON
-DINSTALL_GTEST=ON -DINSTALL_GMOCK=ON -DBUILD_SHARED_LIBS=ON ..
- make "-j$(nproc)"
- $SUDO make install
- popd
-}
-
-function install_fmt {
- wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
- rm -rf /usr/local/lib64/libfmt.a
- rm -rf /usr/local/lib64/cmake/fmt
- rm -rf /usr/local/include/fmt
- cmake_install fmt -DFMT_TEST=OFF
-}
-
-function install_duckdb {
- if $BUILD_DUCKDB ; then
- echo 'Building DuckDB'
- wget_and_untar
https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
- cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF
-DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF
-DCMAKE_BUILD_TYPE=Release
- fi
-}
-
-function install_geos {
- if [[ "$BUILD_GEOS" == "true" ]]; then
- wget_and_untar
https://github.com/libgeos/geos/archive/${GEOS_VERSION}.tar.gz geos
- cmake_install_dir geos -DBUILD_TESTING=OFF
- fi
-}
-
-function install_prerequisites {
- run_and_time install_lzo
- run_and_time install_boost
- run_and_time install_re2
- run_and_time install_flex
- run_and_time install_openssl
- run_and_time install_gflags
- run_and_time install_glog
- run_and_time install_snappy
- run_and_time install_dwarf
-}
-
-function install_velox_deps {
- run_and_time install_fmt
- run_and_time install_folly
- run_and_time install_protobuf
- run_and_time install_gtest
- run_and_time install_duckdb
- run_and_time install_geos
-}
-
-$SUDO dnf makecache
-
-# dnf install dependency libraries
-dnf_install epel-release dnf-plugins-core # For ccache, ninja
-dnf_install ccache wget which libevent-devel \
- yasm \
- openssl-devel libzstd-devel lz4-devel double-conversion-devel \
- curl-devel libxml2-devel libgsasl-devel libuuid-devel patch libicu-devel
tzdata
-
-$SUDO dnf remove -y gflags
-
-# Required for Thrift
-dnf_install autoconf automake libtool bison python3 python3-devel
-
-# Required for build flex
-dnf_install gettext-devel texinfo help2man
-
-$SUDO yum makecache
-yum_install centos-release-scl
-yum_install devtoolset-11
-source /opt/rh/devtoolset-11/enable || exit 1
-gcc --version
-set -u
-
-# Build from source
-[ -d "$DEPENDENCY_DIR" ] || mkdir -p "$DEPENDENCY_DIR"
-
-run_and_time install_cmake
-run_and_time install_ninja
-
-install_prerequisites
-install_velox_deps
diff --git a/ep/build-velox/src/setup-rhel.sh b/ep/build-velox/src/setup-rhel.sh
index e0780d2b1b..0a2c55647c 100755
--- a/ep/build-velox/src/setup-rhel.sh
+++ b/ep/build-velox/src/setup-rhel.sh
@@ -203,8 +203,12 @@ function install_c-ares {
function install_s3 {
install_aws_deps
- local MINIO_OS="linux"
- install_minio ${MINIO_OS}
+ # Velox renamed install_minio to install_silo; support both.
+ if declare -F install_silo > /dev/null; then
+ install_silo linux
+ else
+ install_minio linux
+ fi
}
function install_gcs {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
