This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new a9e0ec58e3 [VL] Add optional Kafka build dependency for Velox backend
(#13182)
a9e0ec58e3 is described below
commit a9e0ec58e3bc45dee1d5605136aeb82c2d955816
Author: Reema <[email protected]>
AuthorDate: Tue Oct 13 09:26:22 2026 +0300
[VL] Add optional Kafka build dependency for Velox backend (#13182)
---
.github/workflows/velox_kafka.yml | 162 ++++++++++++++++++++++++++++
cpp/CMakeLists.txt | 1 +
cpp/velox/CMakeLists.txt | 40 +++++++
dev/builddeps-veloxbe.sh | 8 +-
dev/ci-velox-buildshared-centos-9.sh | 5 +-
dev/docker/Dockerfile.centos8-dynamic-build | 2 +-
dev/docker/Dockerfile.centos9-dynamic-build | 2 +-
dev/vcpkg/init.sh | 8 ++
dev/vcpkg/vcpkg.json | 6 ++
docs/get-started/build-guide.md | 8 ++
ep/build-velox/src/get-velox.sh | 9 ++
ep/build-velox/src/setup-centos7.sh | 2 +-
ep/build-velox/src/setup-centos8.sh | 2 +-
ep/build-velox/src/setup-openeuler24.sh | 2 +-
ep/build-velox/src/setup-rhel.sh | 2 +-
15 files changed, 251 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/velox_kafka.yml
b/.github/workflows/velox_kafka.yml
new file mode 100644
index 0000000000..443f1e7069
--- /dev/null
+++ b/.github/workflows/velox_kafka.yml
@@ -0,0 +1,162 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Velox Kafka
+
+on:
+ pull_request:
+ paths:
+ - '.github/workflows/velox_kafka.yml'
+ - 'pom.xml'
+ - 'backends-velox/**'
+ - 'gluten-kafka/**'
+ - 'gluten-core/**'
+ - 'gluten-substrait/**'
+ - 'gluten-ut/**'
+ - 'shims/**'
+ - 'cpp/**'
+ - 'ep/build-velox/**'
+ - 'dev/**'
+ - 'build/mvn'
+ workflow_dispatch:
+
+permissions:
+ contents: read
+ actions: read
+
+concurrency:
+ group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
+ cancel-in-progress: true
+
+env:
+ CCACHE_DIR: ${{ github.workspace }}/.ccache
+ PROMPT_ALWAYS_RESPOND: 'y'
+
+jobs:
+ build-native:
+ runs-on: ubuntu-22.04
+ container: apache/gluten:vcpkg-almalinux-8-gcc13
+ timeout-minutes: 180
+ steps:
+ - uses: actions/checkout@v7
+ - name: Install build tools
+ run: |
+ dnf install -y jq bison tzdata
+ dnf module -y install python39
+ alternatives --set python3 /usr/bin/python3.9
+ dnf config-manager --add-repo
https://cli.github.com/packages/rpm/gh-cli.repo
+ dnf install -y gh --setopt=gpgcheck=0
+ - name: Get Ccache from Apache Stash
+ id: ccache-stash
+ uses:
apache/infrastructure-actions/stash/restore@69afc125e535c4c41e7f1b7470f583087e0f344b
+ with:
+ path: ${{ env.CCACHE_DIR }}
+ key: ccache-almalinux8-release-default-${{
hashFiles('ep/build-velox/src/**') }}
+ - name: Get Ccache from actions/cache (fallback on Stash miss)
+ if: steps.ccache-stash.outputs.stash-hit != 'true'
+ uses: actions/cache/restore@v4
+ with:
+ path: ${{ env.CCACHE_DIR }}
+ key: ccache-almalinux8-release-default-${{ github.sha }}
+ restore-keys: |
+ ccache-almalinux8-release-default
+ - name: Build Kafka-enabled native library and tests
+ shell: bash
+ run: |
+ source /opt/rh/gcc-toolset-13/enable
+ export NUM_THREADS=2 CCACHE_MAXSIZE=1G
+ ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --enable_kafka=ON \
+ --build_arrow=OFF --build_tests=ON --build_benchmarks=OFF
--build_examples=OFF \
+ --enable_s3=OFF --enable_gcs=OFF --enable_hdfs=OFF
--enable_abfs=OFF
+ # TODO: Re-enable after https://github.com/apache/gluten/pull/13183 is
merged.
+ # - name: Run native Kafka connector tests
+ # run: ./cpp/build/velox/tests/velox_kafka_connector_test
+ # TODO: Re-enable when the Kafka plan-conversion tests are added.
+ # - name: Run Kafka and file plan conversion tests
+ # run: |
+ # ./cpp/build/velox/tests/velox_plan_conversion_test \
+ #
--gtest_filter='Substrait2VeloxPlanConversionTest.serializedFileSplit:Substrait2VeloxPlanConversionTest.kafkaReadValidation:Substrait2VeloxPlanConversionTest.mixedKafkaAndFileSplits:Substrait2VeloxPlanValidatorTest.kafkaReadRequiresNativeSupport'
+ - uses: actions/upload-artifact@v7
+ with:
+ name: velox-kafka-native-${{ github.sha }}
+ path: cpp/build/releases/
+ if-no-files-found: error
+ retention-days: 1
+
+ spark35:
+ needs: build-native
+ runs-on: ubuntu-22.04
+ timeout-minutes: 60
+ env:
+ KAFKA_BOOTSTRAP_SERVERS: localhost:9092
+ SPARK_LOCAL_IP: 127.0.0.1
+ services:
+ kafka:
+ image: apache/kafka:3.9.1
+ ports:
+ - 9092:9092
+ env:
+ CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+ KAFKA_NODE_ID: 1
+ KAFKA_PROCESS_ROLES: broker,controller
+ KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
+ KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
+ KAFKA_LISTENER_SECURITY_PROTOCOL_MAP:
PLAINTEXT:PLAINTEXT,CONTROLLER:PLAINTEXT
+ KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+ KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
+ KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+ KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+ KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+ KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+ options: >-
+ --health-cmd "/opt/kafka/bin/kafka-topics.sh --bootstrap-server
localhost:9092 --list"
+ --health-interval 10s
+ --health-timeout 10s
+ --health-retries 12
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: '11'
+ cache: maven
+ - uses: actions/download-artifact@v8
+ with:
+ name: velox-kafka-native-${{ github.sha }}
+ path: cpp/build/releases/
+ - name: Install Spark 3.5 modules with Kafka support
+ run: |
+ ./build/mvn -B -ntp install -Pbackends-velox -Pspark-3.5
-Pscala-2.12 \
+ -Pjava-11 -Pkafka -Pspark-ut -Pfast-build -DskipTests
+ # TODO: Re-enable when VeloxKafkaScanValidationSuite is added.
+ # - name: Check Kafka schema validation without a broker
+ # run: |
+ # ./build/mvn -B -ntp test -pl gluten-ut/spark35 -Pbackends-velox
-Pspark-3.5 \
+ # -Pscala-2.12 -Pjava-11 -Pspark-ut -Pfast-build \
+ #
-Dsuites=org.apache.gluten.execution.VeloxKafkaScanValidationSuite
+ # TODO: Re-enable when VeloxGlutenKafkaScanSuite is added.
+ # - name: Check native micro-batch reads and headers fallback
+ # run: |
+ # ./build/mvn -B -ntp test -pl backends-velox -Pbackends-velox
-Pspark-3.5 \
+ # -Pscala-2.12 -Pjava-11 -Pkafka -Pfast-build \
+ #
-Dsuites=org.apache.gluten.execution.kafka.VeloxGlutenKafkaScanSuite
+ - name: Upload test results
+ if: always()
+ uses: actions/upload-artifact@v7
+ with:
+ name: velox-kafka-spark35-results
+ path: |
+ **/surefire-reports/*
+ **/target/*.log
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index f732d9da44..f9918de593 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -66,6 +66,7 @@ option(ENABLE_S3 "Enable S3" OFF)
option(ENABLE_HDFS "Enable HDFS" OFF)
option(ENABLE_ORC "Enable ORC" OFF)
option(ENABLE_ABFS "Enable ABFS" OFF)
+option(ENABLE_KAFKA "Enable Kafka streaming read" OFF)
option(ENABLE_GPU "Enable GPU" OFF)
option(ENABLE_ENHANCED_FEATURES "Enable enhanced features" OFF)
option(ENABLE_LTO "Enable IPO/LTO" OFF)
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index fb4b0e4679..03526acaf5 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -496,8 +496,48 @@ else()
target_link_libraries(velox PUBLIC external::simdjson)
endif()
+# Restore the platform's default suffixes before looking for optional dynamic
+# dependencies such as librdkafka.
set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BCK})
+# Find and link librdkafka for Kafka streaming support
+if(ENABLE_KAFKA)
+ target_compile_definitions(velox PUBLIC ENABLE_KAFKA)
+ find_package(RdKafka CONFIG QUIET)
+ if(RdKafka_FOUND AND TARGET RdKafka::rdkafka++)
+ message(STATUS "Found RdKafka: ${RdKafka_DIR}")
+ target_link_libraries(velox PUBLIC RdKafka::rdkafka++)
+ else()
+ # Try to find librdkafka using pkg-config
+ find_package(PkgConfig QUIET)
+ if(PKG_CONFIG_FOUND)
+ pkg_check_modules(RDKAFKA QUIET IMPORTED_TARGET rdkafka++)
+ if(RDKAFKA_FOUND)
+ message(STATUS "Found librdkafka via pkg-config: ${RDKAFKA_LIBRARIES}")
+ target_link_libraries(velox PUBLIC PkgConfig::RDKAFKA)
+ else()
+ message(
+ FATAL_ERROR
+ "ENABLE_KAFKA is ON but the librdkafka C++ library was not found. "
+ "Install librdkafka-dev (Ubuntu/Debian), librdkafka-devel
(RPM-based "
+ "Linux), or librdkafka (Homebrew). For an existing installation, "
+ "set RdKafka_DIR or CMAKE_PREFIX_PATH to locate its CMake package,
"
+ "or PKG_CONFIG_PATH to the directory containing rdkafka++.pc. "
+ "To build without Kafka, pass --enable_kafka=OFF to the build
script "
+ "or -DENABLE_KAFKA=OFF to CMake.")
+ endif()
+ else()
+ message(
+ FATAL_ERROR
+ "ENABLE_KAFKA is ON but the librdkafka C++ CMake target was not
found "
+ "and pkg-config is unavailable. Install pkg-config, or set
RdKafka_DIR "
+ "or CMAKE_PREFIX_PATH to an installation providing
RdKafka::rdkafka++. "
+ "To build without Kafka, pass --enable_kafka=OFF to the build script
"
+ "or -DENABLE_KAFKA=OFF to CMake.")
+ endif()
+ endif()
+endif()
+
# Adopted from Velox's CMakeLists.txt.
# https://github.com/facebookincubator/velox/pull/11410
if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index 11f351b787..b12c9d2efc 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -36,6 +36,7 @@ ENABLE_GCS=OFF
ENABLE_S3=OFF
ENABLE_HDFS=OFF
ENABLE_ABFS=OFF
+ENABLE_KAFKA=OFF
ENABLE_VCPKG=OFF
ENABLE_GPU=OFF
ENABLE_ENHANCED_FEATURES=OFF
@@ -105,6 +106,10 @@ do
ENABLE_ABFS=("${arg#*=}")
shift # Remove argument name from processing
;;
+ --enable_kafka=*)
+ ENABLE_KAFKA=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
--enable_vcpkg=*)
ENABLE_VCPKG=("${arg#*=}")
shift # Remove argument name from processing
@@ -219,7 +224,7 @@ function concat_velox_param {
if [ "$ENABLE_VCPKG" = "ON" ]; then
# vcpkg will install static depends and init build environment
BUILD_OPTIONS="--build_tests=$BUILD_TESTS --enable_s3=$ENABLE_S3
--enable_gcs=$ENABLE_GCS \
- --enable_hdfs=$ENABLE_HDFS --enable_abfs=$ENABLE_ABFS"
+ --enable_hdfs=$ENABLE_HDFS --enable_abfs=$ENABLE_ABFS
--enable_kafka=$ENABLE_KAFKA"
source ./dev/vcpkg/env.sh ${BUILD_OPTIONS}
fi
@@ -290,6 +295,7 @@ function build_gluten_cpp {
"-DENABLE_S3=$ENABLE_S3"
"-DENABLE_HDFS=$ENABLE_HDFS"
"-DENABLE_ABFS=$ENABLE_ABFS"
+ "-DENABLE_KAFKA=$ENABLE_KAFKA"
"-DENABLE_GPU=$ENABLE_GPU"
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
"-DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES"
diff --git a/dev/ci-velox-buildshared-centos-9.sh
b/dev/ci-velox-buildshared-centos-9.sh
index 2d2fa8c9bc..47a2d8276d 100755
--- a/dev/ci-velox-buildshared-centos-9.sh
+++ b/dev/ci-velox-buildshared-centos-9.sh
@@ -27,5 +27,8 @@ fi
export VELOX_BUILD_SHARED=ON
+# Temporary verification for Kafka support in the non-vcpkg dynamic build.
+yum install -y librdkafka-devel
+
./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF
--build_tests=ON \
- --build_examples=ON --build_benchmarks=ON
+ --build_examples=ON --build_benchmarks=ON --enable_kafka=ON
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build
b/dev/docker/Dockerfile.centos8-dynamic-build
index 388f71f922..902368f9b9 100644
--- a/dev/docker/Dockerfile.centos8-dynamic-build
+++ b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -41,7 +41,7 @@ RUN set -ex; \
sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-Stream-* || true; \
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12; \
echo "check_certificate = off" >> ~/.wgetrc; \
- yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl; \
+ yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl
librdkafka-devel; \
mirror_host="https://www.apache.org/dyn/closer.lua";; \
wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
wget -nv
${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.3-bin.tgz; \
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
index 9a8a26378d..93847d232d 100644
--- a/dev/docker/Dockerfile.centos9-dynamic-build
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -35,7 +35,7 @@ RUN set -ex; \
# clang-tidy, installed here so its version is pinned to the image, not to
each CI run.
dnf install -y --setopt=install_weak_deps=False clang-tools-extra; \
echo "check_certificate = off" >> ~/.wgetrc; \
- yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl; \
+ yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl
librdkafka-devel; \
mirror_host="https://www.apache.org/dyn/closer.lua";; \
wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
wget -nv
${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.3-bin.tgz; \
diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh
index 410373f229..4515ec6a4d 100755
--- a/dev/vcpkg/init.sh
+++ b/dev/vcpkg/init.sh
@@ -22,6 +22,7 @@ ENABLE_S3=OFF
ENABLE_GCS=OFF
ENABLE_HDFS=OFF
ENABLE_ABFS=OFF
+ENABLE_KAFKA=OFF
for arg in "$@"; do
case $arg in
@@ -45,6 +46,10 @@ for arg in "$@"; do
ENABLE_ABFS=("${arg#*=}")
shift # Remove argument name from processing
;;
+ --enable_kafka=*)
+ ENABLE_KAFKA=("${arg#*=}")
+ shift # Remove argument name from processing
+ ;;
*)
echo "Unrecognized argument: $arg"
exit 1
@@ -87,6 +92,9 @@ fi
if [ "$ENABLE_ABFS" = "ON" ]; then
EXTRA_FEATURES+="--x-feature=velox-abfs "
fi
+if [ "$ENABLE_KAFKA" = "ON" ]; then
+ EXTRA_FEATURES+="--x-feature=velox-kafka "
+fi
if [ "${VCPKG_DYNAMIC_OPENSSL:-OFF}" = "ON" ]; then
EXTRA_FEATURES+="--x-feature=dynamic-openssl "
fi
diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json
index a7ad399b50..4003edc0ec 100644
--- a/dev/vcpkg/vcpkg.json
+++ b/dev/vcpkg/vcpkg.json
@@ -105,6 +105,12 @@
"azure-identity-cpp"
]
},
+ "velox-kafka": {
+ "description": "Velox Kafka Support",
+ "dependencies": [
+ "librdkafka"
+ ]
+ },
"duckdb": {
"description": "DUCKDB",
"dependencies": [
diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md
index a5244fd4e8..380be9bbd0 100644
--- a/docs/get-started/build-guide.md
+++ b/docs/get-started/build-guide.md
@@ -20,6 +20,7 @@ Please set them via `--`, e.g. `--build_type=Release`.
| enable_gcs | Build with GCS support.
| OFF |
| enable_hdfs | Build with HDFS support.
| OFF |
| enable_abfs | Build with ABFS support.
| OFF |
+| enable_kafka | Build with Kafka streaming read support
(librdkafka). | OFF |
| enable_vcpkg | Enable vcpkg for static build.
| OFF |
| run_setup_script | Run setup script to install Velox dependencies.
| ON |
| velox_repo | Specify your own Velox repo to build.
| "" |
@@ -30,6 +31,13 @@ Please set them via `--`, e.g. `--build_type=Release`.
| build_arrow | Build arrow java/cpp and install the libs in local.
Can turn it OFF after first build. | ON |
| spark_version | Build for specified version of Spark(3.4, 3.5, 4.0,
4.1, ALL). `ALL` means build for all versions. | ALL |
+Use `--enable_kafka=ON` to enable Kafka streaming reads. With
`--enable_vcpkg=ON`,
+vcpkg installs librdkafka. Without vcpkg, the platform setup scripts install
+`librdkafka-dev` on Ubuntu/Debian, `librdkafka-devel` on RPM-based Linux
platforms,
+or `librdkafka` through Homebrew on macOS. With `--run_setup_script=OFF`,
install
+librdkafka separately and make its CMake package or `rdkafka++` pkg-config
metadata
+available to the build.
+
### Environment variables for build
These environment variables can be set before running build scripts to control
build behavior.
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index ef3285fe21..71f1610c3b 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -66,11 +66,17 @@ fi
function process_setup_ubuntu {
sed -i "s|run_and_time install_arrow||g" scripts/setup-ubuntu.sh
+ if ! grep -q 'librdkafka-dev' scripts/setup-ubuntu.sh; then
+ sed -i 's/libevent-dev /libevent-dev librdkafka-dev /'
scripts/setup-ubuntu.sh
+ fi
echo "Using setup script from Velox"
}
function process_setup_centos9 {
sed -i "s|run_and_time install_arrow||g" scripts/setup-centos9.sh
+ if ! grep -q 'librdkafka-devel' scripts/setup-centos9.sh; then
+ sed -i 's/libevent-devel /libevent-devel librdkafka-devel /'
scripts/setup-centos9.sh
+ fi
echo "Using setup script from Velox"
}
@@ -99,6 +105,9 @@ function process_setup_macos {
if ! grep -Fq 'FOLLY_USE_JEMALLOC=OFF' scripts/setup-common.sh; then
sed -i '' 's/local FOLLY_FLAGS=(/local
FOLLY_FLAGS=(-DFOLLY_USE_JEMALLOC=OFF /' scripts/setup-common.sh
fi
+ if ! grep -q 'librdkafka' scripts/setup-macos.sh; then
+ sed -i '' 's/MACOS_VELOX_DEPS="/MACOS_VELOX_DEPS="librdkafka /'
scripts/setup-macos.sh
+ fi
}
function prepare_velox_source_code {
diff --git a/ep/build-velox/src/setup-centos7.sh
b/ep/build-velox/src/setup-centos7.sh
index 3fe457b326..6823bd10d4 100755
--- a/ep/build-velox/src/setup-centos7.sh
+++ b/ep/build-velox/src/setup-centos7.sh
@@ -233,7 +233,7 @@ $SUDO dnf makecache
# dnf install dependency libraries
dnf_install epel-release dnf-plugins-core # For ccache, ninja
-dnf_install ccache wget which libevent-devel \
+dnf_install ccache wget which libevent-devel librdkafka-devel \
yasm \
openssl-devel libzstd-devel lz4-devel double-conversion-devel \
curl-devel libxml2-devel libgsasl-devel libuuid-devel patch libicu-devel
tzdata
diff --git a/ep/build-velox/src/setup-centos8.sh
b/ep/build-velox/src/setup-centos8.sh
index cb3c4e0709..7cf8ad6173 100755
--- a/ep/build-velox/src/setup-centos8.sh
+++ b/ep/build-velox/src/setup-centos8.sh
@@ -87,7 +87,7 @@ function install_build_prerequisites {
# Install dependencies from the package managers.
function install_velox_deps_from_dnf {
- dnf_install libevent-devel \
+ dnf_install libevent-devel librdkafka-devel \
openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
libdwarf-devel curl-devel libicu-devel bison flex libsodium-devel
diff --git a/ep/build-velox/src/setup-openeuler24.sh
b/ep/build-velox/src/setup-openeuler24.sh
index 65ac853286..a0a3422d9a 100755
--- a/ep/build-velox/src/setup-openeuler24.sh
+++ b/ep/build-velox/src/setup-openeuler24.sh
@@ -90,7 +90,7 @@ function install_build_prerequisites {
# Install dependencies from the package managers.
function install_velox_deps_from_dnf {
- dnf_install libevent-devel \
+ dnf_install libevent-devel librdkafka-devel \
openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
libdwarf-devel elfutils-libelf-devel curl-devel libicu-devel bison flex \
libsodium-devel zlib-devel c-ares-devel
diff --git a/ep/build-velox/src/setup-rhel.sh b/ep/build-velox/src/setup-rhel.sh
index e0780d2b1b..3e2ef6fd35 100755
--- a/ep/build-velox/src/setup-rhel.sh
+++ b/ep/build-velox/src/setup-rhel.sh
@@ -73,7 +73,7 @@ function install_build_prerequisites {
# Install dependencies from the package managers.
function install_velox_deps_from_dnf {
- dnf_install libevent-devel \
+ dnf_install libevent-devel librdkafka-devel \
openssl-devel lz4-devel curl-devel libicu-devel zlib-devel
# install sphinx for doc gen
pip install sphinx sphinx-tabs breathe sphinx_rtd_theme
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
