This is an automated email from the ASF dual-hosted git repository.
zhouyuan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 96883745bc [VL] Build librdkafka from source for non vcpkg builds
(#13270)
96883745bc is described below
commit 96883745bc01d478fdf4718696824415d60bbe99
Author: Reema <[email protected]>
AuthorDate: Thu Oct 15 09:57:18 2026 +0300
[VL] Build librdkafka from source for non vcpkg builds (#13270)
---
cpp/velox/CMakeLists.txt | 26 ++++++++++++--------------
dev/build-helper-functions.sh | 17 +++++++++++++++++
dev/builddeps-veloxbe.sh | 8 ++++++++
dev/ci-velox-buildshared-centos-9.sh | 3 ---
dev/docker/Dockerfile.centos8-dynamic-build | 2 +-
dev/docker/Dockerfile.centos9-dynamic-build | 2 +-
dev/vcpkg/vcpkg.json | 5 ++++-
docs/get-started/build-guide.md | 9 +--------
ep/build-velox/src/get-velox.sh | 9 ---------
ep/build-velox/src/setup-centos7.sh | 2 +-
ep/build-velox/src/setup-centos8.sh | 2 +-
ep/build-velox/src/setup-openeuler24.sh | 2 +-
ep/build-velox/src/setup-rhel.sh | 2 +-
13 files changed, 48 insertions(+), 41 deletions(-)
diff --git a/cpp/velox/CMakeLists.txt b/cpp/velox/CMakeLists.txt
index 03526acaf5..a15f201528 100644
--- a/cpp/velox/CMakeLists.txt
+++ b/cpp/velox/CMakeLists.txt
@@ -508,32 +508,30 @@ if(ENABLE_KAFKA)
message(STATUS "Found RdKafka: ${RdKafka_DIR}")
target_link_libraries(velox PUBLIC RdKafka::rdkafka++)
else()
- # Try to find librdkafka using pkg-config
+ # Link the C++ client and C mock API.
find_package(PkgConfig QUIET)
if(PKG_CONFIG_FOUND)
- pkg_check_modules(RDKAFKA QUIET IMPORTED_TARGET rdkafka++)
+ pkg_check_modules(RDKAFKA QUIET IMPORTED_TARGET rdkafka++ rdkafka)
if(RDKAFKA_FOUND)
message(STATUS "Found librdkafka via pkg-config: ${RDKAFKA_LIBRARIES}")
target_link_libraries(velox PUBLIC PkgConfig::RDKAFKA)
else()
message(
FATAL_ERROR
- "ENABLE_KAFKA is ON but the librdkafka C++ library was not found. "
- "Install librdkafka-dev (Ubuntu/Debian), librdkafka-devel
(RPM-based "
- "Linux), or librdkafka (Homebrew). For an existing installation, "
- "set RdKafka_DIR or CMAKE_PREFIX_PATH to locate its CMake package,
"
- "or PKG_CONFIG_PATH to the directory containing rdkafka++.pc. "
- "To build without Kafka, pass --enable_kafka=OFF to the build
script "
- "or -DENABLE_KAFKA=OFF to CMake.")
+ "Kafka is enabled, but librdkafka (C/C++) was not found.\n"
+ "Install it: ./dev/builddeps-veloxbe.sh install_librdkafka\n"
+ "For an existing install, set RdKafka_DIR, CMAKE_PREFIX_PATH or
PKG_CONFIG_PATH.\n"
+ "Disable Kafka with --enable_kafka=OFF (script) or
-DENABLE_KAFKA=OFF (CMake)."
+ )
endif()
else()
message(
FATAL_ERROR
- "ENABLE_KAFKA is ON but the librdkafka C++ CMake target was not
found "
- "and pkg-config is unavailable. Install pkg-config, or set
RdKafka_DIR "
- "or CMAKE_PREFIX_PATH to an installation providing
RdKafka::rdkafka++. "
- "To build without Kafka, pass --enable_kafka=OFF to the build script
"
- "or -DENABLE_KAFKA=OFF to CMake.")
+ "librdkafka was not found through CMake, and pkg-config is
unavailable.\n"
+ "Install it: ./dev/builddeps-veloxbe.sh install_librdkafka\n"
+ "For an existing install, set RdKafka_DIR or CMAKE_PREFIX_PATH.\n"
+ "Disable Kafka with --enable_kafka=OFF (script) or
-DENABLE_KAFKA=OFF (CMake)."
+ )
endif()
endif()
endif()
diff --git a/dev/build-helper-functions.sh b/dev/build-helper-functions.sh
index 631da2847a..2f80731d60 100644
--- a/dev/build-helper-functions.sh
+++ b/dev/build-helper-functions.sh
@@ -212,6 +212,23 @@ function cmake_install {
${SUDO} cmake --install "${BINARY_DIR}"
}
+function install_librdkafka() (
+ local version="v2.13.0"
+ source "${GLUTEN_DIR}/dev/build-helper-functions.sh"
+ # Reuse the Kafka build directory on repeat installs.
+ prompt() { return 1; }
+ local dependency_dir="${DEPENDENCY_DIR:-${GLUTEN_DIR}/ep/_ep}"
+ mkdir -p "${dependency_dir}"
+ cd "${dependency_dir}"
+ wget_and_untar
"https://github.com/confluentinc/librdkafka/archive/refs/tags/${version}.tar.gz";
librdkafka
+ cd librdkafka
+ cmake_install -DCMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
-DCMAKE_INSTALL_LIBDIR=lib \
+ -DRDKAFKA_BUILD_STATIC=ON -DRDKAFKA_BUILD_TESTS=OFF
-DRDKAFKA_BUILD_EXAMPLES=OFF \
+ -DENABLE_LZ4_EXT=OFF -DWITH_SSL=ON -DWITH_ZLIB=ON -DWITH_ZSTD=ON \
+ -DWITH_SASL=OFF -DWITH_CURL=OFF \
+ -DWITH_SASL_SCRAM=ON -DWITH_SASL_OAUTHBEARER=ON
+)
+
function setup_macos {
sed -i '' '/run_and_time install_arrow/d' scripts/setup-macos.sh
if [ $ARCH == 'x86_64' ]; then
diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh
index b12c9d2efc..cbc516fe97 100755
--- a/dev/builddeps-veloxbe.sh
+++ b/dev/builddeps-veloxbe.sh
@@ -246,6 +246,9 @@ concat_velox_param
# Keep VELOX_HOME visible to child processes such as Maven's build-info
generation step.
export VELOX_HOME
+# Expose dependency installers as explicit commands when setup is skipped.
+source "$GLUTEN_DIR/dev/build-helper-functions.sh"
+
function build_arrow {
if vcpkg_is_active; then
echo "ERROR: build_arrow is deprecated with --enable_vcpkg=ON; Arrow is
managed by Gluten vcpkg." >&2
@@ -354,6 +357,9 @@ function setup_dependencies {
if [[ "$ENABLE_S3" == "ON" || "$ENABLE_GCS" == "ON" || "$ENABLE_HDFS" ==
"ON" || "$ENABLE_ABFS" == "ON" ]]; then
install_adapters
fi
+ if [ "$ENABLE_KAFKA" = "ON" ] && ! vcpkg_is_active; then
+ install_librdkafka
+ fi
popd
}
@@ -372,6 +378,8 @@ commands_to_run=(${OTHER_ARGUMENTS[@]:-})
get_velox
if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ];
then
setup_dependencies
+ elif [ "$ENABLE_KAFKA" = "ON" ] && ! vcpkg_is_active; then
+ install_librdkafka
fi
build_velox_backend
else
diff --git a/dev/ci-velox-buildshared-centos-9.sh
b/dev/ci-velox-buildshared-centos-9.sh
index 47a2d8276d..02fcd50a88 100755
--- a/dev/ci-velox-buildshared-centos-9.sh
+++ b/dev/ci-velox-buildshared-centos-9.sh
@@ -27,8 +27,5 @@ fi
export VELOX_BUILD_SHARED=ON
-# Temporary verification for Kafka support in the non-vcpkg dynamic build.
-yum install -y librdkafka-devel
-
./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF
--build_tests=ON \
--build_examples=ON --build_benchmarks=ON --enable_kafka=ON
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build
b/dev/docker/Dockerfile.centos8-dynamic-build
index 902368f9b9..388f71f922 100644
--- a/dev/docker/Dockerfile.centos8-dynamic-build
+++ b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -41,7 +41,7 @@ RUN set -ex; \
sed -i -e
"s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g"
/etc/yum.repos.d/CentOS-Stream-* || true; \
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12; \
echo "check_certificate = off" >> ~/.wgetrc; \
- yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl
librdkafka-devel; \
+ yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl; \
mirror_host="https://www.apache.org/dyn/closer.lua";; \
wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
wget -nv
${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.3-bin.tgz; \
diff --git a/dev/docker/Dockerfile.centos9-dynamic-build
b/dev/docker/Dockerfile.centos9-dynamic-build
index 93847d232d..9a8a26378d 100644
--- a/dev/docker/Dockerfile.centos9-dynamic-build
+++ b/dev/docker/Dockerfile.centos9-dynamic-build
@@ -35,7 +35,7 @@ RUN set -ex; \
# clang-tidy, installed here so its version is pinned to the image, not to
each CI run.
dnf install -y --setopt=install_weak_deps=False clang-tools-extra; \
echo "check_certificate = off" >> ~/.wgetrc; \
- yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl
librdkafka-devel; \
+ yum install -y java-${JAVA_VERSION}-openjdk-devel patch wget git perl; \
mirror_host="https://www.apache.org/dyn/closer.lua";; \
wget -nv
${mirror_host}/celeborn/celeborn-0.5.4/apache-celeborn-0.5.4-bin.tgz?action=download
-O /opt/apache-celeborn-0.5.4-bin.tgz; \
wget -nv
${mirror_host}/celeborn/celeborn-0.6.3/apache-celeborn-0.6.3-bin.tgz?action=download
-O /opt/apache-celeborn-0.6.3-bin.tgz; \
diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json
index 4003edc0ec..403f6299f6 100644
--- a/dev/vcpkg/vcpkg.json
+++ b/dev/vcpkg/vcpkg.json
@@ -108,7 +108,10 @@
"velox-kafka": {
"description": "Velox Kafka Support",
"dependencies": [
- "librdkafka"
+ {
+ "name": "librdkafka",
+ "features": ["ssl", "zlib", "zstd"]
+ }
]
},
"duckdb": {
diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md
index 380be9bbd0..1533fd6f84 100644
--- a/docs/get-started/build-guide.md
+++ b/docs/get-started/build-guide.md
@@ -20,7 +20,7 @@ Please set them via `--`, e.g. `--build_type=Release`.
| enable_gcs | Build with GCS support.
| OFF |
| enable_hdfs | Build with HDFS support.
| OFF |
| enable_abfs | Build with ABFS support.
| OFF |
-| enable_kafka | Build with Kafka streaming read support
(librdkafka). | OFF |
+| enable_kafka | Build with Kafka streaming read support.
Automatically installs librdkafka. | OFF |
| enable_vcpkg | Enable vcpkg for static build.
| OFF |
| run_setup_script | Run setup script to install Velox dependencies.
| ON |
| velox_repo | Specify your own Velox repo to build.
| "" |
@@ -31,13 +31,6 @@ Please set them via `--`, e.g. `--build_type=Release`.
| build_arrow | Build arrow java/cpp and install the libs in local.
Can turn it OFF after first build. | ON |
| spark_version | Build for specified version of Spark(3.4, 3.5, 4.0,
4.1, ALL). `ALL` means build for all versions. | ALL |
-Use `--enable_kafka=ON` to enable Kafka streaming reads. With
`--enable_vcpkg=ON`,
-vcpkg installs librdkafka. Without vcpkg, the platform setup scripts install
-`librdkafka-dev` on Ubuntu/Debian, `librdkafka-devel` on RPM-based Linux
platforms,
-or `librdkafka` through Homebrew on macOS. With `--run_setup_script=OFF`,
install
-librdkafka separately and make its CMake package or `rdkafka++` pkg-config
metadata
-available to the build.
-
### Environment variables for build
These environment variables can be set before running build scripts to control
build behavior.
diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh
index e410709d5a..d4bbf04efd 100755
--- a/ep/build-velox/src/get-velox.sh
+++ b/ep/build-velox/src/get-velox.sh
@@ -66,17 +66,11 @@ fi
function process_setup_ubuntu {
sed -i "s|run_and_time install_arrow||g" scripts/setup-ubuntu.sh
- if ! grep -q 'librdkafka-dev' scripts/setup-ubuntu.sh; then
- sed -i 's/libevent-dev /libevent-dev librdkafka-dev /'
scripts/setup-ubuntu.sh
- fi
echo "Using setup script from Velox"
}
function process_setup_centos9 {
sed -i "s|run_and_time install_arrow||g" scripts/setup-centos9.sh
- if ! grep -q 'librdkafka-devel' scripts/setup-centos9.sh; then
- sed -i 's/libevent-devel /libevent-devel librdkafka-devel /'
scripts/setup-centos9.sh
- fi
echo "Using setup script from Velox"
}
@@ -105,9 +99,6 @@ function process_setup_macos {
if ! grep -Fq 'FOLLY_USE_JEMALLOC=OFF' scripts/setup-common.sh; then
sed -i '' 's/local FOLLY_FLAGS=(/local
FOLLY_FLAGS=(-DFOLLY_USE_JEMALLOC=OFF /' scripts/setup-common.sh
fi
- if ! grep -q 'librdkafka' scripts/setup-macos.sh; then
- sed -i '' 's/MACOS_VELOX_DEPS="/MACOS_VELOX_DEPS="librdkafka /'
scripts/setup-macos.sh
- fi
}
function prepare_velox_source_code {
diff --git a/ep/build-velox/src/setup-centos7.sh
b/ep/build-velox/src/setup-centos7.sh
index 6823bd10d4..3fe457b326 100755
--- a/ep/build-velox/src/setup-centos7.sh
+++ b/ep/build-velox/src/setup-centos7.sh
@@ -233,7 +233,7 @@ $SUDO dnf makecache
# dnf install dependency libraries
dnf_install epel-release dnf-plugins-core # For ccache, ninja
-dnf_install ccache wget which libevent-devel librdkafka-devel \
+dnf_install ccache wget which libevent-devel \
yasm \
openssl-devel libzstd-devel lz4-devel double-conversion-devel \
curl-devel libxml2-devel libgsasl-devel libuuid-devel patch libicu-devel
tzdata
diff --git a/ep/build-velox/src/setup-centos8.sh
b/ep/build-velox/src/setup-centos8.sh
index 7cf8ad6173..cb3c4e0709 100755
--- a/ep/build-velox/src/setup-centos8.sh
+++ b/ep/build-velox/src/setup-centos8.sh
@@ -87,7 +87,7 @@ function install_build_prerequisites {
# Install dependencies from the package managers.
function install_velox_deps_from_dnf {
- dnf_install libevent-devel librdkafka-devel \
+ dnf_install libevent-devel \
openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
libdwarf-devel curl-devel libicu-devel bison flex libsodium-devel
diff --git a/ep/build-velox/src/setup-openeuler24.sh
b/ep/build-velox/src/setup-openeuler24.sh
index a0a3422d9a..65ac853286 100755
--- a/ep/build-velox/src/setup-openeuler24.sh
+++ b/ep/build-velox/src/setup-openeuler24.sh
@@ -90,7 +90,7 @@ function install_build_prerequisites {
# Install dependencies from the package managers.
function install_velox_deps_from_dnf {
- dnf_install libevent-devel librdkafka-devel \
+ dnf_install libevent-devel \
openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \
libdwarf-devel elfutils-libelf-devel curl-devel libicu-devel bison flex \
libsodium-devel zlib-devel c-ares-devel
diff --git a/ep/build-velox/src/setup-rhel.sh b/ep/build-velox/src/setup-rhel.sh
index 3e2ef6fd35..e0780d2b1b 100755
--- a/ep/build-velox/src/setup-rhel.sh
+++ b/ep/build-velox/src/setup-rhel.sh
@@ -73,7 +73,7 @@ function install_build_prerequisites {
# Install dependencies from the package managers.
function install_velox_deps_from_dnf {
- dnf_install libevent-devel librdkafka-devel \
+ dnf_install libevent-devel \
openssl-devel lz4-devel curl-devel libicu-devel zlib-devel
# install sphinx for doc gen
pip install sphinx sphinx-tabs breathe sphinx_rtd_theme
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
