This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
The following commit(s) were added to refs/heads/main by this push:
new 7d130d81 fix(ci): install a modern protoc for the manylinux wheel
build (#751)
7d130d81 is described below
commit 7d130d812850cf34348df9440d0e038a8f4f2f92
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Sep 4 23:00:37 2026 -0700
fix(ci): install a modern protoc for the manylinux wheel build (#751)
---
.github/scripts/manylinux-build-deps.sh | 111 ++++++++++++++++++++++++++++++++
.github/workflows/ci.yml | 43 +++++++++++++
.github/workflows/release.yml | 42 ++++++------
3 files changed, 175 insertions(+), 21 deletions(-)
diff --git a/.github/scripts/manylinux-build-deps.sh
b/.github/scripts/manylinux-build-deps.sh
new file mode 100755
index 00000000..3b313efb
--- /dev/null
+++ b/.github/scripts/manylinux-build-deps.sh
@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Install the system build dependencies for a manylinux wheel.
+#
+# Both release steps and the CI job that guards them run this one script, so a
+# change cannot land in some of them and not others.
+#
+# protoc: the distro package is 2.5.0 on the CentOS 7 base, which predates the
+# --experimental_allow_proto3_optional flag prost-build passes when it compiles
+# lance-encoding/lance-file's .proto files.
+#
+# libclang: librocksdb-sys generates its bindings with bindgen, which needs a
+# libclang the base image does not carry; the distro clang is 3.4, older than
+# bindgen supports. bindgen also needs clang's builtin headers (stdbool.h and
+# friends), which libclang cannot locate on its own when loaded from the SCL
+# prefix, so both are handed over explicitly. The exports only reach the build
+# if this script is SOURCED from before-script-linux, not executed: a
step-level
+# `env:` never enters the container (maturin-action forwards only its own
+# variables), while the before-script runs in the same shell as the build - the
+# long-standing CFLAGS_aarch64 export below relies on the same behavior.
+#
+# perl-IPC-Cmd: needed by openssl.
+
+# This script is sourced, not executed, so that the exports below land in the
+# same shell the build runs in. Keep the caller's shell options: the strictness
+# is for this script, and leaking `set -u` in particular would turn any unset
+# variable in a future maturin-action release into a release-time failure.
+_deps_saved_opts=$(set +o)
+set -euo pipefail
+
+PROTOC_VERSION=36.1
+
+case "$(uname -m)" in
+ x86_64)
+ protoc_arch=x86_64
+
protoc_sha256=c4bc672d9d49214dc8cafdceadf4df92182d6ca8e3ec65a56b2d7de5602669b4
+ ;;
+ aarch64)
+ protoc_arch=aarch_64
+
protoc_sha256=237a68856edf1bd28b6204bddd0596c1cf46d298bc29c620012540b2e44c73e7
+ ;;
+ *)
+ echo "unsupported architecture: $(uname -m)" >&2
+ exit 1
+ ;;
+esac
+
+curl -fsSL -o /tmp/protoc.zip \
+
"https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-${protoc_arch}.zip";
+
+# The wheels this produces are published and signed, so verify the archive
+# rather than trusting whatever the download returned.
+echo "${protoc_sha256} /tmp/protoc.zip" | sha256sum -c -
+
+unzip -qo /tmp/protoc.zip -d /usr/local bin/protoc 'include/*'
+chmod +x /usr/local/bin/protoc
+protoc --version
+
+yum install -y llvm-toolset-7.0-clang llvm-toolset-7.0-clang-libs perl-IPC-Cmd
+
+# libclang.so links against the LLVM runtime in the same prefix, so make the
+# loader aware of it rather than relying on a symlink out of the prefix.
+libclang_dir=/opt/rh/llvm-toolset-7.0/root/usr/lib64
+if [ ! -e "$libclang_dir/libclang.so" ]; then
+ echo "libclang.so not found under $libclang_dir" >&2
+ return 1 2>/dev/null || exit 1
+fi
+
+# Register the prefix so the LLVM runtime libclang links against resolves.
+echo "$libclang_dir" >/etc/ld.so.conf.d/llvm-toolset-7.0.conf
+ldconfig
+
+# Assign in a condition context: a bare `var=$(...)` takes the substitution's
+# status, so a non-matching glob would trip errexit before the check below.
+clang_include=""
+for candidate in "$libclang_dir"/clang/*/include; do
+ if [ -e "$candidate/stdbool.h" ]; then
+ clang_include=$candidate
+ break
+ fi
+done
+if [ -z "$clang_include" ]; then
+ echo "clang builtin headers not found under $libclang_dir/clang" >&2
+ return 1 2>/dev/null || exit 1
+fi
+
+export LIBCLANG_PATH="$libclang_dir"
+export BINDGEN_EXTRA_CLANG_ARGS="-I$clang_include"
+echo "libclang: $LIBCLANG_PATH"
+echo "clang builtin headers: $clang_include"
+
+# Restore the caller's shell options; the exports above survive.
+eval "$_deps_saved_opts"
+unset _deps_saved_opts
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 25f11af2..9ce83fd0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -307,6 +307,49 @@ jobs:
run: |
sudo chown -R $(id -u):$(id -g) $GITHUB_WORKSPACE/.cargo
$GITHUB_WORKSPACE/target $GITHUB_WORKSPACE/python/target
$GITHUB_WORKSPACE/.uv-cache || true
+ manylinux-wheel-build:
+ name: Build the manylinux wheel
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - runner: ubuntu-latest
+ target: x86_64-unknown-linux-gnu
+ - runner: ubuntu-22.04-arm
+ target: aarch64-unknown-linux-gnu
+ runs-on: ${{ matrix.runner }}
+ steps:
+ - uses: actions/checkout@v7
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.13'
+ # Builds the wheel in the same container, running the same dependency
+ # script, that the release workflow publishes from. Regular CI runs on
the
+ # host, whose apt protoc is new enough for lance-encoding's prost-build
and
+ # which carries libclang for librocksdb-sys, so a container-only break
(see
+ # #750) stays invisible until a release tag is pushed - by which point
the
+ # version is burnt on crates.io and cannot be reused.
+ - name: Build wheel
+ uses: PyO3/maturin-action@v1
+ with:
+ target: ${{ matrix.target }}
+ command: build
+ args: --release -m python/Cargo.toml --out dist
+ # Pin the image rather than relying on the default, so this job and
the
+ # release steps are provably the same environment.
+ manylinux: '2014'
+ before-script-linux: |
+ source .github/scripts/manylinux-build-deps.sh
+ # A wheel that compiles is not necessarily a wheel that loads: the
+ # bindings rocksdb generates come from the container's older libclang,
and
+ # bad bindings tend to surface on import or first use rather than at
+ # compile time. Each runner installs the wheel it just built.
+ - name: Import the built wheel
+ run: |
+ python -m pip install --upgrade pip
+ python -m pip install dist/*.whl
+ python -c "import hudi; print(hudi.__file__)"
+
publish-coverage:
name: Publish coverage reports to codecov.io
runs-on: ubuntu-latest
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 653ae04f..eeaf35d2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -119,37 +119,37 @@ jobs:
release-pypi-manylinux:
name: PyPI release manylinux
needs: validate-release-tag
- runs-on: ubuntu-latest
+ # aarch64 builds on a native ARM runner so this job and the CI wheel-build
+ # job use the same image; cross-building it from x86_64 would exercise a
+ # container CI never touches. It also drops the ring workaround that the
+ # cross build needed (briansmith/ring#1728).
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - runner: ubuntu-latest
+ target: x86_64-unknown-linux-gnu
+ # the sdist rides along with exactly one leg
+ args: --skip-existing -m python/Cargo.toml
+ - runner: ubuntu-22.04-arm
+ target: aarch64-unknown-linux-gnu
+ args: --skip-existing -m python/Cargo.toml --no-sdist
+ runs-on: ${{ matrix.runner }}
steps:
- uses: actions/checkout@v7
- uses: actions/setup-python@v6
with:
python-version: '3.13'
- - name: Publish manylinux to pypi x86_64 (with sdist)
+ - name: Publish manylinux to pypi
uses: PyO3/maturin-action@v1
env:
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
MATURIN_REPOSITORY: pypi
with:
- target: x86_64-unknown-linux-gnu
- command: publish
- args: --skip-existing -m python/Cargo.toml
- # protobuf-compiler is needed by lance-encoding/lance-file's
prost-build;
- # perl-IPC-Cmd is needed by openssl.
- before-script-linux: yum install -y protobuf-compiler perl-IPC-Cmd
-
- - name: Publish manylinux to pypi aarch64 (without sdist)
- uses: PyO3/maturin-action@v1
- env:
- MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
- MATURIN_REPOSITORY: pypi
- with:
- target: aarch64-unknown-linux-gnu
+ target: ${{ matrix.target }}
command: publish
- args: --skip-existing -m python/Cargo.toml --no-sdist
+ args: ${{ matrix.args }}
+ manylinux: '2014'
before-script-linux: |
- yum install -y protobuf-compiler
- # We can remove this once we upgrade to 2_28.
- # https://github.com/briansmith/ring/issues/1728
- export CFLAGS_aarch64_unknown_linux_gnu="-D__ARM_ARCH=8"
+ source .github/scripts/manylinux-build-deps.sh
