This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 5a32f42ca fix(ci): Use ref_name tag in Windows, name release runs,
verify wheel versions, use all python versions for release (#2532)
5a32f42ca is described below
commit 5a32f42ca45f134eb1d6882711143602453d95cd
Author: Emre Şafak <[email protected]>
AuthorDate: Wed Aug 27 00:05:51 2025 -0400
fix(ci): Use ref_name tag in Windows, name release runs, verify wheel
versions, use all python versions for release (#2532)
## Why?
The Python release pipeline is broken:
* Windows wheels do not use the ref tag
(pyfory-0.12.1-cp39-cp39-win_amd64.whl vs
pyfory-0.12.1a3-cp310-cp310-macosx_15_0_universal2.whl)
* Linux does not release wheels for the full set of Python versions.
## What does this PR do?
* Introduce `ci/tasks/python_container_build_script.sh` for
containerized wheel builds.
* Update `ci/build_linux_wheels.py` to use the new container script and
define Python versions.
* Enhance `ci/deploy.sh` to force wheel version on Windows.
* Add version verification logic to
`.github/workflows/build-native-pr.yml`.
* Update GitHub Actions workflows (`release-python.yaml`,
`build-native-release.yml`, `build-containerized-release.yml`,
`build-native-pr.yml`) for consistency and artifact naming.
* Refactor `ci/build_linux_wheels.py` to pass `GITHUB_REF_NAME` to
Docker for version verification.
## Related issues
#2527
---
.github/workflows/build-containerized-release.yml | 2 +-
.github/workflows/build-native-release.yml | 19 ++++-
.github/workflows/release-python.yaml | 1 +
ci/build_linux_wheels.py | 84 +++++++++-------------
ci/deploy.sh | 33 +++++++--
ci/tasks/python_container_build_script.sh | 87 +++++++++++++++++++++++
6 files changed, 169 insertions(+), 57 deletions(-)
diff --git a/.github/workflows/build-containerized-release.yml
b/.github/workflows/build-containerized-release.yml
index 4637c577b..c3d1d4afc 100644
--- a/.github/workflows/build-containerized-release.yml
+++ b/.github/workflows/build-containerized-release.yml
@@ -35,7 +35,7 @@ jobs:
run: ./ci/run_ci.sh install_bazel
- name: Build and test wheels
run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release
- - name: Upload artifacts
+ - name: Upload wheels as artifacts
uses: actions/upload-artifact@v4
with:
name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{
github.ref_name }}
diff --git a/.github/workflows/build-native-release.yml
b/.github/workflows/build-native-release.yml
index b4e0afda9..7baf25f40 100644
--- a/.github/workflows/build-native-release.yml
+++ b/.github/workflows/build-native-release.yml
@@ -45,13 +45,28 @@ jobs:
- name: Build wheel
shell: bash
run: ./ci/deploy.sh build_pyfory
+ env:
+ GITHUB_REF_NAME: ${{ github.ref_name }}
- name: Install and verify wheel
shell: bash
run: |
python -m pip install --upgrade pip
pip install dist/*.whl
- python -c "import pyfory; print(pyfory.__version__)"
- - name: Upload artifacts
+ INSTALLED_VERSION=$(python -c "import pyfory;
print(pyfory.__version__)")
+ echo "Installed version: $INSTALLED_VERSION"
+ # Verify version matches the tag
+ EXPECTED_VERSION="${{ github.ref_name }}"
+ EXPECTED_VERSION=${EXPECTED_VERSION#v}
+ EXPECTED_VERSION="${EXPECTED_VERSION//-alpha/a}"
+ EXPECTED_VERSION="${EXPECTED_VERSION//-beta/b}"
+ EXPECTED_VERSION="${EXPECTED_VERSION//-rc/rc}"
+ echo "Expected version: $EXPECTED_VERSION"
+ if [ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]; then
+ echo "Version mismatch: Expected $EXPECTED_VERSION but got
$INSTALLED_VERSION"
+ exit 1
+ fi
+ echo "Version verification successful"
+ - name: Upload wheels as artifacts
uses: actions/upload-artifact@v4
with:
name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version
}}-${{ github.ref_name }}
diff --git a/.github/workflows/release-python.yaml
b/.github/workflows/release-python.yaml
index 9062f995e..eb97c0564 100644
--- a/.github/workflows/release-python.yaml
+++ b/.github/workflows/release-python.yaml
@@ -16,6 +16,7 @@
# under the License.
name: Publish Python
+run-name: "Python Release: ${{ contains(github.event.workflow_run.name,
'Containerized') && 'Containerized' || 'Native' }} (Run ID: ${{
github.event.workflow_run.id }})"
on:
workflow_run:
diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py
index 4e0a67b0e..4f4d6010a 100755
--- a/ci/build_linux_wheels.py
+++ b/ci/build_linux_wheels.py
@@ -32,45 +32,12 @@ import subprocess
import sys
from typing import List
-SCRIPT = r'''set -e
-yum install -y git sudo wget || true
-
-git config --global --add safe.directory /work
-
-# Determine Python versions to test
-if [ "$RELEASE" = "1" ]; then
- PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312
cp313-cp313"
-else
- PYTHON_VERSIONS="cp38-cp38 cp313-cp313"
-fi
-
-ci/run_ci.sh install_bazel
-export PATH="$HOME/.local/bin:$PATH"
-
-# use the python interpreters preinstalled in manylinux
-OLD_PATH=$PATH
-for PY in $PYTHON_VERSIONS; do
- export PYTHON_PATH="/opt/python/$PY/bin/python"
- export PATH="/opt/python/$PY/bin:$OLD_PATH"
- echo "Using $PYTHON_PATH"
- ARCH=$(uname -m)
- if [ "$ARCH" = "aarch64" ]; then
- export PLAT="manylinux2014_aarch64"
- else
- export PLAT="manylinux2014_x86_64"
- fi
- python -m pip install cython wheel pytest auditwheel
- ci/deploy.sh build_pyfory
-
- latest_wheel=$(ls -t dist/*.whl | head -n1)
- echo "Attempting to install $latest_wheel"
- python -m pip install "$latest_wheel"
- python -c "import pyfory; print(pyfory.__version__)"
-
- bazel clean --expunge
-done
-export PATH=$OLD_PATH
-'''
+# Define Python version sets directly in the Python script
+RELEASE_PYTHON_VERSIONS = "cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311
cp312-cp312 cp313-cp313"
+DEFAULT_PYTHON_VERSIONS = "cp38-cp38 cp313-cp313"
+
+# Path to the container build script
+CONTAINER_SCRIPT_PATH = "ci/tasks/python_container_build_script.sh"
DEFAULT_X86_IMAGES = [
"quay.io/pypa/manylinux2014_x86_64:latest",
@@ -118,26 +85,38 @@ def collect_images_for_arch(arch_normalized: str) ->
List[str]:
raise SystemExit(f"Unsupported arch: {arch_normalized!r}")
return imgs
-def build_docker_cmd(workspace: str, image: str) -> List[str]:
+def build_docker_cmd(workspace: str, image: str, release: bool = False) ->
List[str]:
workspace = os.path.abspath(workspace)
- return [
+ python_versions = RELEASE_PYTHON_VERSIONS if release else
DEFAULT_PYTHON_VERSIONS
+
+ # Get GitHub reference name from environment
+ github_ref_name = os.environ.get("GITHUB_REF_NAME", "")
+
+ cmd = [
"docker", "run", "-i", "--rm",
- "-v", f"{workspace}:/work",
- "-w", "/work",
- image,
- "bash", "-s", "--"
+ "-v", f"{workspace}:/work", # (v)olume
+ "-w", "/work", # (w)orking directory
+ "-e", f"PYTHON_VERSIONS={python_versions}", # (e)nvironment variables
+ "-e", f"RELEASE_BUILD={'1' if release else '0'}"
]
-def run_for_images(images: List[str], workspace: str, dry_run: bool) -> int:
+ # Pass GitHub reference name if available
+ if github_ref_name:
+ cmd.extend(["-e", f"GITHUB_REF_NAME={github_ref_name}"])
+
+ cmd.extend([image, "bash", CONTAINER_SCRIPT_PATH])
+ return cmd
+
+def run_for_images(images: List[str], workspace: str, dry_run: bool, release:
bool = False) -> int:
rc_overall = 0
for image in images:
- docker_cmd = build_docker_cmd(workspace, image)
+ docker_cmd = build_docker_cmd(workspace, image, release=release)
printable = " ".join(shlex.quote(c) for c in docker_cmd)
print(f"+ {printable}")
if dry_run:
continue
try:
- completed = subprocess.run(docker_cmd,
input=SCRIPT.encode("utf-8"))
+ completed = subprocess.run(docker_cmd)
if completed.returncode != 0:
print(f"Container {image} exited with {completed.returncode}",
file=sys.stderr)
rc_overall = completed.returncode if rc_overall == 0 else
rc_overall
@@ -159,8 +138,15 @@ def main() -> int:
print(f"No images configured for arch {arch}", file=sys.stderr)
return 2
workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd())
+
+ # Check if the container script exists
+ script_path = os.path.join(workspace, CONTAINER_SCRIPT_PATH)
+ if not os.path.exists(script_path):
+ print(f"Container script not found at {script_path}", file=sys.stderr)
+ return 2
+
print(f"Selected images for arch {args.arch}: {images}")
- return run_for_images(images, workspace, args.dry_run)
+ return run_for_images(images, workspace, args.dry_run,
release=args.release)
if __name__ == "__main__":
sys.exit(main())
diff --git a/ci/deploy.sh b/ci/deploy.sh
index cb966b57f..5ec402e05 100755
--- a/ci/deploy.sh
+++ b/ci/deploy.sh
@@ -44,16 +44,26 @@ bump_java_version() {
python "$ROOT/ci/release.py" bump_version -l java -version "$1"
}
-bump_py_version() {
+# Replicates the behavior of _update_python_version in ci/release.py
+parse_py_version() {
local version="$1"
if [ -z "$version" ]; then
# Get the latest tag from the current Git repository
version=$(git describe --tags --abbrev=0)
- # Check if the tag starts with 'v' and strip it
- if [[ $version == v* ]]; then
- version="${version:1}"
- fi
fi
+ # Check if the tag starts with 'v' and strip it
+ if [[ $version == v* ]]; then
+ version="${version:1}"
+ fi
+ version="${version//-alpha/a}"
+ version="${version//-beta/b}"
+ version="${version//-rc/rc}"
+ echo "$version"
+}
+
+bump_py_version() {
+ local version
+ version=$(parse_py_version "$1")
python "$ROOT/ci/release.py" bump_version -l python -version "$version"
}
@@ -85,6 +95,19 @@ build_pyfory() {
else
$PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist"
fi
+ elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then
+
+ # Windows tends to drop alpha/beta markers - force it through setup.cfg
+ if [ -n "$GITHUB_REF_NAME" ]; then
+ version=$(parse_py_version "$GITHUB_REF_NAME")
+ echo "Using version from GITHUB_REF_NAME: $version"
+ echo "[metadata]" > setup.cfg
+ echo "version = $version" >> setup.cfg
+ fi
+
+ $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist"
+ # Clean up
+ rm setup.cfg
else
$PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist"
fi
diff --git a/ci/tasks/python_container_build_script.sh
b/ci/tasks/python_container_build_script.sh
new file mode 100644
index 000000000..c89e961f9
--- /dev/null
+++ b/ci/tasks/python_container_build_script.sh
@@ -0,0 +1,87 @@
+#!/usr/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+yum install -y git sudo wget || true
+
+git config --global --add safe.directory /work
+
+ci/run_ci.sh install_bazel
+export PATH="$HOME/.local/bin:$PATH"
+
+# Function to verify the installed version against expected version
+verify_version() {
+ local installed_version=$1
+
+ echo "Installed version: $installed_version"
+
+ # Check if GITHUB_REF_NAME is available and use it for verification
+ if [ -n "$GITHUB_REF_NAME" ]; then
+ # Strip leading 'v' if present
+ local expected_version
+ expected_version="$(ci/deploy.sh parse_py_version $GITHUB_REF_NAME)"
+ echo "Expected version: $expected_version"
+
+ if [ "$installed_version" != "$expected_version" ]; then
+ echo "Version mismatch: Expected $expected_version but got
$installed_version"
+ exit 1
+ fi
+ echo "Version verification successful"
+ else
+ echo "GITHUB_REF_NAME not available, skipping version verification"
+ fi
+}
+
+# use the python interpreters preinstalled in manylinux
+OLD_PATH=$PATH
+for PY in $PYTHON_VERSIONS; do
+ export PYTHON_PATH="/opt/python/$PY/bin/python"
+ export PATH="/opt/python/$PY/bin:$OLD_PATH"
+ echo "Using $PYTHON_PATH"
+ ARCH=$(uname -m)
+ if [ "$ARCH" = "aarch64" ]; then
+ export PLAT="manylinux2014_aarch64"
+ else
+ export PLAT="manylinux2014_x86_64"
+ fi
+ python -m pip install cython wheel pytest auditwheel
+ ci/deploy.sh build_pyfory
+
+ latest_wheel=$(find dist -maxdepth 1 -type f -name '*.whl' -print0 | xargs
-0 ls -t | head -n1)
+ if [ -z "$latest_wheel" ]; then
+ echo "No wheel found" >&2
+ exit 1
+ fi
+
+ echo "Attempting to install $latest_wheel"
+ python -m pip install "$latest_wheel"
+
+ # Verify the installed version matches the expected version
+ INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)")
+
+ # Only run version verification for release builds
+ if [ "${RELEASE_BUILD:-0}" = "1" ]; then
+ echo "Running version verification for release build"
+ verify_version "$INSTALLED_VERSION"
+ else
+ echo "Skipping version verification for test build"
+ fi
+
+ bazel clean --expunge
+done
+export PATH=$OLD_PATH
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
