This is an automated email from the ASF dual-hosted git repository. chaokunyang pushed a commit to branch releases-0.12 in repository https://gitbox.apache.org/repos/asf/fory.git
commit f47b440e438a49af97656d45c3df0bb65da864a3 Author: Emre Şafak <[email protected]> AuthorDate: Wed Aug 27 00:05:51 2025 -0400 fix(ci): Use ref_name tag in Windows, name release runs, verify wheel versions, use all python versions for release (#2532) ## Why? The Python release pipeline is broken: * Windows wheels do not use the ref tag (pyfory-0.12.1-cp39-cp39-win_amd64.whl vs pyfory-0.12.1a3-cp310-cp310-macosx_15_0_universal2.whl) * Linux does not release wheels for the full set of Python versions. ## What does this PR do? * Introduce `ci/tasks/python_container_build_script.sh` for containerized wheel builds. * Update `ci/build_linux_wheels.py` to use the new container script and define Python versions. * Enhance `ci/deploy.sh` to force wheel version on Windows. * Add version verification logic to `.github/workflows/build-native-pr.yml`. * Update GitHub Actions workflows (`release-python.yaml`, `build-native-release.yml`, `build-containerized-release.yml`, `build-native-pr.yml`) for consistency and artifact naming. * Refactor `ci/build_linux_wheels.py` to pass `GITHUB_REF_NAME` to Docker for version verification. ## Related issues #2527 --- .github/workflows/build-containerized-release.yml | 2 +- .github/workflows/build-native-release.yml | 19 ++++- .github/workflows/release-python.yaml | 1 + ci/build_linux_wheels.py | 84 +++++++++------------- ci/deploy.sh | 33 +++++++-- ci/tasks/python_container_build_script.sh | 87 +++++++++++++++++++++++ 6 files changed, 169 insertions(+), 57 deletions(-) diff --git a/.github/workflows/build-containerized-release.yml b/.github/workflows/build-containerized-release.yml index 4637c577b..c3d1d4afc 100644 --- a/.github/workflows/build-containerized-release.yml +++ b/.github/workflows/build-containerized-release.yml @@ -35,7 +35,7 @@ jobs: run: ./ci/run_ci.sh install_bazel - name: Build and test wheels run: ./ci/build_linux_wheels.py --arch ${{ runner.arch }} --release - - name: Upload artifacts + - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: name: pyfory-wheels-${{ matrix.os }}-${{ runner.arch }}-${{ github.ref_name }} diff --git a/.github/workflows/build-native-release.yml b/.github/workflows/build-native-release.yml index b4e0afda9..7baf25f40 100644 --- a/.github/workflows/build-native-release.yml +++ b/.github/workflows/build-native-release.yml @@ -45,13 +45,28 @@ jobs: - name: Build wheel shell: bash run: ./ci/deploy.sh build_pyfory + env: + GITHUB_REF_NAME: ${{ github.ref_name }} - name: Install and verify wheel shell: bash run: | python -m pip install --upgrade pip pip install dist/*.whl - python -c "import pyfory; print(pyfory.__version__)" - - name: Upload artifacts + INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)") + echo "Installed version: $INSTALLED_VERSION" + # Verify version matches the tag + EXPECTED_VERSION="${{ github.ref_name }}" + EXPECTED_VERSION=${EXPECTED_VERSION#v} + EXPECTED_VERSION="${EXPECTED_VERSION//-alpha/a}" + EXPECTED_VERSION="${EXPECTED_VERSION//-beta/b}" + EXPECTED_VERSION="${EXPECTED_VERSION//-rc/rc}" + echo "Expected version: $EXPECTED_VERSION" + if [ "$INSTALLED_VERSION" != "$EXPECTED_VERSION" ]; then + echo "Version mismatch: Expected $EXPECTED_VERSION but got $INSTALLED_VERSION" + exit 1 + fi + echo "Version verification successful" + - name: Upload wheels as artifacts uses: actions/upload-artifact@v4 with: name: pyfory-wheels-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref_name }} diff --git a/.github/workflows/release-python.yaml b/.github/workflows/release-python.yaml index 9062f995e..eb97c0564 100644 --- a/.github/workflows/release-python.yaml +++ b/.github/workflows/release-python.yaml @@ -16,6 +16,7 @@ # under the License. name: Publish Python +run-name: "Python Release: ${{ contains(github.event.workflow_run.name, 'Containerized') && 'Containerized' || 'Native' }} (Run ID: ${{ github.event.workflow_run.id }})" on: workflow_run: diff --git a/ci/build_linux_wheels.py b/ci/build_linux_wheels.py index 4e0a67b0e..4f4d6010a 100755 --- a/ci/build_linux_wheels.py +++ b/ci/build_linux_wheels.py @@ -32,45 +32,12 @@ import subprocess import sys from typing import List -SCRIPT = r'''set -e -yum install -y git sudo wget || true - -git config --global --add safe.directory /work - -# Determine Python versions to test -if [ "$RELEASE" = "1" ]; then - PYTHON_VERSIONS="cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" -else - PYTHON_VERSIONS="cp38-cp38 cp313-cp313" -fi - -ci/run_ci.sh install_bazel -export PATH="$HOME/.local/bin:$PATH" - -# use the python interpreters preinstalled in manylinux -OLD_PATH=$PATH -for PY in $PYTHON_VERSIONS; do - export PYTHON_PATH="/opt/python/$PY/bin/python" - export PATH="/opt/python/$PY/bin:$OLD_PATH" - echo "Using $PYTHON_PATH" - ARCH=$(uname -m) - if [ "$ARCH" = "aarch64" ]; then - export PLAT="manylinux2014_aarch64" - else - export PLAT="manylinux2014_x86_64" - fi - python -m pip install cython wheel pytest auditwheel - ci/deploy.sh build_pyfory - - latest_wheel=$(ls -t dist/*.whl | head -n1) - echo "Attempting to install $latest_wheel" - python -m pip install "$latest_wheel" - python -c "import pyfory; print(pyfory.__version__)" - - bazel clean --expunge -done -export PATH=$OLD_PATH -''' +# Define Python version sets directly in the Python script +RELEASE_PYTHON_VERSIONS = "cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311 cp312-cp312 cp313-cp313" +DEFAULT_PYTHON_VERSIONS = "cp38-cp38 cp313-cp313" + +# Path to the container build script +CONTAINER_SCRIPT_PATH = "ci/tasks/python_container_build_script.sh" DEFAULT_X86_IMAGES = [ "quay.io/pypa/manylinux2014_x86_64:latest", @@ -118,26 +85,38 @@ def collect_images_for_arch(arch_normalized: str) -> List[str]: raise SystemExit(f"Unsupported arch: {arch_normalized!r}") return imgs -def build_docker_cmd(workspace: str, image: str) -> List[str]: +def build_docker_cmd(workspace: str, image: str, release: bool = False) -> List[str]: workspace = os.path.abspath(workspace) - return [ + python_versions = RELEASE_PYTHON_VERSIONS if release else DEFAULT_PYTHON_VERSIONS + + # Get GitHub reference name from environment + github_ref_name = os.environ.get("GITHUB_REF_NAME", "") + + cmd = [ "docker", "run", "-i", "--rm", - "-v", f"{workspace}:/work", - "-w", "/work", - image, - "bash", "-s", "--" + "-v", f"{workspace}:/work", # (v)olume + "-w", "/work", # (w)orking directory + "-e", f"PYTHON_VERSIONS={python_versions}", # (e)nvironment variables + "-e", f"RELEASE_BUILD={'1' if release else '0'}" ] -def run_for_images(images: List[str], workspace: str, dry_run: bool) -> int: + # Pass GitHub reference name if available + if github_ref_name: + cmd.extend(["-e", f"GITHUB_REF_NAME={github_ref_name}"]) + + cmd.extend([image, "bash", CONTAINER_SCRIPT_PATH]) + return cmd + +def run_for_images(images: List[str], workspace: str, dry_run: bool, release: bool = False) -> int: rc_overall = 0 for image in images: - docker_cmd = build_docker_cmd(workspace, image) + docker_cmd = build_docker_cmd(workspace, image, release=release) printable = " ".join(shlex.quote(c) for c in docker_cmd) print(f"+ {printable}") if dry_run: continue try: - completed = subprocess.run(docker_cmd, input=SCRIPT.encode("utf-8")) + completed = subprocess.run(docker_cmd) if completed.returncode != 0: print(f"Container {image} exited with {completed.returncode}", file=sys.stderr) rc_overall = completed.returncode if rc_overall == 0 else rc_overall @@ -159,8 +138,15 @@ def main() -> int: print(f"No images configured for arch {arch}", file=sys.stderr) return 2 workspace = os.environ.get("GITHUB_WORKSPACE", os.getcwd()) + + # Check if the container script exists + script_path = os.path.join(workspace, CONTAINER_SCRIPT_PATH) + if not os.path.exists(script_path): + print(f"Container script not found at {script_path}", file=sys.stderr) + return 2 + print(f"Selected images for arch {args.arch}: {images}") - return run_for_images(images, workspace, args.dry_run) + return run_for_images(images, workspace, args.dry_run, release=args.release) if __name__ == "__main__": sys.exit(main()) diff --git a/ci/deploy.sh b/ci/deploy.sh index cb966b57f..5ec402e05 100755 --- a/ci/deploy.sh +++ b/ci/deploy.sh @@ -44,16 +44,26 @@ bump_java_version() { python "$ROOT/ci/release.py" bump_version -l java -version "$1" } -bump_py_version() { +# Replicates the behavior of _update_python_version in ci/release.py +parse_py_version() { local version="$1" if [ -z "$version" ]; then # Get the latest tag from the current Git repository version=$(git describe --tags --abbrev=0) - # Check if the tag starts with 'v' and strip it - if [[ $version == v* ]]; then - version="${version:1}" - fi fi + # Check if the tag starts with 'v' and strip it + if [[ $version == v* ]]; then + version="${version:1}" + fi + version="${version//-alpha/a}" + version="${version//-beta/b}" + version="${version//-rc/rc}" + echo "$version" +} + +bump_py_version() { + local version + version=$(parse_py_version "$1") python "$ROOT/ci/release.py" bump_version -l python -version "$version" } @@ -85,6 +95,19 @@ build_pyfory() { else $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi + elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "win32" ]]; then + + # Windows tends to drop alpha/beta markers - force it through setup.cfg + if [ -n "$GITHUB_REF_NAME" ]; then + version=$(parse_py_version "$GITHUB_REF_NAME") + echo "Using version from GITHUB_REF_NAME: $version" + echo "[metadata]" > setup.cfg + echo "version = $version" >> setup.cfg + fi + + $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" + # Clean up + rm setup.cfg else $PYTHON_CMD setup.py bdist_wheel --dist-dir="$ROOT/dist" fi diff --git a/ci/tasks/python_container_build_script.sh b/ci/tasks/python_container_build_script.sh new file mode 100644 index 000000000..c89e961f9 --- /dev/null +++ b/ci/tasks/python_container_build_script.sh @@ -0,0 +1,87 @@ +#!/usr/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -e +yum install -y git sudo wget || true + +git config --global --add safe.directory /work + +ci/run_ci.sh install_bazel +export PATH="$HOME/.local/bin:$PATH" + +# Function to verify the installed version against expected version +verify_version() { + local installed_version=$1 + + echo "Installed version: $installed_version" + + # Check if GITHUB_REF_NAME is available and use it for verification + if [ -n "$GITHUB_REF_NAME" ]; then + # Strip leading 'v' if present + local expected_version + expected_version="$(ci/deploy.sh parse_py_version $GITHUB_REF_NAME)" + echo "Expected version: $expected_version" + + if [ "$installed_version" != "$expected_version" ]; then + echo "Version mismatch: Expected $expected_version but got $installed_version" + exit 1 + fi + echo "Version verification successful" + else + echo "GITHUB_REF_NAME not available, skipping version verification" + fi +} + +# use the python interpreters preinstalled in manylinux +OLD_PATH=$PATH +for PY in $PYTHON_VERSIONS; do + export PYTHON_PATH="/opt/python/$PY/bin/python" + export PATH="/opt/python/$PY/bin:$OLD_PATH" + echo "Using $PYTHON_PATH" + ARCH=$(uname -m) + if [ "$ARCH" = "aarch64" ]; then + export PLAT="manylinux2014_aarch64" + else + export PLAT="manylinux2014_x86_64" + fi + python -m pip install cython wheel pytest auditwheel + ci/deploy.sh build_pyfory + + latest_wheel=$(find dist -maxdepth 1 -type f -name '*.whl' -print0 | xargs -0 ls -t | head -n1) + if [ -z "$latest_wheel" ]; then + echo "No wheel found" >&2 + exit 1 + fi + + echo "Attempting to install $latest_wheel" + python -m pip install "$latest_wheel" + + # Verify the installed version matches the expected version + INSTALLED_VERSION=$(python -c "import pyfory; print(pyfory.__version__)") + + # Only run version verification for release builds + if [ "${RELEASE_BUILD:-0}" = "1" ]; then + echo "Running version verification for release build" + verify_version "$INSTALLED_VERSION" + else + echo "Skipping version verification for test build" + fi + + bazel clean --expunge +done +export PATH=$OLD_PATH --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
