This is an automated email from the ASF dual-hosted git repository.
jorisvandenbossche pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 3e9384bbf4 GH-43519: [Python] Set up wheel building for Python 3.13
(#43539)
3e9384bbf4 is described below
commit 3e9384bbf4162ea060e867a753bce464b31e5e1c
Author: Lysandros Nikolaou <[email protected]>
AuthorDate: Thu Aug 22 15:27:40 2024 +0200
GH-43519: [Python] Set up wheel building for Python 3.13 (#43539)
### Rationale for this change
Like #43519 mentionies, now that the first `rc` is out, it's probably time
to add CI coverage for Python 3.13 (and also start building wheels).
### What changes are included in this PR?
I'm fairly new to the build/CI processes of the project, but I tried to
follow the same template as #37901. I'll follow up afterwards with adding CI
coverage for the free-threaded build as well.
* GitHub Issue: #43519
Lead-authored-by: Lysandros Nikolaou <[email protected]>
Co-authored-by: Joris Van den Bossche <[email protected]>
Signed-off-by: Joris Van den Bossche <[email protected]>
---
.env | 2 +-
ci/docker/python-wheel-manylinux-test.dockerfile | 7 ++++---
ci/docker/python-wheel-manylinux.dockerfile | 2 +-
ci/docker/python-wheel-windows-test-vs2019.dockerfile | 7 ++++---
ci/docker/python-wheel-windows-vs2019.dockerfile | 7 ++++---
ci/scripts/install_gcs_testbench.sh | 10 +++++++---
ci/scripts/install_python.sh | 14 +++++++++++---
ci/scripts/python_wheel_macos_build.sh | 2 --
dev/release/verify-release-candidate.sh | 6 +++---
dev/tasks/python-wheels/github.linux.yml | 5 +++++
dev/tasks/python-wheels/github.osx.yml | 2 +-
dev/tasks/tasks.yml | 3 ++-
docker-compose.yml | 9 ++++++---
python/pyproject.toml | 1 +
python/requirements-wheel-build.txt | 5 +++++
python/requirements-wheel-test.txt | 7 +++++++
16 files changed, 62 insertions(+), 27 deletions(-)
diff --git a/.env b/.env
index 1358aafe82..21f904c320 100644
--- a/.env
+++ b/.env
@@ -95,7 +95,7 @@ VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" #
2024.04.26 Release
# ci/docker/python-wheel-windows-vs2019.dockerfile.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-06-18
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-08-06
# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker-compose run --rm
conan".
# See https://github.com/conan-io/conan-docker-tools#readme and
diff --git a/ci/docker/python-wheel-manylinux-test.dockerfile
b/ci/docker/python-wheel-manylinux-test.dockerfile
index cdd0ae3ced..443ff9c53c 100644
--- a/ci/docker/python-wheel-manylinux-test.dockerfile
+++ b/ci/docker/python-wheel-manylinux-test.dockerfile
@@ -16,8 +16,8 @@
# under the License.
ARG arch
-ARG python
-FROM ${arch}/python:${python}
+ARG python_image_tag
+FROM ${arch}/python:${python_image_tag}
# RUN pip install --upgrade pip
@@ -27,4 +27,5 @@ COPY python/requirements-wheel-test.txt /arrow/python/
RUN pip install -r /arrow/python/requirements-wheel-test.txt
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
-RUN PYTHON=python /arrow/ci/scripts/install_gcs_testbench.sh default
+ARG python
+RUN PYTHON_VERSION=${python} /arrow/ci/scripts/install_gcs_testbench.sh default
diff --git a/ci/docker/python-wheel-manylinux.dockerfile
b/ci/docker/python-wheel-manylinux.dockerfile
index cb39667af1..42f088fd8a 100644
--- a/ci/docker/python-wheel-manylinux.dockerfile
+++ b/ci/docker/python-wheel-manylinux.dockerfile
@@ -103,7 +103,7 @@ RUN vcpkg install \
# Configure Python for applications running in the bash shell of this
Dockerfile
ARG python=3.8
ENV PYTHON_VERSION=${python}
-RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \
+RUN PYTHON_ROOT=$(find /opt/python -name
cp${PYTHON_VERSION/./}-cp${PYTHON_VERSION/./}) && \
echo "export PATH=$PYTHON_ROOT/bin:\$PATH" >> /etc/profile.d/python.sh
SHELL ["/bin/bash", "-i", "-c"]
diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 32bbb55e82..5f488a4c28 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -40,10 +40,11 @@ ARG python=3.8
RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH
"%PATH%;C:\Python38;C:\Python38\Scripts") & \
(if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH
"%PATH%;C:\Python39;C:\Python39\Scripts") & \
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH
"%PATH%;C:\Python310;C:\Python310\Scripts") & \
- (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH
"%PATH%;C:\Python311;C:\Python311\Scripts") & \
- (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH
"%PATH%;C:\Python312;C:\Python312\Scripts")
+ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH
"%PATH%;C:\Python311;C:\Python311\Scripts") & \
+ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4" && setx PATH
"%PATH%;C:\Python312;C:\Python312\Scripts") & \
+ (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH
"%PATH%;C:\Python313;C:\Python313\Scripts")
# Install archiver to extract xz archives
-RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION% & \
+RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
& \
python -m pip install --no-cache-dir -U pip setuptools & \
choco install --no-progress -r -y archiver
diff --git a/ci/docker/python-wheel-windows-vs2019.dockerfile
b/ci/docker/python-wheel-windows-vs2019.dockerfile
index ff42de939d..5a17e3e4c5 100644
--- a/ci/docker/python-wheel-windows-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2019.dockerfile
@@ -83,9 +83,10 @@ ARG python=3.8
RUN (if "%python%"=="3.8" setx PYTHON_VERSION "3.8.10" && setx PATH
"%PATH%;C:\Python38;C:\Python38\Scripts") & \
(if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH
"%PATH%;C:\Python39;C:\Python39\Scripts") & \
(if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH
"%PATH%;C:\Python310;C:\Python310\Scripts") & \
- (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.5" && setx PATH
"%PATH%;C:\Python311;C:\Python311\Scripts") & \
- (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.0" && setx PATH
"%PATH%;C:\Python312;C:\Python312\Scripts")
-RUN choco install -r -y --no-progress python --version=%PYTHON_VERSION%
+ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH
"%PATH%;C:\Python311;C:\Python311\Scripts") & \
+ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.4" && setx PATH
"%PATH%;C:\Python312;C:\Python312\Scripts") & \
+ (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH
"%PATH%;C:\Python313;C:\Python313\Scripts")
+RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
RUN python -m pip install -U pip setuptools
COPY python/requirements-wheel-build.txt arrow/python/
diff --git a/ci/scripts/install_gcs_testbench.sh
b/ci/scripts/install_gcs_testbench.sh
index 2090290c99..5471b3cc23 100755
--- a/ci/scripts/install_gcs_testbench.sh
+++ b/ci/scripts/install_gcs_testbench.sh
@@ -41,8 +41,12 @@ version=$1
if [[ "${version}" -eq "default" ]]; then
version="v0.39.0"
# Latests versions of Testbench require newer setuptools
- ${PYTHON:-python3} -m pip install --upgrade setuptools
+ python3 -m pip install --upgrade setuptools
fi
-${PYTHON:-python3} -m pip install \
- "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz";
+# This script is run with PYTHON undefined in some places,
+# but those only use older pythons.
+if [[ -z "${PYTHON_VERSION}" ]] || [[ "${PYTHON_VERSION}" != "3.13" ]]; then
+ python3 -m pip install \
+ "https://github.com/googleapis/storage-testbench/archive/${version}.tar.gz";
+fi
diff --git a/ci/scripts/install_python.sh b/ci/scripts/install_python.sh
index 5f962f02b9..42d0e9ca17 100755
--- a/ci/scripts/install_python.sh
+++ b/ci/scripts/install_python.sh
@@ -28,8 +28,9 @@ declare -A versions
versions=([3.8]=3.8.10
[3.9]=3.9.13
[3.10]=3.10.11
- [3.11]=3.11.5
- [3.12]=3.12.0)
+ [3.11]=3.11.9
+ [3.12]=3.12.4
+ [3.13]=3.13.0)
if [ "$#" -ne 2 ]; then
echo "Usage: $0 <platform> <version>"
@@ -46,7 +47,14 @@ full_version=${versions[$2]}
if [ $platform = "macOS" ]; then
echo "Downloading Python installer..."
- if [ "$(uname -m)" = "arm64" ] || [ "$version" = "3.10" ] || [ "$version"
= "3.11" ] || [ "$version" = "3.12" ]; then
+ if [ "$version" = "3.13" ];
+ then
+ fname="python-${full_version}rc1-macos11.pkg"
+ elif [ "$(uname -m)" = "arm64" ] || \
+ [ "$version" = "3.10" ] || \
+ [ "$version" = "3.11" ] || \
+ [ "$version" = "3.12" ];
+ then
fname="python-${full_version}-macos11.pkg"
else
fname="python-${full_version}-macosx10.9.pkg"
diff --git a/ci/scripts/python_wheel_macos_build.sh
b/ci/scripts/python_wheel_macos_build.sh
index 3ed9d5d8dd..d5430f2674 100755
--- a/ci/scripts/python_wheel_macos_build.sh
+++ b/ci/scripts/python_wheel_macos_build.sh
@@ -48,13 +48,11 @@ fi
echo "=== (${PYTHON_VERSION}) Install Python build dependencies ==="
export PIP_SITE_PACKAGES=$(python -c 'import site;
print(site.getsitepackages()[0])')
-export PIP_TARGET_PLATFORM="macosx_${MACOSX_DEPLOYMENT_TARGET//./_}_${arch}"
pip install \
--upgrade \
--only-binary=:all: \
--target $PIP_SITE_PACKAGES \
- --platform $PIP_TARGET_PLATFORM \
-r ${source_dir}/python/requirements-wheel-build.txt
pip install "delocate>=0.10.3"
diff --git a/dev/release/verify-release-candidate.sh
b/dev/release/verify-release-candidate.sh
index 6a36109dc2..07e765a759 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -1146,7 +1146,7 @@ test_linux_wheels() {
local arch="x86_64"
fi
- local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11 3.12}"
+ local python_versions="${TEST_PYTHON_VERSIONS:-3.8 3.9 3.10 3.11 3.12 3.13}"
local
platform_tags="${TEST_WHEEL_PLATFORM_TAGS:-manylinux_2_17_${arch}.manylinux2014_${arch}
manylinux_2_28_${arch}}"
for python in ${python_versions}; do
@@ -1170,11 +1170,11 @@ test_macos_wheels() {
# apple silicon processor
if [ "$(uname -m)" = "arm64" ]; then
- local python_versions="3.8 3.9 3.10 3.11 3.12"
+ local python_versions="3.8 3.9 3.10 3.11 3.12 3.13"
local platform_tags="macosx_11_0_arm64"
local check_flight=OFF
else
- local python_versions="3.8 3.9 3.10 3.11 3.12"
+ local python_versions="3.8 3.9 3.10 3.11 3.12 3.13"
local platform_tags="macosx_10_15_x86_64"
fi
diff --git a/dev/tasks/python-wheels/github.linux.yml
b/dev/tasks/python-wheels/github.linux.yml
index 968c5da218..2854d4349f 100644
--- a/dev/tasks/python-wheels/github.linux.yml
+++ b/dev/tasks/python-wheels/github.linux.yml
@@ -36,6 +36,11 @@ jobs:
ARCHERY_USE_LEGACY_DOCKER_COMPOSE: 1
{% endif %}
PYTHON: "{{ python_version }}"
+ {% if python_version == "3.13" %}
+ PYTHON_IMAGE_TAG: "3.13-rc"
+ {% else %}
+ PYTHON_IMAGE_TAG: "{{ python_version }}"
+ {% endif %}
steps:
{{ macros.github_checkout_arrow()|indent }}
diff --git a/dev/tasks/python-wheels/github.osx.yml
b/dev/tasks/python-wheels/github.osx.yml
index 8ceb468af8..b26aeba32b 100644
--- a/dev/tasks/python-wheels/github.osx.yml
+++ b/dev/tasks/python-wheels/github.osx.yml
@@ -121,7 +121,7 @@ jobs:
source test-env/bin/activate
pip install --upgrade pip wheel
arch -{{ arch }} pip install -r
arrow/python/requirements-wheel-test.txt
- PYTHON=python arch -{{ arch }}
arrow/ci/scripts/install_gcs_testbench.sh default
+ PYTHON_VERSION={{ python_version }} arch -{{ arch }}
arrow/ci/scripts/install_gcs_testbench.sh default
arch -{{ arch }} arrow/ci/scripts/python_wheel_unix_test.sh
$(pwd)/arrow
{{
macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index fe02fe9ce6..60114d6930 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -389,7 +389,8 @@ tasks:
("3.9", "cp39", "cp39"),
("3.10", "cp310", "cp310"),
("3.11", "cp311", "cp311"),
- ("3.12", "cp312", "cp312")] %}
+ ("3.12", "cp312", "cp312"),
+ ("3.13", "cp313", "cp313")] %}
{############################## Wheel Linux ##################################}
diff --git a/docker-compose.yml b/docker-compose.yml
index 14eeeeee6e..3045cf015b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1096,9 +1096,10 @@ services:
args:
arch: ${ARCH}
arch_short: ${ARCH_SHORT}
- base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2024-02-04-ea37246
+ base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2024-08-03-32dfa47
vcpkg: ${VCPKG}
python: ${PYTHON}
+ python_image_tag: ${PYTHON_IMAGE_TAG}
manylinux: 2014
context: .
dockerfile: ci/docker/python-wheel-manylinux.dockerfile
@@ -1119,9 +1120,10 @@ services:
args:
arch: ${ARCH}
arch_short: ${ARCH_SHORT}
- base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2024-02-04-ea37246
+ base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2024-08-03-32dfa47
vcpkg: ${VCPKG}
python: ${PYTHON}
+ python_image_tag: ${PYTHON_IMAGE_TAG}
manylinux: 2_28
context: .
dockerfile: ci/docker/python-wheel-manylinux.dockerfile
@@ -1135,7 +1137,7 @@ services:
command: /arrow/ci/scripts/python_wheel_manylinux_build.sh
python-wheel-manylinux-test-imports:
- image: ${ARCH}/python:${PYTHON}
+ image: ${ARCH}/python:${PYTHON_IMAGE_TAG}
shm_size: 2G
volumes:
- .:/arrow:delegated
@@ -1151,6 +1153,7 @@ services:
args:
arch: ${ARCH}
python: ${PYTHON}
+ python_image_tag: ${PYTHON_IMAGE_TAG}
context: .
dockerfile: ci/docker/python-wheel-manylinux-test.dockerfile
cache_from:
diff --git a/python/pyproject.toml b/python/pyproject.toml
index d863bb3e5f..8ece65dd46 100644
--- a/python/pyproject.toml
+++ b/python/pyproject.toml
@@ -48,6 +48,7 @@ classifiers = [
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: 3.11',
'Programming Language :: Python :: 3.12',
+ 'Programming Language :: Python :: 3.13',
]
maintainers = [
{name = "Apache Arrow Developers", email = "[email protected]"}
diff --git a/python/requirements-wheel-build.txt
b/python/requirements-wheel-build.txt
index faa078d3d7..2d44800476 100644
--- a/python/requirements-wheel-build.txt
+++ b/python/requirements-wheel-build.txt
@@ -1,3 +1,8 @@
+# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13
+# on PyPI
+--pre
+--extra-index-url
"https://pypi.anaconda.org/scientific-python-nightly-wheels/simple";
+
cython>=0.29.31
oldest-supported-numpy>=0.14; python_version<'3.9'
numpy>=2.0.0; python_version>='3.9'
diff --git a/python/requirements-wheel-test.txt
b/python/requirements-wheel-test.txt
index c7ff63e339..98ec2bd4fd 100644
--- a/python/requirements-wheel-test.txt
+++ b/python/requirements-wheel-test.txt
@@ -1,3 +1,9 @@
+# Remove pre and extra index url once there's NumPy and Cython wheels for 3.13
+# on PyPI
+--pre
+--prefer-binary
+--extra-index-url
"https://pypi.anaconda.org/scientific-python-nightly-wheels/simple";
+
cffi
cython
hypothesis
@@ -12,5 +18,6 @@ tzdata; sys_platform == 'win32'
numpy~=1.21.3; python_version < "3.11"
numpy~=1.23.2; python_version == "3.11"
numpy~=1.26.0; python_version == "3.12"
+numpy~=2.1.0; python_version >= "3.13"
pandas
