This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 4ede48c89b GH-44421: [Python] Add configuration for building & testing
free-threaded wheels on Windows (#44804)
4ede48c89b is described below
commit 4ede48c89b8ec80bbd1895357f272c5fb61bc9b6
Author: Lysandros Nikolaou <[email protected]>
AuthorDate: Wed Jan 8 12:23:31 2025 +0100
GH-44421: [Python] Add configuration for building & testing free-threaded
wheels on Windows (#44804)
### Rationale for this change
There's no blockers anymore for building Windows wheels for the
free-threaded build, so this PR adds the necessary configuration to do that.
### What changes are included in this PR?
- Add jobs to build free-threaded wheels on Windows
- Move VCPKG-related stuff to a reusable base job
### Are these changes tested?
No tests required.
### Are there any user-facing changes?
No.
* GitHub Issue: #44421
Lead-authored-by: Antoine Pitrou <[email protected]>
Co-authored-by: Lysandros Nikolaou <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
---
.env | 5 +-
.pre-commit-config.yaml | 2 +-
...e-threaded-wheel-windows-test-vs2019.dockerfile | 52 ++++++++++++++
...n-free-threaded-wheel-windows-vs2019.dockerfile | 49 +++++++++++++
...thon-wheel-windows-test-vs2019-base.dockerfile} | 28 +++-----
.../python-wheel-windows-test-vs2019.dockerfile | 56 +++++----------
...=> python-wheel-windows-vs2019-base.dockerfile} | 29 ++------
ci/docker/python-wheel-windows-vs2019.dockerfile | 83 ++++------------------
ci/scripts/python_wheel_windows_build.bat | 19 +++--
ci/scripts/python_wheel_windows_test.bat | 14 +---
dev/tasks/python-wheels/github.windows.yml | 31 ++++++--
dev/tasks/tasks.yml | 8 +--
docker-compose.yml | 73 +++++++++++++++----
13 files changed, 255 insertions(+), 194 deletions(-)
diff --git a/.env b/.env
index 3db2673ae9..252f59ccfb 100644
--- a/.env
+++ b/.env
@@ -92,10 +92,11 @@ TZ=UTC
VCPKG="943c5ef1c8f6b5e6ced092b242c8299caae2ff01" # 2024.04.26 Release
# This must be updated when we update
-# ci/docker/python-wheel-windows-vs2019.dockerfile.
+# ci/docker/python-*-windows-*.dockerfile.
# This is a workaround for our CI problem that "archery docker build" doesn't
# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
-PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2024-08-06
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2025-01-08
+PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION=2025-01-08
# Use conanio/${CONAN_BASE}:{CONAN_VERSION} for "docker compose run --rm
conan".
# See https://github.com/conan-io/conan-docker-tools#readme and
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 27823cae5f..e54fe2393c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -39,7 +39,7 @@ repos:
files: >-
(
?^ci/docker/conda-python-emscripten\.dockerfile$|
- ?^ci/docker/python-wheel-windows-test-vs2019\.dockerfile$|
+ ?^ci/docker/python-.*-wheel-windows-test-vs2019.*\.dockerfile$|
)
types: []
- repo: https://github.com/pycqa/flake8
diff --git
a/ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile
b/ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile
new file mode 100644
index 0000000000..4b972999b0
--- /dev/null
+++ b/ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env
+# when you update this file.
+
+ARG base
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# (Hadolint does not expand variables and thinks '${base}' is an untagged
image)
+# hadolint ignore=DL3006
+FROM ${base}
+
+ARG python=3.13
+
+SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference =
'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+RUN $filename = 'python-3.13.1-amd64.exe'; \
+ $url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
+ Invoke-WebRequest -Uri $url -OutFile $filename; \
+ Start-Process -FilePath $filename -ArgumentList '/quiet',
'Include_freethreaded=1' -Wait
+
+ENV PYTHON_CMD="py -${python}t"
+
+SHELL ["cmd", "/S", "/C"]
+RUN %PYTHON_CMD% -m pip install -U pip setuptools
+
+COPY python/requirements-wheel-test.txt C:/arrow/python/
+# Cython and Pandas wheels for 3.13 free-threaded are not released yet
+RUN %PYTHON_CMD% -m pip install \
+ --extra-index-url
https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+ --pre \
+ --prefer-binary \
+ -r C:/arrow/python/requirements-wheel-test.txt
+# cffi-based tests would crash when importing cffi.
+# hadolint ignore=DL3059
+RUN %PYTHON_CMD% -m pip uninstall -y cffi
+
+ENV PYTHON="${python}t"
+ENV PYTHON_GIL=0
diff --git a/ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile
b/ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile
new file mode 100644
index 0000000000..adbdccde71
--- /dev/null
+++ b/ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
+# when you update this file.
+
+ARG base
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# (Hadolint does not expand variables and thinks '${base}' is an untagged
image)
+# hadolint ignore=DL3006
+FROM ${base}
+
+ARG python=3.13
+
+SHELL ["powershell", "-NoProfile", "-Command", "$ErrorActionPreference =
'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+RUN $filename = 'python-3.13.1-amd64.exe'; \
+ $url = 'https://www.python.org/ftp/python/3.13.1/' + $filename; \
+ Invoke-WebRequest -Uri $url -OutFile $filename; \
+ Start-Process -FilePath $filename -ArgumentList '/quiet',
'Include_freethreaded=1' -Wait
+
+ENV PYTHON_CMD="py -${python}t"
+
+SHELL ["cmd", "/S", "/C"]
+RUN %PYTHON_CMD% -m pip install -U pip setuptools
+
+COPY python/requirements-wheel-build.txt C:/arrow/python/
+# Cython wheels for 3.13 free-threaded are not released yet
+RUN %PYTHON_CMD% -m pip install \
+ --extra-index-url
https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
+ --pre \
+ --prefer-binary \
+ cython
+RUN %PYTHON_CMD% -m pip install -r C:/arrow/python/requirements-wheel-build.txt
+
+ENV PYTHON="${python}t"
diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
b/ci/docker/python-wheel-windows-test-vs2019-base.dockerfile
similarity index 68%
copy from ci/docker/python-wheel-windows-test-vs2019.dockerfile
copy to ci/docker/python-wheel-windows-test-vs2019-base.dockerfile
index 564bca329f..73a78da30b 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019-base.dockerfile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
+# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env
# when you update this file.
# based on mcr.microsoft.com/windows/servercore:ltsc2019
@@ -27,10 +27,10 @@ FROM abrarov/msvc-2019:2.11.0
# Add unix tools to path
RUN setx path "%path%;C:\Program Files\Git\usr\bin"
-# 1. Remove previous installations of python from the base image
-# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
-# and the msi installers are failing to remove pip and tcl/tk "products" making
-# the subsequent choco python installation step failing for installing python
+# 1. Remove previous installations of Python from the base image
+# NOTE: a more recent base image (tried with 2.12.1) comes with Python 3.9.7
+# and the MSI installers are failing to remove pip and tcl/tk "products" making
+# the subsequent choco python installation step failing for installing Python
# version 3.9.* due to existing python version
# 2. Install Minio for S3 testing.
RUN wmic product where "name like 'python%%'" call uninstall /nointeractive &&
\
@@ -38,26 +38,14 @@ RUN wmic product where "name like 'python%%'" call
uninstall /nointeractive && \
curl
https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z
\
--output "C:\Windows\Minio.exe"
+# Install archiver to extract xz archives (for timezone database).
# Install the GCS testbench using a well-known Python version.
# NOTE: cannot use pipx's `--fetch-missing-python` because of
# https://github.com/pypa/pipx/issues/1521, therefore download Python
ourselves.
-RUN choco install -r -y --pre --no-progress python --version=3.11.9
+RUN choco install --no-progress -r -y archiver && \
+ choco install -r -y --pre --no-progress python --version=3.11.9
ENV PIPX_BIN_DIR=C:\\Windows\\
ENV PIPX_PYTHON="C:\Python311\python.exe"
COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/
RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
storage-testbench -h
-
-# Define the full version number otherwise choco falls back to patch number 0
(3.9 => 3.9.0)
-ARG python=3.9
-RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13") & \
- (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11") & \
- (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9") & \
- (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.5") & \
- (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1")
-
-# Install archiver to extract xz archives
-RUN choco install -r -y --pre --no-progress --force python
--version=%PYTHON_VERSION% && \
- choco install --no-progress -r -y archiver
-
-ENV PYTHON=$python
diff --git a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
index 564bca329f..f80e7ba0fe 100644
--- a/ci/docker/python-wheel-windows-test-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-test-vs2019.dockerfile
@@ -15,49 +15,31 @@
# specific language governing permissions and limitations
# under the License.
-# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
+# NOTE: You must update PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION in .env
# when you update this file.
-# based on mcr.microsoft.com/windows/servercore:ltsc2019
-# contains choco and vs2019 preinstalled
-FROM abrarov/msvc-2019:2.11.0
+ARG base
+# https://github.com/hadolint/hadolint/wiki/DL3006
+# (Hadolint does not expand variables and thinks '${base}' is an untagged
image)
+# hadolint ignore=DL3006
+FROM ${base}
# hadolint shell=cmd.exe
-# Add unix tools to path
-RUN setx path "%path%;C:\Program Files\Git\usr\bin"
-
-# 1. Remove previous installations of python from the base image
-# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
-# and the msi installers are failing to remove pip and tcl/tk "products" making
-# the subsequent choco python installation step failing for installing python
-# version 3.9.* due to existing python version
-# 2. Install Minio for S3 testing.
-RUN wmic product where "name like 'python%%'" call uninstall /nointeractive &&
\
- rm -rf Python* && \
- curl
https://dl.min.io/server/minio/release/windows-amd64/archive/minio.RELEASE.2024-09-13T20-26-02Z
\
- --output "C:\Windows\Minio.exe"
-
-# Install the GCS testbench using a well-known Python version.
-# NOTE: cannot use pipx's `--fetch-missing-python` because of
-# https://github.com/pypa/pipx/issues/1521, therefore download Python
ourselves.
-RUN choco install -r -y --pre --no-progress python --version=3.11.9
-ENV PIPX_BIN_DIR=C:\\Windows\\
-ENV PIPX_PYTHON="C:\Python311\python.exe"
-COPY ci/scripts/install_gcs_testbench.bat C:/arrow/ci/scripts/
-RUN call "C:\arrow\ci\scripts\install_gcs_testbench.bat" && \
- storage-testbench -h
-
# Define the full version number otherwise choco falls back to patch number 0
(3.9 => 3.9.0)
ARG python=3.9
-RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13") & \
- (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11") & \
- (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9") & \
- (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.5") & \
- (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1")
-
-# Install archiver to extract xz archives
-RUN choco install -r -y --pre --no-progress --force python
--version=%PYTHON_VERSION% && \
- choco install --no-progress -r -y archiver
+RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PYTHON_CMD
"C:\Python39\python") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD
"py -3.10") & \
+ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py
-3.11") & \
+ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.8" && setx PYTHON_CMD "py
-3.12") & \
+ (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1" && setx PYTHON_CMD "py
-3.13")
+
+# hadolint ignore=DL3059
+RUN choco install -r -y --pre --no-progress --force python
--version=%PYTHON_VERSION%
+# hadolint ignore=DL3059
+RUN %PYTHON_CMD% -m pip install -U pip setuptools
+
+COPY python/requirements-wheel-test.txt C:/arrow/python/
+RUN %PYTHON_CMD% -m pip install -r C:/arrow/python/requirements-wheel-test.txt
ENV PYTHON=$python
diff --git a/ci/docker/python-wheel-windows-vs2019.dockerfile
b/ci/docker/python-wheel-windows-vs2019-base.dockerfile
similarity index 69%
copy from ci/docker/python-wheel-windows-vs2019.dockerfile
copy to ci/docker/python-wheel-windows-vs2019-base.dockerfile
index cc8c17efe4..bd91f01bf9 100644
--- a/ci/docker/python-wheel-windows-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2019-base.dockerfile
@@ -23,7 +23,7 @@
FROM abrarov/msvc-2019:2.11.0
# Install CMake and Ninja
-ARG cmake=3.21.4
+ARG cmake=3.31.2
RUN choco install --no-progress -r -y cmake --version=%cmake% --installargs
'ADD_CMAKE_TO_PATH=System' && \
choco install --no-progress -r -y gzip wget ninja
@@ -70,27 +70,10 @@ RUN vcpkg install \
--x-feature=parquet \
--x-feature=s3
-# Remove previous installations of python from the base image
-# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
-# and the msi installers are failing to remove pip and tcl/tk "products" making
-# the subsequent choco python installation step failing for installing python
-# version 3.9.* due to existing python version
+# Remove previous installations of Python from the base image
+# NOTE: a more recent base image (tried with 2.12.1) comes with Python 3.9.7
+# and the MSI installers are failing to remove pip and tcl/tk "products" making
+# the subsequent choco python installation step failing for installing Python
+# version 3.9.* due to existing Python version
RUN wmic product where "name like 'python%%'" call uninstall /nointeractive &&
\
rm -rf Python*
-
-# Define the full version number otherwise choco falls back to patch number 0
(3.9 => 3.9.0)
-ARG python=3.9
-RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH
"%PATH%;C:\Python39;C:\Python39\Scripts") & \
- (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH
"%PATH%;C:\Python310;C:\Python310\Scripts") & \
- (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH
"%PATH%;C:\Python311;C:\Python311\Scripts") & \
- (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.5" && setx PATH
"%PATH%;C:\Python312;C:\Python312\Scripts") & \
- (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH
"%PATH%;C:\Python313;C:\Python313\Scripts")
-RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
-RUN python -m pip install -U pip setuptools
-
-COPY python/requirements-wheel-build.txt arrow/python/
-RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
-
-# For debugging purposes
-# RUN wget --no-check-certificate
https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip
-# RUN unzip Dependencies_x64_Release.zip -d Dependencies && setx path
"%path%;C:\Dependencies"
diff --git a/ci/docker/python-wheel-windows-vs2019.dockerfile
b/ci/docker/python-wheel-windows-vs2019.dockerfile
index cc8c17efe4..50e942fd6b 100644
--- a/ci/docker/python-wheel-windows-vs2019.dockerfile
+++ b/ci/docker/python-wheel-windows-vs2019.dockerfile
@@ -18,79 +18,22 @@
# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
# when you update this file.
-# based on mcr.microsoft.com/windows/servercore:ltsc2019
-# contains choco and vs2019 preinstalled
-FROM abrarov/msvc-2019:2.11.0
-
-# Install CMake and Ninja
-ARG cmake=3.21.4
-RUN choco install --no-progress -r -y cmake --version=%cmake% --installargs
'ADD_CMAKE_TO_PATH=System' && \
- choco install --no-progress -r -y gzip wget ninja
-
-# Add unix tools to path
-RUN setx path "%path%;C:\Program Files\Git\usr\bin"
-
-# Install vcpkg
-#
-# Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has
-# started to ship precompiled binaries for the vcpkg-tool.
-ARG vcpkg
-COPY ci/vcpkg/*.patch \
- ci/vcpkg/*windows*.cmake \
- arrow/ci/vcpkg/
-COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
-ENV VCPKG_ROOT=C:\\vcpkg
-RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
- setx PATH "%PATH%;%VCPKG_ROOT%"
-
-# Configure vcpkg and install dependencies
-# NOTE: use windows batch environment notation for build arguments in RUN
-# statements but bash notation in ENV statements
-# VCPKG_FORCE_SYSTEM_BINARIES=1 spare around ~750MB of image size if the system
-# cmake's and ninja's versions are recent enough
-ARG build_type=release
-ENV CMAKE_BUILD_TYPE=${build_type} \
- VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
- VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
- VCPKG_FEATURE_FLAGS="manifests"
-COPY ci/vcpkg/vcpkg.json arrow/ci/vcpkg/
-# cannot use the S3 feature here because while aws-sdk-cpp=1.9.160 contains
-# ssl related fixes as well as we can patch the vcpkg portfile to support
-# arm machines it hits ARROW-15141 where we would need to fall back to 1.8.186
-# but we cannot patch those portfiles since vcpkg-tool handles the checkout of
-# previous versions => use bundled S3 build
-RUN vcpkg install \
- --clean-after-build \
- --x-install-root=%VCPKG_ROOT%\installed \
- --x-manifest-root=arrow/ci/vcpkg \
- --x-feature=flight \
- --x-feature=gcs \
- --x-feature=json \
- --x-feature=orc \
- --x-feature=parquet \
- --x-feature=s3
-
-# Remove previous installations of python from the base image
-# NOTE: a more recent base image (tried with 2.12.1) comes with python 3.9.7
-# and the msi installers are failing to remove pip and tcl/tk "products" making
-# the subsequent choco python installation step failing for installing python
-# version 3.9.* due to existing python version
-RUN wmic product where "name like 'python%%'" call uninstall /nointeractive &&
\
- rm -rf Python*
+ARG base
+FROM ${base}
# Define the full version number otherwise choco falls back to patch number 0
(3.9 => 3.9.0)
+# Note that Python 3.9 does not come with the "py" launcher
ARG python=3.9
-RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PATH
"%PATH%;C:\Python39;C:\Python39\Scripts") & \
- (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PATH
"%PATH%;C:\Python310;C:\Python310\Scripts") & \
- (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PATH
"%PATH%;C:\Python311;C:\Python311\Scripts") & \
- (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.5" && setx PATH
"%PATH%;C:\Python312;C:\Python312\Scripts") & \
- (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.0-rc1" && setx PATH
"%PATH%;C:\Python313;C:\Python313\Scripts")
+RUN (if "%python%"=="3.9" setx PYTHON_VERSION "3.9.13" && setx PYTHON_CMD
"C:\Python39\python") & \
+ (if "%python%"=="3.10" setx PYTHON_VERSION "3.10.11" && setx PYTHON_CMD
"py -3.10") & \
+ (if "%python%"=="3.11" setx PYTHON_VERSION "3.11.9" && setx PYTHON_CMD "py
-3.11") & \
+ (if "%python%"=="3.12" setx PYTHON_VERSION "3.12.8" && setx PYTHON_CMD "py
-3.12") & \
+ (if "%python%"=="3.13" setx PYTHON_VERSION "3.13.1" && setx PYTHON_CMD "py
-3.13")
+
RUN choco install -r -y --pre --no-progress python --version=%PYTHON_VERSION%
-RUN python -m pip install -U pip setuptools
+RUN %PYTHON_CMD% -m pip install -U pip setuptools
-COPY python/requirements-wheel-build.txt arrow/python/
-RUN python -m pip install -r arrow/python/requirements-wheel-build.txt
+COPY python/requirements-wheel-build.txt C:/arrow/python/
+RUN %PYTHON_CMD% -m pip install -r C:/arrow/python/requirements-wheel-build.txt
-# For debugging purposes
-# RUN wget --no-check-certificate
https://github.com/lucasg/Dependencies/releases/download/v1.10/Dependencies_x64_Release.zip
-# RUN unzip Dependencies_x64_Release.zip -d Dependencies && setx path
"%path%;C:\Dependencies"
+ENV PYTHON=${python}
diff --git a/ci/scripts/python_wheel_windows_build.bat
b/ci/scripts/python_wheel_windows_build.bat
index 999f40980f..9287d2471d 100644
--- a/ci/scripts/python_wheel_windows_build.bat
+++ b/ci/scripts/python_wheel_windows_build.bat
@@ -19,6 +19,11 @@
echo "Building windows wheel..."
+@REM List installed Pythons
+py -0p
+
+%PYTHON_CMD% -m sysconfig || exit /B 1
+
call "C:\Program Files (x86)\Microsoft Visual
Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat"
@echo on
@@ -27,7 +32,7 @@ choco install -r -y --no-progress vcredist140
choco upgrade -r -y --no-progress vcredist140
dir C:\Windows\System32\msvcp140.dll
-echo "=== (%PYTHON_VERSION%) Clear output directories and leftovers ==="
+echo "=== (%PYTHON%) Clear output directories and leftovers ==="
del /s /q C:\arrow-build
del /s /q C:\arrow-dist
del /s /q C:\arrow\python\dist
@@ -35,7 +40,7 @@ del /s /q C:\arrow\python\build
del /s /q C:\arrow\python\pyarrow\*.so
del /s /q C:\arrow\python\pyarrow\*.so.*
-echo "=== (%PYTHON_VERSION%) Building Arrow C++ libraries ==="
+echo "=== (%PYTHON%) Building Arrow C++ libraries ==="
set ARROW_ACERO=ON
set ARROW_DATASET=ON
set ARROW_FLIGHT=ON
@@ -107,8 +112,9 @@ cmake ^
cmake --build . --config %CMAKE_BUILD_TYPE% --target install || exit /B 1
popd
-echo "=== (%PYTHON_VERSION%) Building wheel ==="
+echo "=== (%PYTHON%) Building wheel ==="
set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
+set PYARROW_BUILD_VERBOSE=1
set PYARROW_BUNDLE_ARROW_CPP=ON
set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
set PYARROW_WITH_ACERO=%ARROW_ACERO%
@@ -131,7 +137,7 @@ pushd C:\arrow\python
cp C:\Windows\System32\msvcp140.dll pyarrow\
@REM Build wheel
-python setup.py bdist_wheel || exit /B 1
+%PYTHON_CMD% setup.py bdist_wheel || exit /B 1
@REM Repair the wheel with delvewheel
@REM
@@ -141,12 +147,13 @@ python setup.py bdist_wheel || exit /B 1
@REM
@REM For now this requires a custom version of delvewheel:
@REM https://github.com/adang1345/delvewheel/pull/59
-pip install
https://github.com/pitrou/delvewheel/archive/refs/heads/fixes-for-arrow.zip ||
exit /B 1
+%PYTHON_CMD% -m pip install
https://github.com/pitrou/delvewheel/archive/refs/heads/fixes-for-arrow.zip ||
exit /B 1
for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i)
|| exit /B 1
echo "Wheel name: %WHEEL_NAME%"
-delvewheel repair -vv --mangle-only=msvcp140.dll --no-patch ^
+%PYTHON_CMD% -m delvewheel repair -vv ^
+ --mangle-only=msvcp140.dll --no-patch ^
-w repaired_wheels %WHEEL_NAME% || exit /B 1
popd
diff --git a/ci/scripts/python_wheel_windows_test.bat
b/ci/scripts/python_wheel_windows_test.bat
index 12d35216b1..ffe8b388f9 100755
--- a/ci/scripts/python_wheel_windows_test.bat
+++ b/ci/scripts/python_wheel_windows_test.bat
@@ -25,28 +25,19 @@ set PYARROW_TEST_GANDIVA=OFF
set PYARROW_TEST_GCS=ON
set PYARROW_TEST_HDFS=ON
set PYARROW_TEST_ORC=ON
+set PYARROW_TEST_PANDAS=ON
set PYARROW_TEST_PARQUET=ON
set PYARROW_TEST_PARQUET_ENCRYPTION=ON
set PYARROW_TEST_SUBSTRAIT=ON
set PYARROW_TEST_S3=ON
set PYARROW_TEST_TENSORFLOW=ON
-@REM Enable again once https://github.com/scipy/oldest-supported-numpy/pull/27
gets merged
-@REM set PYARROW_TEST_PANDAS=ON
-
set ARROW_TEST_DATA=C:\arrow\testing\data
set PARQUET_TEST_DATA=C:\arrow\cpp\submodules\parquet-testing\data
@REM List installed Pythons
py -0p
-set PYTHON_CMD=py -%PYTHON%
-
-%PYTHON_CMD% -m pip install -U pip setuptools || exit /B 1
-
-@REM Install testing dependencies
-%PYTHON_CMD% -m pip install -r C:\arrow\python\requirements-wheel-test.txt ||
exit /B 1
-
@REM Install the built wheels
%PYTHON_CMD% -m pip install --no-index
--find-links=C:\arrow\python\repaired_wheels pyarrow || exit /B 1
@@ -70,8 +61,9 @@ set PYTHON_CMD=py -%PYTHON%
@rem Download IANA Timezone Database for ORC C++
curl https://cygwin.osuosl.org/noarch/release/tzdata/tzdata-2024a-1.tar.xz
--output tzdata.tar.xz || exit /B
mkdir %USERPROFILE%\Downloads\test\tzdata
-arc unarchive tzdata.tar.xz %USERPROFILE%\Downloads\test\tzdata
+arc unarchive tzdata.tar.xz %USERPROFILE%\Downloads\test\tzdata || exit /B
set TZDIR=%USERPROFILE%\Downloads\test\tzdata\usr\share\zoneinfo
+dir %TZDIR%
@REM Execute unittest
%PYTHON_CMD% -m pytest -r s --pyargs pyarrow || exit /B 1
diff --git a/dev/tasks/python-wheels/github.windows.yml
b/dev/tasks/python-wheels/github.windows.yml
index de0efedbff..36b25e9819 100644
--- a/dev/tasks/python-wheels/github.windows.yml
+++ b/dev/tasks/python-wheels/github.windows.yml
@@ -26,6 +26,7 @@ jobs:
env:
# archery uses this environment variable
PYTHON: "{{ python_version }}"
+ PYTHON_ABI_TAG: "{{ python_abi_tag }}"
# this is a private repository at the moment (mostly because of licensing
# consideration of windows images with visual studio), but anyone can
# recreate the image by manually building it via:
@@ -43,6 +44,19 @@ jobs:
{{ macros.github_login_ghcr()|indent }}
{{ macros.github_install_archery()|indent }}
+ - name: Prepare
+ shell: bash
+ run: |
+ case "${PYTHON_ABI_TAG}" in
+ *t)
+ test_image_prefix=python-free-threaded
+ ;;
+ *)
+ test_image_prefix=python
+ ;;
+ esac
+ echo "TEST_IMAGE_PREFIX=${test_image_prefix}" >> ${GITHUB_ENV}
+
- name: Build wheel
shell: cmd
run: |
@@ -54,11 +68,11 @@ jobs:
@rem We can remove this workaround once we find a way to use
@rem pulled caches when build an image.
echo on
- archery docker pull --no-ignore-pull-failures
python-wheel-windows-vs2019
+ archery docker pull --no-ignore-pull-failures
%TEST_IMAGE_PREFIX%-wheel-windows-vs2019
if errorlevel 1 (
- archery docker build --no-pull python-wheel-windows-vs2019 || exit
/B 1
+ archery docker build --no-pull
%TEST_IMAGE_PREFIX%-wheel-windows-vs2019 || exit /B 1
)
- archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{
arrow.no_rc_version }} python-wheel-windows-vs2019
+ archery docker run --no-build -e SETUPTOOLS_SCM_PRETEND_VERSION={{
arrow.no_rc_version }} %TEST_IMAGE_PREFIX%-wheel-windows-vs2019
- uses: actions/upload-artifact@v4
with:
@@ -69,16 +83,21 @@ jobs:
shell: cmd
run: |
cd arrow
- archery docker run python-wheel-windows-test
+ archery docker pull --no-ignore-pull-failures
%TEST_IMAGE_PREFIX%-wheel-windows-test
+ if errorlevel 1 (
+ archery docker build --no-pull
%TEST_IMAGE_PREFIX%-wheel-windows-test || exit /B 1
+ )
+ archery docker run %TEST_IMAGE_PREFIX%-wheel-windows-test
{{
macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
{{
macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
{{
macros.github_upload_wheel_scientific_python("arrow/repaired_wheels/repaired_wheels/*.whl")|indent
}}
{% if arrow.is_default_branch() %}
- - name: Push Docker Image
+ - name: Push Docker image
shell: cmd
run: |
cd arrow
- archery docker push python-wheel-windows-vs2019
+ archery docker push %TEST_IMAGE_PREFIX%-wheel-windows-vs2019
+ archery docker push %TEST_IMAGE_PREFIX%-wheel-windows-test
{% endif %}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 9f04d33f83..f5b5ab53eb 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -445,19 +445,15 @@ tasks:
{############################## Wheel Windows ################################}
-# TODO: Remove this when there's NumPy wheels for Windows.
-# See https://github.com/numpy/numpy/issues/26157 for more info.
-{% if abi_tag != "cp313t" %}
-
- wheel-windows-{{ python_tag }}-amd64:
+ wheel-windows-{{ python_tag }}-{{ abi_tag }}-amd64:
ci: github
template: python-wheels/github.windows.yml
params:
python_version: "{{ python_version }}"
+ python_abi_tag: "{{ abi_tag }}"
artifacts:
- pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-win_amd64.whl
-{% endif %}
{% endfor %}
{############################ Python sdist
####################################}
diff --git a/docker-compose.yml b/docker-compose.yml
index bd91209563..5cb96e62c1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -176,8 +176,12 @@ x-hierarchy:
- python-free-threaded-wheel-manylinux-test-imports
- python-wheel-manylinux-test-unittests
- python-free-threaded-wheel-manylinux-test-unittests
- - python-wheel-windows-vs2019
- - python-wheel-windows-test
+ - python-wheel-windows-vs2019-base:
+ - python-wheel-windows-vs2019
+ - python-free-threaded-wheel-windows-vs2019
+ - python-wheel-windows-test-base:
+ - python-wheel-windows-test
+ - python-free-threaded-wheel-windows-test
volumes:
almalinux-ccache:
@@ -1262,12 +1266,29 @@ services:
CHECK_UNITTESTS: "ON"
command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow
+ python-wheel-windows-vs2019-base:
+ image:
${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+ build:
+ args:
+ vcpkg: ${VCPKG}
+ context: .
+ dockerfile: ci/docker/python-wheel-windows-vs2019-base.dockerfile
+ # This should make the pushed images reusable, but the image gets
rebuilt.
+ # Uncomment if no local cache is available.
+ # cache_from:
+ # - abrarov/msvc-2019:2.11.0
+ # -
${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+ volumes: &python-wheel-windows-vs2019-volumes
+ - type: bind
+ source: .
+ target: "C:/arrow"
+
python-wheel-windows-vs2019:
image:
${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
build:
args:
+ base:
${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
python: ${PYTHON}
- vcpkg: ${VCPKG}
context: .
dockerfile: ci/docker/python-wheel-windows-vs2019.dockerfile
# This should make the pushed images reusable, but the image gets
rebuilt.
@@ -1275,23 +1296,51 @@ services:
# cache_from:
# - abrarov/msvc-2019:2.11.0
# -
${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
- volumes:
- - type: bind
- source: .
- target: "C:/arrow"
+ volumes: *python-wheel-windows-vs2019-volumes
command: arrow\\ci\\scripts\\python_wheel_windows_build.bat
+ python-free-threaded-wheel-windows-vs2019:
+ image:
${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+ build:
+ args:
+ base:
${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+ python: ${PYTHON}
+ context: .
+ dockerfile:
ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile
+ # This should make the pushed images reusable, but the image gets
rebuilt.
+ # Uncomment if no local cache is available.
+ # cache_from:
+ # - abrarov/msvc-2019:2.11.0
+ # -
${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+ volumes: *python-wheel-windows-vs2019-volumes
+ command: arrow\\ci\\scripts\\python_wheel_windows_build.bat
+
+ python-wheel-windows-test-base:
+ image:
${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
+ build:
+ context: .
+ dockerfile: ci/docker/python-wheel-windows-test-vs2019-base.dockerfile
+ volumes: *python-wheel-windows-vs2019-volumes
+
python-wheel-windows-test:
- image:
${REPO}:python-${PYTHON}-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
+ image:
${REPO}:python-${PYTHON}-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
build:
args:
+ base:
${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
python: ${PYTHON}
context: .
dockerfile: ci/docker/python-wheel-windows-test-vs2019.dockerfile
- volumes:
- - type: bind
- source: .
- target: "C:/arrow"
+ volumes: *python-wheel-windows-vs2019-volumes
+ command: arrow\\ci\\scripts\\python_wheel_windows_test.bat
+
+ python-free-threaded-wheel-windows-test:
+ image:
${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
+ build:
+ args:
+ base:
${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
+ context: .
+ dockerfile:
ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile
+ volumes: *python-wheel-windows-vs2019-volumes
command: arrow\\ci\\scripts\\python_wheel_windows_test.bat
############################## Integration #################################
