This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new d311bbf22f ci: run binary verification on Windows (x64 and ARM) (#4796)
d311bbf22f is described below
commit d311bbf22f7a1dfeeafc2d1e78094382fc71d6fb
Author: David Li <[email protected]>
AuthorDate: Thu Sep 24 10:34:21 2026 +0900
ci: run binary verification on Windows (x64 and ARM) (#4796)
Closes #4685.
---
.github/workflows/verify.yml | 78 ++++++-
dev/release/verify-release-candidate.ps1 | 234 ++++++++++++++-------
.../adbc/verify/VerifyReleaseCandidateTest.java | 17 +-
3 files changed, 248 insertions(+), 81 deletions(-)
diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml
index f6ccfefac0..f9ce4a6ed4 100644
--- a/.github/workflows/verify.yml
+++ b/.github/workflows/verify.yml
@@ -44,18 +44,69 @@ defaults:
shell: bash
jobs:
- binary-unix:
+ binary:
name: "Verify Binaries/${{ matrix.os }}"
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
- os: ["macos-15-intel", "macos-latest", "ubuntu-latest",
"windows-latest"]
+ os: ["macos-15-intel", "macos-latest", "ubuntu-latest",
"windows-latest", "windows-11-arm"]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 #
v7.0.1
with:
fetch-depth: 0
persist-credentials: false
+ - uses:
conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 #
v4.0.1
+ if: matrix.os == 'windows-latest'
+ with:
+ miniforge-version: latest
+ use-mamba: true
+ - name: Install Python 3.11
+ if: matrix.os == 'windows-11-arm'
+ id: python311
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
+ with:
+ architecture: arm64
+ check-latest: true
+ python-version: "3.11"
+ # Keep this interpreter on PATH for the artifact downloader.
+ update-environment: true
+ - name: Install Python 3.12
+ if: matrix.os == 'windows-11-arm'
+ id: python312
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
+ with:
+ architecture: arm64
+ check-latest: true
+ python-version: "3.12"
+ update-environment: false
+ - name: Install Python 3.13
+ if: matrix.os == 'windows-11-arm'
+ id: python313
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
+ with:
+ architecture: arm64
+ check-latest: true
+ python-version: "3.13"
+ update-environment: false
+ - name: Install Python 3.14
+ if: matrix.os == 'windows-11-arm'
+ id: python314
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
+ with:
+ architecture: arm64
+ check-latest: true
+ python-version: "3.14"
+ update-environment: false
+ - name: Install Python 3.14t
+ if: matrix.os == 'windows-11-arm'
+ id: python314t
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 #
v7.0.0
+ with:
+ architecture: arm64
+ check-latest: true
+ python-version: "3.14t"
+ update-environment: false
# No Docker on M1
- name: Skip Binary Verifiction
if: matrix.os == 'macos-latest'
@@ -63,6 +114,7 @@ jobs:
echo "TEST_APT=0" | tee -a $GITHUB_ENV
echo "TEST_YUM=0" | tee -a $GITHUB_ENV
- name: Verify
+ if: runner.os != 'Windows'
env:
CI: "false"
REPOSITORY: ${{ github.repository }}
@@ -77,6 +129,28 @@ jobs:
run: |
./dev/release/verify-release-candidate.sh ${INPUTS_VERSION}
${INPUTS_RC}
+ - name: Verify
+ if: runner.os == 'Windows'
+ shell: pwsh
+ env:
+ CI: "false"
+ PYTHON_311_PATH: ${{ steps.python311.outputs.python-path }}
+ PYTHON_312_PATH: ${{ steps.python312.outputs.python-path }}
+ PYTHON_313_PATH: ${{ steps.python313.outputs.python-path }}
+ PYTHON_314_PATH: ${{ steps.python314.outputs.python-path }}
+ PYTHON_314T_PATH: ${{ steps.python314t.outputs.python-path }}
+ REPOSITORY: ${{ github.repository }}
+ TEST_DEFAULT: "0"
+ TEST_BINARIES: "1"
+ USE_CONDA: "1"
+ VERBOSE: "1"
+ # Make this available to download_rc_binaries.py
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ INPUTS_VERSION: ${{ inputs.version }}
+ INPUTS_RC: ${{ inputs.rc }}
+ run: |
+ ./dev/release/verify-release-candidate.ps1 $env:INPUTS_VERSION
$env:INPUTS_RC
+
source-conda:
name: "Verify Source (Conda)/${{ matrix.os }}"
runs-on: ${{ matrix.os }}
diff --git a/dev/release/verify-release-candidate.ps1
b/dev/release/verify-release-candidate.ps1
index 9f1feaa540..86897cbd4c 100755
--- a/dev/release/verify-release-candidate.ps1
+++ b/dev/release/verify-release-candidate.ps1
@@ -51,7 +51,9 @@ Usage:
`$env:TEST_WHEELS = "1"
$($script) X.Y.Z RC_NUMBER
-Assumes Mamba is set up and available on the path.
+Assumes Mamba is set up and available on the path, except for Windows ARM64
+binary verification. On Windows ARM64, supply Python executable paths using
+PYTHON_310_PATH, etc. (Source verification is not supported on Windows/ARM64.)
"@
exit 1
}
@@ -127,6 +129,145 @@ function Enable-Conda {
}
}
+function Test-PythonWheels($PythonVersion, $PythonPath) {
+ Show-Header "Verify Python $($PythonVersion) Wheels for
$($Platform)/$($Architecture)"
+ $FreeThreaded = $PythonVersion.EndsWith("t")
+ $PythonTag = "cp$($PythonVersion.TrimEnd('t').Replace('.', ''))"
+ if ($FreeThreaded) {
+ $AbiTag = "$($PythonTag)t"
+ } else {
+ $AbiTag = $PythonTag
+ }
+ $IsWindowsArm = $IsWindowsPlatform -and $Architecture -eq "arm64"
+
+ $env:PYTHON_VERSION = $PythonVersion
+ $env:ADBC_NO_SKIP_TESTS = "0"
+ if ($IsWindowsArm -and -not $FreeThreaded) {
+ $env:ADBC_NO_SKIP_TESTS = "1"
+ }
+ try {
+ if ($FreeThreaded) {
+ $env:PYTHON_GIL = "0"
+ } else {
+ Remove-Item -Path Env:PYTHON_GIL -ErrorAction SilentlyContinue
+ }
+
+ $WheelPaths = @()
+ foreach ($Component in $Components) {
+ if ($Component -eq "adbc_driver_manager") {
+ $WheelPattern =
"$($Component)-*-$($PythonTag)-$($AbiTag)-$($PlatformPattern).whl"
+ } else {
+ $WheelPattern =
"$($Component)-*-py3-none-$($PlatformPattern).whl"
+ }
+ $MatchingWheels = @(Get-ChildItem -Path $BinaryDir -Filter
$WheelPattern)
+ if ($MatchingWheels.Count -ne 1) {
+ throw "Expected exactly one $($Component) wheel matching
$($WheelPattern), found $($MatchingWheels.Count)"
+ }
+ $WheelPaths += $MatchingWheels[0].FullName
+ }
+
+ & $PythonPath -m pip install --no-deps --force-reinstall $WheelPaths
+ if (-not $?) { throw "Failed to install Python $($PythonVersion)
wheels" }
+
+ $TestDependencies = @("pytest", "typing-extensions")
+ if ($IsWindowsPlatform -and -not $IsWindowsArm) {
+ $TestDependencies += @("pyarrow", "pandas", "protobuf")
+ }
+ & $PythonPath -m pip install $TestDependencies
+ if (-not $?) { throw "Failed to install Python $($PythonVersion) wheel
test dependencies" }
+
+ if (-not $IsWindowsPlatform) {
+ & $(Join-Path $AdbcSourceDir
"ci/scripts/python_wheel_unix_test.sh") $AdbcSourceDir
+ if (-not $?) { throw "Failed to test wheels with Python
$($PythonVersion)" }
+ return
+ }
+
+ foreach ($Component in $Components) {
+ & $PythonPath -c "import $($Component); import $($Component).dbapi"
+ if (-not $?) { throw "Failed to import $($Component) with Python
$($PythonVersion)" }
+ }
+
+ if (-not $IsWindowsArm) {
+ foreach ($Component in $Components) {
+ echo "Testing $($Component)"
+ $TestsPath = Join-Path $AdbcSourceDir
"python/$($Component)/tests"
+ & $PythonPath -m pytest -vvx --import-mode=append `
+ -k "not duckdb and not sqlite and not polars" $TestsPath
+ if (-not $?) { throw "Failed to test $($Component) with Python
$($PythonVersion)" }
+ }
+ } elseif ($FreeThreaded) {
+ & $PythonPath -m pytest -vvx --import-mode append `
+ -k "unknown_driver or missing_platform or bad" `
+ (Join-Path $AdbcSourceDir
"python/adbc_driver_manager/tests/test_manifest.py")
+ if (-not $?) { throw "Failed to test the free-threaded driver
manager with Python $($PythonVersion)" }
+ } else {
+ & $PythonPath -m pip install polars
+ if (-not $?) { throw "Failed to install Polars with Python
$($PythonVersion)" }
+ # The fixture loads SQLite by name, but wheels bundle the library
inside
+ # the Python package as libadbc_driver_sqlite.so, even on Windows.
+ $SqliteDriverPath = & $PythonPath -c "import adbc_driver_sqlite;
print(adbc_driver_sqlite._driver_path())"
+ if (-not $?) { throw "Failed to locate the SQLite driver with
Python $($PythonVersion)" }
+ $DriverManifestDir = Join-Path $ArrowTempDir
"wheel-drivers-$($PythonVersion)-$($Platform)-$($Architecture)"
+ New-Item -ItemType Directory -Force -Path $DriverManifestDir |
Out-Null
+ $DriverManifest = Join-Path $DriverManifestDir
"adbc_driver_sqlite.toml"
+ $DriverPathJson = $SqliteDriverPath | ConvertTo-Json -Compress
+ [System.IO.File]::WriteAllText($DriverManifest, "[Driver]`nshared
= $($DriverPathJson)`n")
+ try {
+ $env:ADBC_DRIVER_PATH = $DriverManifestDir
+ & $PythonPath -m pytest -vvx --import-mode append `
+ (Join-Path $AdbcSourceDir
"python/adbc_driver_manager/tests/test_dbapi_polars_nopyarrow.py")
+ if (-not $?) { throw "Failed to test the driver manager with
Polars with Python $($PythonVersion)" }
+ } finally {
+ Remove-Item -Path Env:ADBC_DRIVER_PATH -ErrorAction
SilentlyContinue
+ }
+ }
+ } finally {
+ Remove-Item -Path Env:PYTHON_GIL -ErrorAction SilentlyContinue
+ }
+}
+
+function Test-PythonWheelsConda($PythonVersion) {
+ Enable-Conda
+ $CondaEnv = Join-Path $ArrowTempDir
"wheel-$($PythonVersion)-$($Platform)-$($Architecture)"
+ if ($IsWindowsPlatform) {
+ $CondaPython = Join-Path $CondaEnv "python.exe"
+ } else {
+ $CondaPython = Join-Path $CondaEnv "bin/python"
+ }
+ if (-not (Test-Path -Path $CondaPython -PathType Leaf)) {
+ $CondaPackages = @("python=$($PythonVersion.TrimEnd('t'))")
+ if ($PythonVersion.EndsWith("t")) {
+ $CondaPackages += "python-freethreading"
+ }
+ mamba create -c conda-forge --yes --prefix $CondaEnv $CondaPackages
+ if (-not $?) { throw "Failed to create Python $($PythonVersion) Conda
environment" }
+ } else {
+ echo "Using cached $($CondaEnv)"
+ }
+ conda activate $CondaEnv
+ Test-PythonWheels $PythonVersion $CondaPython
+}
+
+function Test-PythonWheelsWithInterpreter($PythonVersion) {
+ $PythonPathVariable = "PYTHON_$($PythonVersion.Replace('.',
'').ToUpperInvariant())_PATH"
+ $PythonPath = [Environment]::GetEnvironmentVariable($PythonPathVariable)
+ if ([string]::IsNullOrEmpty($PythonPath)) {
+ echo "Skipping Python $($PythonVersion): $($PythonPathVariable) is not
set or empty"
+ return
+ }
+ if (-not (Test-Path -LiteralPath $PythonPath -PathType Leaf)) {
+ throw "Invalid Python executable in $($PythonPathVariable):
$($PythonPath)"
+ }
+
+ # Let the supplied interpreter use its default GIL setting during setup.
+ Remove-Item -Path Env:PYTHON_GIL -ErrorAction SilentlyContinue
+ $VenvDir = Join-Path $ArrowTempDir
"wheel-venv-$($PythonVersion)-$($Platform)-$($Architecture)"
+ & $PythonPath -m venv --clear $VenvDir
+ if (-not $?) { throw "Failed to create Python $($PythonVersion) virtual
environment" }
+ $VenvPython = Join-Path $VenvDir "Scripts/python.exe"
+ Test-PythonWheels $PythonVersion $VenvPython
+}
+
# ============================================================
# Set up common artifacts
# ============================================================
@@ -264,14 +405,20 @@ if ($TestBinaryArtifacts) {
if ($TestWheels) {
Show-Header "Verify Python Wheels"
+ $UseCondaForWheels = $true
$IsWindowsPlatform = $env:OS -eq "Windows_NT"
if ($IsWindowsPlatform) {
- if ($env:PROCESSOR_ARCHITECTURE -ne "AMD64") {
- throw "Unsupported Windows architecture:
$($env:PROCESSOR_ARCHITECTURE)"
- }
$Platform = "windows"
- $PlatformPattern = "win_amd64"
- $Architecture = "amd64"
+ if ($env:PROCESSOR_ARCHITECTURE -eq "AMD64") {
+ $PlatformPattern = "win_amd64"
+ $Architecture = "amd64"
+ } elseif ($env:PROCESSOR_ARCHITECTURE -eq "ARM64") {
+ $UseCondaForWheels = $false
+ $PlatformPattern = "win_arm64"
+ $Architecture = "arm64"
+ } else {
+ throw "Unknown Windows architecture:
$($env:PROCESSOR_ARCHITECTURE)"
+ }
} else {
$Kernel = (uname -s)
$Machine = (uname -m)
@@ -295,8 +442,6 @@ if ($TestBinaryArtifacts) {
$PlatformPattern = "$($Platform)*$($Architecture)*"
}
- Enable-Conda
-
if ($env:TEST_PYTHON_VERSIONS -eq $null) {
$PythonVersions = @("3.10", "3.11", "3.12", "3.13", "3.14",
"3.14t")
} else {
@@ -311,79 +456,12 @@ if ($TestBinaryArtifacts) {
)
foreach ($PythonVersion in $PythonVersions) {
- Show-Header "Verify Python $($PythonVersion) Wheels for
$($Platform)/$($Architecture)"
-
- $FreeThreaded = $PythonVersion.EndsWith("t")
- $CondaPythonVersion = $PythonVersion.TrimEnd("t")
- $PythonTag = "cp$($CondaPythonVersion.Replace('.', ''))"
- if ($FreeThreaded) {
- $AbiTag = "$($PythonTag)t"
- } else {
- $AbiTag = $PythonTag
- }
-
- $CondaEnv = Join-Path $ArrowTempDir
"wheel-$($PythonVersion)-$($Platform)-$($Architecture)"
- if ($IsWindowsPlatform) {
- $CondaPython = Join-Path $CondaEnv "python.exe"
+ if ($UseCondaForWheels) {
+ Test-PythonWheelsConda $PythonVersion
} else {
- $CondaPython = Join-Path $CondaEnv "bin/python"
- }
- if (-not (Test-Path -Path $CondaPython -PathType Leaf)) {
- $CondaPackages = @("python=$($CondaPythonVersion)")
- if ($FreeThreaded) {
- $CondaPackages += "python-freethreading"
- }
- mamba create -c conda-forge --yes --prefix $CondaEnv
$CondaPackages
- if (-not $?) { throw "Failed to create Python
$($PythonVersion) Conda environment" }
- } else {
- echo "Using cached $($CondaEnv)"
- }
- conda activate $CondaEnv
-
- $WheelPaths = @()
- foreach ($Component in $Components) {
- if ($Component -eq "adbc_driver_manager") {
- $WheelPattern =
"$($Component)-*-$($PythonTag)-$($AbiTag)-$($PlatformPattern).whl"
- } else {
- $WheelPattern =
"$($Component)-*-py3-none-$($PlatformPattern).whl"
- }
- $MatchingWheels = @(Get-ChildItem -Path $BinaryDir -Filter
$WheelPattern)
- if ($MatchingWheels.Count -ne 1) {
- throw "Expected exactly one $($Component) wheel matching
$($WheelPattern), found $($MatchingWheels.Count)"
- }
- $WheelPaths += $MatchingWheels[0].FullName
- }
-
- python -m pip install --force-reinstall $WheelPaths
- if (-not $?) { throw "Failed to install Python $($PythonVersion)
wheels" }
-
- $env:PYTHON_VERSION = $PythonVersion
- if ($FreeThreaded) {
- $env:PYTHON_GIL = "0"
- } else {
- Remove-Item -Path Env:PYTHON_GIL -ErrorAction SilentlyContinue
- }
-
- if ($IsWindowsPlatform) {
- python -m pip install pytest pyarrow pandas protobuf
- if (-not $?) { throw "Failed to install Python
$($PythonVersion) wheel test dependencies" }
-
- foreach ($Component in $Components) {
- echo "Testing $($Component)"
- python -c "import $($Component)"
- if (-not $?) { throw "Failed to import $($Component) with
Python $($PythonVersion)" }
- python -c "import $($Component).dbapi"
- if (-not $?) { throw "Failed to import $($Component).dbapi
with Python $($PythonVersion)" }
- $TestsPath = Join-Path $AdbcSourceDir
"python\$($Component)\tests"
- python -m pytest -vvx --import-mode=append -k "not duckdb
and not sqlite and not polars" $TestsPath
- if (-not $?) { throw "Failed to test $($Component) with
Python $($PythonVersion)" }
- }
- } else {
- & $(Join-Path $AdbcSourceDir
"ci/scripts/python_wheel_unix_test.sh") $AdbcSourceDir
- if (-not $?) { throw "Failed to test wheels with Python
$($PythonVersion)" }
+ Test-PythonWheelsWithInterpreter $PythonVersion
}
}
- Remove-Item -Path Env:PYTHON_GIL -ErrorAction SilentlyContinue
} else {
Show-Header "Skipping Python Wheels"
}
diff --git
a/dev/release/verify/java/src/test/java/org/apache/arrow/adbc/verify/VerifyReleaseCandidateTest.java
b/dev/release/verify/java/src/test/java/org/apache/arrow/adbc/verify/VerifyReleaseCandidateTest.java
index c52a4f755c..83f6d5512d 100644
---
a/dev/release/verify/java/src/test/java/org/apache/arrow/adbc/verify/VerifyReleaseCandidateTest.java
+++
b/dev/release/verify/java/src/test/java/org/apache/arrow/adbc/verify/VerifyReleaseCandidateTest.java
@@ -20,6 +20,7 @@ package org.apache.arrow.adbc.verify;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import java.io.FileNotFoundException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -35,6 +36,7 @@ import org.apache.arrow.memory.RootAllocator;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.OS;
/** Briefly test that shipped JARs actually work. */
class VerifyReleaseCandidateTest {
@@ -82,9 +84,22 @@ class VerifyReleaseCandidateTest {
@Test
void testDriverJni() {
+ var driver = new JniDriver(allocator);
+ if (OS.WINDOWS.isCurrentOs() &&
"aarch64".equalsIgnoreCase(System.getProperty("os.arch"))) {
+ // JNI binaries are not available for Windows ARM64.
+ ExceptionInInitializerError exception =
+ assertThrows(
+ ExceptionInInitializerError.class, () ->
driver.load().driver("nonexistent").open());
+ assertThat(exception)
+ .hasRootCauseInstanceOf(FileNotFoundException.class)
+ .hasRootCauseMessage(
+ "No JNI library for current platform, missing from JAR: "
+ + "adbc_driver_jni/aarch_64/adbc_driver_jni.dll");
+ return;
+ }
+
// smoke test: try to load a driver that does not exist; should fail with
a proper error
// (not something like a linker error, which would imply the JNI shim
doesn't work)
- var driver = new JniDriver(allocator);
AdbcException exception =
assertThrows(AdbcException.class, () ->
driver.load().driver("nonexistent").open());
assertThat(exception).hasMessageContaining("Could not load `nonexistent`");
