This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-adbc.git
The following commit(s) were added to refs/heads/main by this push:
new 2a61e99 ci: add AArch64 wheels via cross-compilation (#217)
2a61e99 is described below
commit 2a61e99c0fa41672a8be629987f3f1b97bb84cf9
Author: David Li <[email protected]>
AuthorDate: Tue Dec 6 08:35:09 2022 -0500
ci: add AArch64 wheels via cross-compilation (#217)
- Use cibuildwheel for adbc_driver_manager
- Consolidate the different Python version builds into a single job
- Split the job by architecture instead
---
.github/workflows/packaging-wheels.yml | 69 +++++++++++------
NOTICE.txt | 3 +
.../docker/python-wheel-manylinux.dockerfile | 27 ++-----
ci/scripts/python_sdist_build.sh | 2 +-
ci/scripts/python_sdist_test.sh | 15 ++--
ci/scripts/python_util.sh | 50 +++++++++++-
ci/scripts/python_wheel_fix_tag.py | 89 +++++++++++-----------
ci/scripts/python_wheel_unix_build.sh | 43 +++++++----
ci/scripts/python_wheel_unix_test.sh | 10 ++-
ci/vcpkg/0001-Work-around-inconsistent-path.patch | 50 ++++++++++++
ci/vcpkg/0002-Retry-downloads.patch | 41 ++++++++++
.../triplets/arm64-linux-static-release.cmake | 26 ++-----
.../vcpkg/triplets/arm64-osx-static-release.cmake | 27 ++-----
docker-compose.yml | 20 ++++-
python/adbc_driver_manager/pyproject.toml | 4 +
python/adbc_driver_postgres/pyproject.toml | 2 +
python/adbc_driver_postgres/setup.py | 9 ++-
python/adbc_driver_sqlite/pyproject.toml | 2 +
python/adbc_driver_sqlite/setup.py | 6 +-
19 files changed, 328 insertions(+), 167 deletions(-)
diff --git a/.github/workflows/packaging-wheels.yml
b/.github/workflows/packaging-wheels.yml
index 74ff66a..47ee541 100644
--- a/.github/workflows/packaging-wheels.yml
+++ b/.github/workflows/packaging-wheels.yml
@@ -123,14 +123,13 @@ jobs:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
python-manylinux:
- name: "Python ${{ matrix.python_version }} manylinux${{
matrix.manylinux_version }}"
+ name: "Python ${{ matrix.arch }} manylinux${{ matrix.manylinux_version }}"
runs-on: ubuntu-latest
strategy:
matrix:
+ arch: ["amd64", "arm64v8"]
manylinux_version: ["2014"]
- python_version: ["3.9", "3.10", "3.11"]
- # Limit parallelism; gemfury appears to return 409 CONFLICT on
concurrent uploads
- max-parallel: 1
+ # Let this one run in parallel because the arm64 build is so much slower
steps:
- uses: actions/checkout@v3
with:
@@ -149,17 +148,21 @@ jobs:
echo "schedule: ${{ github.event.schedule }}"
echo "ref: ${{ github.ref }}"
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v2
+
- name: Build wheel
shell: bash
+ env:
+ ARCH: ${{ matrix.arch }}
+ MANYLINUX: ${{ matrix.manylinux_version }}
run: |
- export MANYLINUX=${{ matrix.manylinux_version }}
- export PYTHON=${{ matrix.python_version }}
docker-compose run python-wheel-manylinux
- name: Archive wheels
uses: actions/upload-artifact@v3
with:
- name: python${{ matrix.python_version }}-manylinux${{
matrix.manylinux_version }}
+ name: python-${{ matrix.arch }}-manylinux${{
matrix.manylinux_version }}
retention-days: 7
path: |
python/adbc_driver_manager/repaired_wheels/*.whl
@@ -168,10 +171,13 @@ jobs:
- name: Test wheel
shell: bash
+ env:
+ ARCH: ${{ matrix.arch }}
+ MANYLINUX: ${{ matrix.manylinux_version }}
run: |
- export MANYLINUX=${{ matrix.manylinux_version }}
- export PYTHON=${{ matrix.python_version }}
- docker-compose run python-wheel-manylinux-test
+ env PYTHON=3.9 docker-compose run python-wheel-manylinux-test
+ env PYTHON=3.10 docker-compose run python-wheel-manylinux-test
+ env PYTHON=3.11 docker-compose run python-wheel-manylinux-test
- name: Upload wheels to Gemfury
shell: bash
@@ -182,19 +188,16 @@ jobs:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
python-macos:
- name: "Python ${{ matrix.python_version }} macOS"
+ name: "Python ${{ matrix.arch }} macOS"
runs-on: macos-latest
strategy:
matrix:
- python_version: ["3.9", "3.10", "3.11"]
+ arch: ["amd64", "arm64v8"]
# Limit parallelism; gemfury appears to return 409 CONFLICT on
concurrent uploads
max-parallel: 1
env:
MACOSX_DEPLOYMENT_TARGET: "10.15"
- PYTHON: "/Library/Frameworks/Python.framework/Versions/${{
matrix.python_version }}/bin/python${{ matrix.python_version }}"
- PYTHON_VERSION: "${{ matrix.python_version }}"
- # Use a custom triplet to only build release packages
- VCPKG_DEFAULT_TRIPLET: "x64-osx-static-release"
+ PYTHON:
"/Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10"
# Where to install vcpkg
VCPKG_ROOT: "${{ github.workspace }}/vcpkg"
steps:
@@ -229,21 +232,26 @@ jobs:
shell: bash
run: ci/scripts/install_vcpkg.sh $VCPKG_ROOT $VCPKG_VERSION
- - name: Install Python ${{ matrix.python_version }}
+ - name: Install Python
shell: bash
- run: sudo ci/scripts/install_python.sh macos ${{ matrix.python_version
}}
+ run: |
+ sudo ci/scripts/install_python.sh macos 3.9
+ sudo ci/scripts/install_python.sh macos 3.10
+ sudo ci/scripts/install_python.sh macos 3.11
- name: Build wheel
shell: bash
+ env:
+ ARCH: ${{ matrix.arch }}
run: |
$PYTHON -m venv build-env
source build-env/bin/activate
- ./ci/scripts/python_wheel_unix_build.sh x86_64 $(pwd) $(pwd)/build
+ ./ci/scripts/python_wheel_unix_build.sh $ARCH $(pwd) $(pwd)/build
- name: Archive wheels
uses: actions/upload-artifact@v3
with:
- name: python${{ matrix.python_version }}-macOS
+ name: python-${{ matrix.arch }}-macos
retention-days: 7
path: |
python/adbc_driver_manager/repaired_wheels/*.whl
@@ -251,17 +259,32 @@ jobs:
python/adbc_driver_sqlite/repaired_wheels/*.whl
- name: Test wheel
+ if: matrix.arch == 'amd64'
shell: bash
run: |
- $PYTHON -m venv test-env
- source test-env/bin/activate
+ /Library/Frameworks/Python.framework/Versions/3.9/bin/python3.9 -m
venv test-env-39
+ source test-env-39/bin/activate
+ export PYTHON_VERSION=3.9
+ ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+ deactivate
+
+ /Library/Frameworks/Python.framework/Versions/3.10/bin/python3.10 -m
venv test-env-310
+ source test-env-310/bin/activate
+ export PYTHON_VERSION=3.10
./ci/scripts/python_wheel_unix_test.sh $(pwd)
+ deactivate
+
+ /Library/Frameworks/Python.framework/Versions/3.11/bin/python3.11 -m
venv test-env-311
+ source test-env-311/bin/activate
+ export PYTHON_VERSION=3.11
+ ./ci/scripts/python_wheel_unix_test.sh $(pwd)
+ deactivate
- name: Upload wheels to Gemfury
shell: bash
if: github.ref == 'refs/heads/main' && (github.event.schedule ||
github.event.inputs.upload_artifacts == true)
run: |
- ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres,sqlite}/dist/*.whl
+ ./ci/scripts/python_wheel_upload.sh
python/adbc_driver_{manager,postgres,sqlite}/repaired_wheels/*.whl
env:
GEMFURY_PUSH_TOKEN: ${{ secrets.GEMFURY_PUSH_TOKEN }}
diff --git a/NOTICE.txt b/NOTICE.txt
index a5ef1a2..d84a529 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -6,3 +6,6 @@ The Apache Software Foundation (http://www.apache.org/).
This product includes software from the miniver project (CC0).
https://github.com/jbweston/miniver
+
+This product includes software from the vcpkg project (MIT).
+Copyright (c) Microsoft Corporation
diff --git a/python/adbc_driver_sqlite/pyproject.toml
b/ci/docker/python-wheel-manylinux.dockerfile
similarity index 56%
copy from python/adbc_driver_sqlite/pyproject.toml
copy to ci/docker/python-wheel-manylinux.dockerfile
index c556045..6147a74 100644
--- a/python/adbc_driver_sqlite/pyproject.toml
+++ b/ci/docker/python-wheel-manylinux.dockerfile
@@ -15,25 +15,12 @@
# specific language governing permissions and limitations
# under the License.
-[project]
-name = "adbc_driver_sqlite"
-description = "An ADBC driver for working with SQLite."
-authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
-license = {text = "Apache-2.0"}
-requires-python = ">=3.9"
-dynamic = ["version"]
+ARG ARCH
+ARG MANYLINUX
+ARG PYTHON
+ARG REPO
+ARG VCPKG
-[project.optional-dependencies]
-dbapi = ["pandas", "pyarrow>=8.0.0"]
-test = ["pandas", "pyarrow>=8.0.0", "pytest"]
+FROM
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
-[project.urls]
-homepage = "https://arrow.apache.org";
-repository = "https://github.com/apache/arrow-adbc";
-
-[build-system]
-requires = ["setuptools >= 61.0.0"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools]
-include-package-data = true
+RUN yum install -y docker
diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh
index 535331c..eca67fd 100755
--- a/ci/scripts/python_sdist_build.sh
+++ b/ci/scripts/python_sdist_build.sh
@@ -36,7 +36,7 @@ export _ADBC_IS_SDIST=1
for component in ${COMPONENTS}; do
pushd ${source_dir}/python/$component
- echo "=== (${PYTHON_VERSION}) Building $component sdist ==="
+ echo "=== Building $component sdist ==="
# python -m build copies to a tempdir, so we can't reference other files
in the repo
# https://github.com/pypa/pip/issues/5519
python setup.py sdist
diff --git a/ci/scripts/python_sdist_test.sh b/ci/scripts/python_sdist_test.sh
index e339533..3bb9029 100755
--- a/ci/scripts/python_sdist_test.sh
+++ b/ci/scripts/python_sdist_test.sh
@@ -21,22 +21,25 @@ set -e
set -x
set -o pipefail
-if [ "$#" -ne 1 ]; then
- echo "Usage: $0 <adbc-src-dir>"
+if [ "$#" -ne 2 ]; then
+ echo "Usage: $0 <arch> <adbc-src-dir>"
exit 1
fi
-source_dir=${1}
+arch=${1}
+source_dir=${2}
build_dir="${source_dir}/build"
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${script_dir}/python_util.sh"
-echo "=== (${PYTHON_VERSION}) Building ADBC libpq driver ==="
-# Sets ADBC_POSTGRES_LIBRARY
+echo "=== Set up platform variables ==="
+setup_build_vars "${arch}"
+
+echo "=== Building C/C++ driver components ==="
build_drivers "${source_dir}" "${build_dir}"
-echo "=== (${PYTHON_VERSION}) Installing sdists ==="
+echo "=== Installing sdists ==="
for component in ${COMPONENTS}; do
pip install --force-reinstall
${source_dir}/python/${component}/dist/*.tar.gz
done
diff --git a/ci/scripts/python_util.sh b/ci/scripts/python_util.sh
index c8512a5..65cffd2 100644
--- a/ci/scripts/python_util.sh
+++ b/ci/scripts/python_util.sh
@@ -21,7 +21,7 @@ COMPONENTS="adbc_driver_manager adbc_driver_postgres
adbc_driver_sqlite"
function build_drivers {
local -r source_dir="$1"
- local -r build_dir="$2"
+ local -r build_dir="$2/${VCPKG_ARCH}"
: ${CMAKE_BUILD_TYPE:=release}
: ${CMAKE_UNITY_BUILD:=ON}
@@ -35,13 +35,21 @@ function build_drivers {
if [[ $(uname) == "Linux" ]]; then
export
ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.so
export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.so
- export VCPKG_DEFAULT_TRIPLET="x64-linux-static-release"
+ export VCPKG_DEFAULT_TRIPLET="${VCPKG_ARCH}-linux-static-release"
else # macOS
export
ADBC_POSTGRES_LIBRARY=${build_dir}/lib/libadbc_driver_postgres.dylib
export ADBC_SQLITE_LIBRARY=${build_dir}/lib/libadbc_driver_sqlite.dylib
- export VCPKG_DEFAULT_TRIPLET="x64-osx-static-release"
+ export VCPKG_DEFAULT_TRIPLET="${VCPKG_ARCH}-osx-static-release"
fi
+ pushd "${VCPKG_ROOT}"
+ # XXX: patch an odd issue where the path of some file is inconsistent
between builds
+ patch -N -p1 <
"${source_dir}/ci/vcpkg/0001-Work-around-inconsistent-path.patch" || true
+
+ # XXX: make vcpkg retry downloads
https://github.com/microsoft/vcpkg/discussions/20583
+ patch -N -p1 < "${source_dir}/ci/vcpkg/0002-Retry-downloads.patch" || true
+ popd
+
"${VCPKG_ROOT}/vcpkg" install libpq sqlite3 \
--overlay-triplets "${VCPKG_OVERLAY_TRIPLETS}" \
--triplet "${VCPKG_DEFAULT_TRIPLET}"
@@ -81,6 +89,42 @@ function build_drivers {
popd
}
+function setup_build_vars {
+ local -r arch="${1}"
+ if [[ "$(uname)" = "Darwin" ]]; then
+ if [[ "${arch}" = "amd64" ]]; then
+ export CIBW_ARCHS="x86_64"
+ export PYTHON_ARCH="x86_64"
+ export VCPKG_ARCH="x64"
+ elif [[ "${arch}" = "arm64v8" ]]; then
+ export CIBW_ARCHS="arm64"
+ export PYTHON_ARCH="arm64"
+ export VCPKG_ARCH="arm64"
+ else
+ echo "Unknown architecture: ${arch}"
+ exit 1
+ fi
+ export CIBW_BUILD='*-macosx_*'
+ export CIBW_PLATFORM="macos"
+ else
+ if [[ "${arch}" = "amd64" ]]; then
+ export CIBW_ARCHS="x86_64"
+ export PYTHON_ARCH="x86_64"
+ export VCPKG_ARCH="x64"
+ elif [[ "${arch}" = "arm64v8" ]]; then
+ export CIBW_ARCHS="aarch64"
+ export PYTHON_ARCH="arm64"
+ export VCPKG_ARCH="arm64"
+ else
+ echo "Unknown architecture: ${arch}"
+ exit 1
+ fi
+ export CIBW_BUILD='*-manylinux_*'
+ export CIBW_PLATFORM="linux"
+ fi
+ export CIBW_SKIP="pp* ${CIBW_SKIP}"
+}
+
function test_packages {
for component in ${COMPONENTS}; do
echo "=== Testing $component ==="
diff --git a/ci/scripts/python_wheel_fix_tag.py
b/ci/scripts/python_wheel_fix_tag.py
index 890c791..b82ef24 100755
--- a/ci/scripts/python_wheel_fix_tag.py
+++ b/ci/scripts/python_wheel_fix_tag.py
@@ -30,7 +30,7 @@ from pathlib import Path
def main():
parser = argparse.ArgumentParser()
- parser.add_argument("wheel")
+ parser.add_argument("wheel", nargs="+")
args = parser.parse_args()
plat_tag = sysconfig.get_platform()
@@ -38,56 +38,57 @@ def main():
plat_tag = plat_tag.replace(".", "_")
print("Using platform tag: ", plat_tag)
- with tempfile.TemporaryDirectory() as dest:
- dest = Path(dest)
- subprocess.check_call(
- [
- sys.executable,
- "-m",
- "wheel",
- "unpack",
- "--dest",
- str(dest),
- args.wheel,
- ]
- )
+ for wheel in args.wheel:
+ with tempfile.TemporaryDirectory() as dest:
+ dest = Path(dest)
+ wheel = Path(wheel).resolve()
+ subprocess.check_call(
+ [
+ sys.executable,
+ "-m",
+ "wheel",
+ "unpack",
+ "--dest",
+ str(dest),
+ str(wheel),
+ ]
+ )
- (root,) = list(dest.glob("*"))
+ (root,) = list(dest.glob("*"))
- metadata_files = list(root.rglob("*.dist-info/WHEEL"))
- if len(metadata_files) != 1:
- raise ValueError(f"Expected one WHEEL file, found:
{metadata_files}")
+ metadata_files = list(root.rglob("*.dist-info/WHEEL"))
+ if len(metadata_files) != 1:
+ raise ValueError(f"Expected one WHEEL file, found:
{metadata_files}")
- with metadata_files[0].open() as source:
- contents = source.read()
+ with metadata_files[0].open() as source:
+ contents = source.read()
- new_contents = contents.replace(
- "Root-Is-Purelib: true", "Root-Is-Purelib: false"
- )
- new_contents = new_contents.replace(
- "Tag: py3-none-any", f"Tag: py3-none-{plat_tag}"
- )
+ new_contents = contents.replace(
+ "Root-Is-Purelib: true", "Root-Is-Purelib: false"
+ )
+ new_contents = new_contents.replace(
+ "Tag: py3-none-any", f"Tag: py3-none-{plat_tag}"
+ )
- if new_contents == contents:
- print("No changes made (wheel already properly tagged?)")
- return
+ if new_contents == contents:
+ print("No changes made (wheel already properly tagged?)")
+ continue
- with metadata_files[0].open("w") as sink:
- sink.write(new_contents)
+ with metadata_files[0].open("w") as sink:
+ sink.write(new_contents)
- wheel = Path(args.wheel).resolve()
- subprocess.check_call(
- [
- sys.executable,
- "-m",
- "wheel",
- "pack",
- "--dest-dir",
- str(wheel.parent),
- str(root),
- ]
- )
- wheel.unlink()
+ subprocess.check_call(
+ [
+ sys.executable,
+ "-m",
+ "wheel",
+ "pack",
+ "--dest-dir",
+ str(wheel.parent),
+ str(root),
+ ]
+ )
+ wheel.unlink()
if __name__ == "__main__":
diff --git a/ci/scripts/python_wheel_unix_build.sh
b/ci/scripts/python_wheel_unix_build.sh
index 0ca0e59..b5d27a2 100755
--- a/ci/scripts/python_wheel_unix_build.sh
+++ b/ci/scripts/python_wheel_unix_build.sh
@@ -50,15 +50,18 @@ function check_visibility {
function check_wheels {
if [[ $(uname) == "Linux" ]]; then
- echo "=== (${PYTHON_VERSION}) Tag $component wheel with
manylinux${MANYLINUX_VERSION} ==="
+ echo "=== Tag $component wheel with manylinux${MANYLINUX_VERSION} ==="
auditwheel repair "$@" -L . -w repaired_wheels
else # macOS
- echo "=== (${PYTHON_VERSION}) Tag $component wheel with macOS ==="
+ echo "=== Tag $component wheel with macOS ==="
delocate-wheel -v -k -w repaired_wheels "$@"
fi
}
-echo "=== (${PYTHON_VERSION}) Building C/C++ driver components ==="
+echo "=== Set up platform variables ==="
+setup_build_vars "${arch}"
+
+echo "=== Building C/C++ driver components ==="
# Sets ADBC_POSTGRES_LIBRARY, ADBC_SQLITE_LIBRARY
build_drivers "${source_dir}" "${build_dir}"
@@ -68,30 +71,38 @@ check_visibility $ADBC_SQLITE_LIBRARY
# https://github.com/pypa/pip/issues/7555
# Get the latest pip so we have in-tree-build by default
-# Get wheel so we can manually invoke bdist_wheel
-python -m pip install --upgrade pip auditwheel delocate setuptools wheel
+python -m pip install --upgrade pip auditwheel cibuildwheel delocate
setuptools wheel
-PLAT_NAME=$(python -c "import sysconfig; print(sysconfig.get_platform())")
+# XXX: when we manually retag the wheel, we have to use the right arch
+# tag, hence the replacements
+PLAT_NAME=$(python -c "import sysconfig;
print(sysconfig.get_platform().replace('-x86_64',
'${PYTHON_ARCH}').replace('-arm64', '${PYTHON_ARCH}'))")
for component in $COMPONENTS; do
pushd ${source_dir}/python/$component
- echo "=== (${PYTHON_VERSION}) Clean build artifacts ==="
+ echo "=== Clean build artifacts ==="
rm -rf ./build ./dist ./repaired_wheels ./$component/*.so
./$component/*.so.*
- echo "=== (${PYTHON_VERSION}) Check $component version ==="
+ echo "=== Check $component version ==="
python $component/_version.py
- git describe --long --always --first-parent
- git describe --long --always
- echo "=== (${PYTHON_VERSION}) Building $component wheel ==="
- # python -m build copies to a tempdir, so we can't reference other files
in the repo
- python -m pip wheel -w dist -vvv .
+ echo "=== Building $component wheel ==="
+ # First, create an sdist, which 1) bundles the C++ sources and 2)
+ # embeds the git tag. cibuildwheel may copy into a Docker
+ # container during build, but it only copies the package
+ # directory, which omits the C++ sources and .git directory,
+ # causing the build to fail.
+ python setup.py sdist
+ if [[ "$component" = "adbc_driver_manager" ]]; then
+ python -m cibuildwheel --output-dir repaired_wheels/
dist/$component-*.tar.gz
+ else
+ python -m pip wheel -w dist -vvv .
- # Retag the wheel
- python "${script_dir}/python_wheel_fix_tag.py" dist/$component-*.whl
+ # Retag the wheel
+ python "${script_dir}/python_wheel_fix_tag.py" dist/$component-*.whl
- check_wheels dist/$component-*.whl
+ check_wheels dist/$component-*.whl
+ fi
popd
done
diff --git a/ci/scripts/python_wheel_unix_test.sh
b/ci/scripts/python_wheel_unix_test.sh
index 2f9b6d8..c55e7c7 100755
--- a/ci/scripts/python_wheel_unix_test.sh
+++ b/ci/scripts/python_wheel_unix_test.sh
@@ -33,12 +33,18 @@ source "${script_dir}/python_util.sh"
echo "=== (${PYTHON_VERSION}) Installing wheels ==="
for component in ${COMPONENTS}; do
+ if [[ "${component}" = "adbc_driver_manager" ]]; then
+ PYTHON_TAG=cp$(python -c "import sysconfig;
print(sysconfig.get_python_version().replace('.', ''))")
+ else
+ PYTHON_TAG=py3
+ fi
+
if [[ -d ${source_dir}/python/${component}/repaired_wheels/ ]]; then
pip install --force-reinstall \
- ${source_dir}/python/${component}/repaired_wheels/*.whl
+
${source_dir}/python/${component}/repaired_wheels/*-${PYTHON_TAG}-*.whl
else
pip install --force-reinstall \
- ${source_dir}/python/${component}/dist/*.whl
+ ${source_dir}/python/${component}/dist/*-${PYTHON_TAG}-*.whl
fi
done
pip install pytest pyarrow pandas
diff --git a/ci/vcpkg/0001-Work-around-inconsistent-path.patch
b/ci/vcpkg/0001-Work-around-inconsistent-path.patch
new file mode 100644
index 0000000..139e935
--- /dev/null
+++ b/ci/vcpkg/0001-Work-around-inconsistent-path.patch
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+From a018503281e2738261a76175d722142e62778f72 Mon Sep 17 00:00:00 2001
+From: David Li <[email protected]>
+Date: Mon, 5 Dec 2022 14:00:05 -0500
+Subject: [PATCH] Work around inconsistent path
+
+---
+ ports/libpq/portfile.cmake | 11 +++++++++--
+ 1 file changed, 9 insertions(+), 2 deletions(-)
+
+diff --git a/ports/libpq/portfile.cmake b/ports/libpq/portfile.cmake
+index 378d41109..19ce64dee 100644
+--- a/ports/libpq/portfile.cmake
++++ b/ports/libpq/portfile.cmake
+@@ -343,8 +343,15 @@ else()
+ set(USE_DL ON)
+ endif()
+
+-
vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/postgresql/server/pg_config.h"
"#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS")
+- vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/pg_config.h"
"#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS")
++ if (EXISTS
"${CURRENT_PACKAGES_DIR}/include/postgresql/server/pg_config.h")
++
vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/postgresql/server/pg_config.h"
"#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS")
++ endif()
++ if (EXISTS "${CURRENT_PACKAGES_DIR}/include/server/pg_config.h")
++
vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/server/pg_config.h"
"#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS")
++ endif()
++ if (EXISTS "${CURRENT_PACKAGES_DIR}/include/pg_config.h")
++ vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/include/pg_config.h"
"#define CONFIGURE_ARGS" "// #define CONFIGURE_ARGS")
++ endif()
+ endif()
+
+ configure_file("${CMAKE_CURRENT_LIST_DIR}/vcpkg-cmake-wrapper.cmake"
"${CURRENT_PACKAGES_DIR}/share/postgresql/vcpkg-cmake-wrapper.cmake" @ONLY)
+--
+2.17.1
diff --git a/ci/vcpkg/0002-Retry-downloads.patch
b/ci/vcpkg/0002-Retry-downloads.patch
new file mode 100644
index 0000000..673b2cc
--- /dev/null
+++ b/ci/vcpkg/0002-Retry-downloads.patch
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+From 6bac044938a6c9fd4832ec636a98cb684ea6f609 Mon Sep 17 00:00:00 2001
+From: David Li <[email protected]>
+Date: Mon, 5 Dec 2022 14:26:42 -0500
+Subject: [PATCH] Retry downloads
+
+---
+ scripts/cmake/vcpkg_download_distfile.cmake | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/scripts/cmake/vcpkg_download_distfile.cmake
b/scripts/cmake/vcpkg_download_distfile.cmake
+index 08ca55deb..a1ba00347 100644
+--- a/scripts/cmake/vcpkg_download_distfile.cmake
++++ b/scripts/cmake/vcpkg_download_distfile.cmake
+@@ -73,6 +73,8 @@ function(z_vcpkg_download_distfile_via_aria)
+ debug_message("Download Command: ${ARIA2} ${URL} -o temp/${filename}
-l download-${filename}-detailed.log ${headers_param}")
+ vcpkg_execute_in_download_mode(
+ COMMAND ${ARIA2} ${URL}
++ --max-tries=5
++ --retry-wait=20
+ -o temp/${arg_FILENAME}
+ -l download-${arg_FILENAME}-detailed.log
+ ${headers_param}
+--
+2.17.1
diff --git a/python/adbc_driver_sqlite/pyproject.toml
b/ci/vcpkg/triplets/arm64-linux-static-release.cmake
similarity index 56%
copy from python/adbc_driver_sqlite/pyproject.toml
copy to ci/vcpkg/triplets/arm64-linux-static-release.cmake
index c556045..9bf9564 100644
--- a/python/adbc_driver_sqlite/pyproject.toml
+++ b/ci/vcpkg/triplets/arm64-linux-static-release.cmake
@@ -15,25 +15,9 @@
# specific language governing permissions and limitations
# under the License.
-[project]
-name = "adbc_driver_sqlite"
-description = "An ADBC driver for working with SQLite."
-authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
-license = {text = "Apache-2.0"}
-requires-python = ">=3.9"
-dynamic = ["version"]
+set(VCPKG_TARGET_ARCHITECTURE arm64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE static)
+set(VCPKG_BUILD_TYPE release)
-[project.optional-dependencies]
-dbapi = ["pandas", "pyarrow>=8.0.0"]
-test = ["pandas", "pyarrow>=8.0.0", "pytest"]
-
-[project.urls]
-homepage = "https://arrow.apache.org";
-repository = "https://github.com/apache/arrow-adbc";
-
-[build-system]
-requires = ["setuptools >= 61.0.0"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools]
-include-package-data = true
+set(VCPKG_CMAKE_SYSTEM_NAME Linux)
diff --git a/python/adbc_driver_sqlite/pyproject.toml
b/ci/vcpkg/triplets/arm64-osx-static-release.cmake
similarity index 56%
copy from python/adbc_driver_sqlite/pyproject.toml
copy to ci/vcpkg/triplets/arm64-osx-static-release.cmake
index c556045..dba1611 100644
--- a/python/adbc_driver_sqlite/pyproject.toml
+++ b/ci/vcpkg/triplets/arm64-osx-static-release.cmake
@@ -15,25 +15,10 @@
# specific language governing permissions and limitations
# under the License.
-[project]
-name = "adbc_driver_sqlite"
-description = "An ADBC driver for working with SQLite."
-authors = [{name = "Apache Arrow Developers", email = "[email protected]"}]
-license = {text = "Apache-2.0"}
-requires-python = ">=3.9"
-dynamic = ["version"]
+set(VCPKG_TARGET_ARCHITECTURE arm64)
+set(VCPKG_CRT_LINKAGE dynamic)
+set(VCPKG_LIBRARY_LINKAGE static)
+set(VCPKG_BUILD_TYPE release)
-[project.optional-dependencies]
-dbapi = ["pandas", "pyarrow>=8.0.0"]
-test = ["pandas", "pyarrow>=8.0.0", "pytest"]
-
-[project.urls]
-homepage = "https://arrow.apache.org";
-repository = "https://github.com/apache/arrow-adbc";
-
-[build-system]
-requires = ["setuptools >= 61.0.0"]
-build-backend = "setuptools.build_meta"
-
-[tool.setuptools]
-include-package-data = true
+set(VCPKG_CMAKE_SYSTEM_NAME Darwin)
+set(VCPKG_OSX_ARCHITECTURES arm64)
diff --git a/docker-compose.yml b/docker-compose.yml
index 3f9d0b7..b789300 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -50,19 +50,31 @@ services:
image:
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
volumes:
- .:/adbc:delegated
- command: "'/adbc/ci/scripts/python_sdist_test.sh /adbc'"
+ command: "'/adbc/ci/scripts/python_sdist_test.sh ${ARCH} /adbc'"
############################ Python wheels ##################################
python-wheel-manylinux:
- image:
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}
+ image:
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}-adbc
+ build:
+ context: .
+ cache_from:
+ -
${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-${MANYLINUX}-vcpkg-${VCPKG}-adbc
+ dockerfile: ci/docker/python-wheel-manylinux.dockerfile
+ args:
+ ARCH: ${ARCH}
+ MANYLINUX: ${MANYLINUX}
+ PYTHON: ${PYTHON}
+ REPO: ${REPO}
+ VCPKG: ${VCPKG}
volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
- .:/adbc:delegated
# Must set safe.directory so miniver won't error when calling git
- command: "'git config --global --add safe.directory /adbc && git config
--global --get safe.directory && /adbc/ci/scripts/python_wheel_unix_build.sh
x86_64 /adbc /adbc/build'"
+ command: "'git config --global --add safe.directory /adbc && git config
--global --get safe.directory && /adbc/ci/scripts/python_wheel_unix_build.sh
${ARCH} /adbc /adbc/build'"
python-wheel-manylinux-test:
- image: ${ARCH}/python:${PYTHON}
+ image: ${ARCH}/python:${PYTHON}-slim
volumes:
- .:/adbc:delegated
command: /adbc/ci/scripts/python_wheel_unix_test.sh /adbc
diff --git a/python/adbc_driver_manager/pyproject.toml
b/python/adbc_driver_manager/pyproject.toml
index ee12b0d..3587210 100644
--- a/python/adbc_driver_manager/pyproject.toml
+++ b/python/adbc_driver_manager/pyproject.toml
@@ -39,3 +39,7 @@ build-backend = "setuptools.build_meta"
markers = [
"sqlite: tests that require the SQLite driver",
]
+
+[tool.setuptools]
+packages = ["adbc_driver_manager"]
+py-modules = ["adbc_driver_manager"]
diff --git a/python/adbc_driver_postgres/pyproject.toml
b/python/adbc_driver_postgres/pyproject.toml
index 35a4331..52657ab 100644
--- a/python/adbc_driver_postgres/pyproject.toml
+++ b/python/adbc_driver_postgres/pyproject.toml
@@ -37,3 +37,5 @@ build-backend = "setuptools.build_meta"
[tool.setuptools]
include-package-data = true
+packages = ["adbc_driver_postgres"]
+py-modules = ["adbc_driver_postgres"]
diff --git a/python/adbc_driver_postgres/setup.py
b/python/adbc_driver_postgres/setup.py
index f4882eb..6699ed9 100644
--- a/python/adbc_driver_postgres/setup.py
+++ b/python/adbc_driver_postgres/setup.py
@@ -30,15 +30,18 @@ repo_root = source_root.joinpath("../../")
# Resolve Shared Library
library = os.environ.get("ADBC_POSTGRES_LIBRARY")
+target = source_root.joinpath(
+ "./adbc_driver_postgres/libadbc_driver_postgres.so"
+).resolve()
+
if not library:
if os.environ.get("_ADBC_IS_SDIST", "").strip().lower() in ("1", "true"):
print("Building sdist, not requiring ADBC_POSTGRES_LIBRARY")
+ elif target.is_file():
+ print("Driver already exists (but may be stale?), continuing")
else:
raise ValueError("Must provide ADBC_POSTGRES_LIBRARY")
else:
- target = source_root.joinpath(
- "./adbc_driver_postgres/libadbc_driver_postgres.so"
- ).resolve()
shutil.copy(library, target)
# ------------------------------------------------------------
diff --git a/python/adbc_driver_sqlite/pyproject.toml
b/python/adbc_driver_sqlite/pyproject.toml
index c556045..cdce9b3 100644
--- a/python/adbc_driver_sqlite/pyproject.toml
+++ b/python/adbc_driver_sqlite/pyproject.toml
@@ -37,3 +37,5 @@ build-backend = "setuptools.build_meta"
[tool.setuptools]
include-package-data = true
+packages = ["adbc_driver_sqlite"]
+py-modules = ["adbc_driver_sqlite"]
diff --git a/python/adbc_driver_sqlite/setup.py
b/python/adbc_driver_sqlite/setup.py
index 4f72ceb..eaa47a7 100644
--- a/python/adbc_driver_sqlite/setup.py
+++ b/python/adbc_driver_sqlite/setup.py
@@ -30,15 +30,15 @@ repo_root = source_root.joinpath("../../")
# Resolve Shared Library
library = os.environ.get("ADBC_SQLITE_LIBRARY")
+target =
source_root.joinpath("./adbc_driver_sqlite/libadbc_driver_sqlite.so").resolve()
if not library:
if os.environ.get("_ADBC_IS_SDIST", "").strip().lower() in ("1", "true"):
print("Building sdist, not requiring ADBC_SQLITE_LIBRARY")
+ elif target.is_file():
+ print("Driver already exists (but may be stale?), continuing")
else:
raise ValueError("Must provide ADBC_SQLITE_LIBRARY")
else:
- target = source_root.joinpath(
- "./adbc_driver_sqlite/libadbc_driver_sqlite.so"
- ).resolve()
shutil.copy(library, target)
# ------------------------------------------------------------
