This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ray.git
The following commit(s) were added to refs/heads/main by this push:
new 6217cbe Add CI to build python wheels (#76)
6217cbe is described below
commit 6217cbe588303e01877f372f9a08325783bb6e11
Author: robtandy <[email protected]>
AuthorDate: Wed Mar 5 12:55:19 2025 -0500
Add CI to build python wheels (#76)
---
.github/workflows/build.yml | 293 +++++++++++++++++++++++++++++++++++++++
Cargo.lock | 19 +++
Cargo.toml | 1 +
LICENSE => LICENSE.txt | 0
LICENSE => dev/create_license.py | 51 +++++++
5 files changed, 364 insertions(+)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 0000000..11ff562
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,293 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Python Release Build
+on:
+ pull_request:
+ branches: ["main"]
+ push:
+ tags: ["*-rc*"]
+ branches: ["branch-*"]
+ workflow_dispatch:
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+ python_version: "3.12"
+
+ # Use the --no-install-package to only install the dependencies
+ # but do not yet build the rust library
+ - name: Install dependencies
+ run: uv sync --dev --no-install-package datafusion-ray
+
+ generate-license:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+
+ - name: Generate license file
+ run: uv run --no-project python ./dev/create_license.py
+ - uses: actions/upload-artifact@v4
+ with:
+ name: python-wheel-license
+ path: LICENSE.txt
+
+ build-python-mac-win:
+ needs: [generate-license]
+ name: Mac/Win
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.13"]
+ #os: [macos-latest, windows-latest]
+ os: [macos-latest]
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - uses: dtolnay/rust-toolchain@stable
+
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v4
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Install Protoc
+ uses: arduino/setup-protoc@v3
+ with:
+ version: "27.4"
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+
+ - name: Build Python package
+ run: |
+ uv sync --dev --no-install-package datafusion-ray
+ uv run --no-project maturin build --release --strip
+
+ - name: List Windows wheels
+ if: matrix.os == 'windows-latest'
+ run: dir target\wheels\
+ # since the runner is dynamic shellcheck (from actionlint) can't infer
this is powershell
+ # so we specify it explicitly
+ shell: powershell
+
+ - name: List Mac wheels
+ if: matrix.os != 'windows-latest'
+ run: find target/wheels/
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v4
+ with:
+ name: dist-${{ matrix.os }}
+ path: target/wheels/*
+
+ build-macos-x86_64:
+ needs: [generate-license]
+ name: Mac x86_64
+ runs-on: macos-13
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: ["3.12"]
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - uses: dtolnay/rust-toolchain@stable
+
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v4
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Install Protoc
+ uses: arduino/setup-protoc@v3
+ with:
+ version: "27.4"
+ repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+ - uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+
+ - name: Build Python package
+ run: |
+ uv sync --dev --no-install-package datafusion-ray
+ uv run --no-project maturin build --release --strip
+
+ - name: List Mac wheels
+ run: find target/wheels/
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v4
+ with:
+ name: dist-macos-aarch64
+ path: target/wheels/*
+
+ build-manylinux-x86_64:
+ needs: [generate-license]
+ name: Manylinux x86_64
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v4
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Build wheels
+ uses: PyO3/maturin-action@v1
+ env:
+ RUST_BACKTRACE: 1
+ with:
+ rust-toolchain: nightly
+ target: x86_64
+ manylinux: auto
+ rustup-components: rust-std rustfmt # Keep them in one line due to
https://github.com/PyO3/maturin-action/issues/153
+ args: --release --manylinux 2014
+ before-script-linux: |
+ yum install -y wget
+ cd /
+ wget
https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
+ unzip -o proto*zip
+ cd -
+ which protoc
+ protoc --version
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v4
+ with:
+ name: dist-manylinux-x86_64
+ path: target/wheels/*
+
+ build-manylinux-aarch64:
+ needs: [generate-license]
+ name: Manylinux arm64
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v4
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Build wheels
+ uses: PyO3/maturin-action@v1
+ env:
+ RUST_BACKTRACE: 1
+ with:
+ rust-toolchain: nightly
+ target: aarch64
+ # Use manylinux_2_28-cross because the manylinux2014-cross has GCC
4.8.5, which causes the build to fail
+ manylinux: 2_28
+ rustup-components: rust-std rustfmt # Keep them in one line due to
https://github.com/PyO3/maturin-action/issues/153
+ args: --release
+ before-script-linux: |
+ apt-get install -y unzip
+ cd /
+ wget
https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
+ unzip -o proto*zip
+ cd -
+ which protoc
+ protoc --version
+
+ - name: Archive wheels
+ uses: actions/upload-artifact@v4
+ with:
+ name: dist-manylinux-aarch64
+ path: target/wheels/*
+
+ build-sdist:
+ needs: [generate-license]
+ name: Source distribution
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - run: rm LICENSE.txt
+ - name: Download LICENSE.txt
+ uses: actions/download-artifact@v4
+ with:
+ name: python-wheel-license
+ path: .
+
+ - name: Build sdist
+ uses: PyO3/maturin-action@v1
+ with:
+ rust-toolchain: stable
+ manylinux: auto
+ rustup-components: rust-std rustfmt
+ args: --release --sdist --out dist
+ before-script-linux: |
+ yum install -y wget
+ cd /
+ wget
https://github.com/protocolbuffers/protobuf/releases/download/v29.3/protoc-29.3-linux-x86_64.zip
+ unzip -o proto*zip
+ cd -
+ which protoc
+ protoc --version
+
+ - name: Assert sdist build does not generate wheels
+ run: |
+ if [ "$(ls -A target/wheels)" ]; then
+ echo "Error: Sdist build generated wheels"
+ exit 1
+ else
+ echo "Directory is clean"
+ fi
+ shell: bash
+
+ merge-build-artifacts:
+ runs-on: ubuntu-latest
+ needs:
+ - build-python-mac-win
+ - build-macos-x86_64
+ - build-manylinux-x86_64
+ - build-manylinux-aarch64
+ - build-sdist
+ steps:
+ - name: Merge Build Artifacts
+ uses: actions/upload-artifact/merge@v4
+ with:
+ name: dist
+ pattern: dist-*
diff --git a/Cargo.lock b/Cargo.lock
index 38244d8..e5c5ffb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -887,6 +887,15 @@ dependencies = [
"phf_codegen",
]
+[[package]]
+name = "cmake"
+version = "0.1.54"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "colorchoice"
version = "1.0.3"
@@ -1616,6 +1625,7 @@ dependencies = [
"parking_lot",
"prost 0.13.5",
"prost-types 0.13.5",
+ "protobuf-src",
"pyo3",
"pyo3-async-runtimes",
"pyo3-pylogger",
@@ -3120,6 +3130,15 @@ dependencies = [
"prost 0.13.5",
]
+[[package]]
+name = "protobuf-src"
+version = "2.1.1+27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6217c3504da19b85a3a4b2e9a5183d635822d83507ba0986624b5c05b83bfc40"
+dependencies = [
+ "cmake",
+]
+
[[package]]
name = "psm"
version = "0.1.25"
diff --git a/Cargo.toml b/Cargo.toml
index 0638fea..fa4dc89 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -85,6 +85,7 @@ tonic-build = { version = "0.8", default-features = false,
features = [
"prost",
] }
url = "2"
+protobuf-src = "2.1"
[dev-dependencies]
tempfile = "3.17"
diff --git a/LICENSE b/LICENSE.txt
similarity index 100%
copy from LICENSE
copy to LICENSE.txt
diff --git a/LICENSE b/dev/create_license.py
similarity index 86%
rename from LICENSE
rename to dev/create_license.py
index 261eeb9..2a67cb8 100644
--- a/LICENSE
+++ b/dev/create_license.py
@@ -1,3 +1,40 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This file is a mirror of
https://github.com/apache/arrow-datafusion/blob/master/dev/create_license.py
+
+import json
+import subprocess
+
+subprocess.check_output(["cargo", "install", "cargo-license"])
+data = subprocess.check_output(
+ [
+ "cargo",
+ "license",
+ "--avoid-build-deps",
+ "--avoid-dev-deps",
+ "--do-not-bundle",
+ "--json",
+ ]
+)
+data = json.loads(data)
+
+result = """
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
@@ -199,3 +236,17 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+"""
+result += "\n------------------\n\n"
+result += "This software is built and contains the following software:\n\n"
+result += "(automatically generated via
[cargo-license](https://crates.io/crates/cargo-license))\n\n"
+for item in data:
+ license = item["license"]
+ name = item["name"]
+ version = item["version"]
+ repository = item["repository"]
+ result += "------------------\n\n"
+ result += f"### {name} {version}\n* source:
[{repository}]({repository})\n* license: {license}\n\n"
+
+with open("LICENSE.txt", "w") as f:
+ f.write(result)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
