This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git
The following commit(s) were added to refs/heads/main by this push:
new 7dd25a6b infra: optimize python ci (#2931)
7dd25a6b is described below
commit 7dd25a6b74e5800ad28daa23254d4b1f4c6b9866
Author: Kevin Liu <[email protected]>
AuthorDate: Tue Jan 20 10:23:37 2026 -0500
infra: optimize python ci (#2931)
<!--
Thanks for opening a pull request!
-->
<!-- In the case this PR will resolve an issue, please replace
${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
<!-- Closes #${GITHUB_ISSUE_ID} -->
# Rationale for this change
This PR speeds up CI and reduces amount of resources required
The overall time doesnt change much (12m -> 10m), since `make
test-integration` dominates most of the time.
- reduces integration tests to run only on Python 3.12 (we dont need to
run for each python version anymore now that we're using spark connect)
- parallelizes integration tests (run s3/gcs/adls integration tests in
parallel)
- add UV dependency caching
- add fail-fast strategy
## Are these changes tested?
## Are there any user-facing changes?
<!-- In the case of user-facing changes, please add the changelog label.
-->
---
.github/workflows/python-ci.yml | 104 ++++++++++++++++++++++++++++++++++++----
1 file changed, 96 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
index e7299d3f..ab706692 100644
--- a/.github/workflows/python-ci.yml
+++ b/.github/workflows/python-ci.yml
@@ -43,9 +43,10 @@ concurrency:
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
- lint-and-test:
+ lint-and-unit-test:
runs-on: ubuntu-latest
strategy:
+ fail-fast: true
matrix:
python: ['3.10', '3.11', '3.12', '3.13']
@@ -56,6 +57,8 @@ jobs:
python-version: ${{ matrix.python }}
- name: Install UV
uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
- name: Install system dependencies
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for
kerberos
- name: Install
@@ -69,45 +72,130 @@ jobs:
integration-test:
runs-on: ubuntu-latest
- strategy:
- matrix:
- python: ['3.10', '3.11', '3.12', '3.13']
-
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
- python-version: ${{ matrix.python }}
+ python-version: '3.12'
- name: Install UV
uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
- name: Install system dependencies
run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for
kerberos
- name: Install
run: make install
-
- name: Run integration tests with coverage
run: COVERAGE=1 make test-integration
- name: Show debug logs
if: ${{ failure() }}
- run: docker compose -f dev/docker-compose.yml logs
+ run: docker compose -f dev/docker-compose-integration.yml logs
+ - name: Upload coverage data
+ uses: actions/upload-artifact@v4
+ with:
+ name: coverage-integration
+ path: .coverage*
+ include-hidden-files: true
+ integration-test-s3:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.12'
+ - name: Install UV
+ uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+ - name: Install system dependencies
+ run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for
kerberos
+ - name: Install
+ run: make install
- name: Run s3 integration tests with coverage
run: COVERAGE=1 make test-s3
- name: Show debug logs
if: ${{ failure() }}
run: docker compose -f dev/docker-compose.yml logs
+ - name: Upload coverage data
+ uses: actions/upload-artifact@v4
+ with:
+ name: coverage-s3
+ path: .coverage*
+ include-hidden-files: true
+ integration-test-adls:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.12'
+ - name: Install UV
+ uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+ - name: Install system dependencies
+ run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for
kerberos
+ - name: Install
+ run: make install
- name: Run adls integration tests with coverage
run: COVERAGE=1 make test-adls
- name: Show debug logs
if: ${{ failure() }}
run: docker compose -f dev/docker-compose-azurite.yml logs
+ - name: Upload coverage data
+ uses: actions/upload-artifact@v4
+ with:
+ name: coverage-adls
+ path: .coverage*
+ include-hidden-files: true
+ integration-test-gcs:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.12'
+ - name: Install UV
+ uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+ - name: Install system dependencies
+ run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for
kerberos
+ - name: Install
+ run: make install
- name: Run gcs integration tests with coverage
run: COVERAGE=1 make test-gcs
- name: Show debug logs
if: ${{ failure() }}
run: docker compose -f dev/docker-compose-gcs-server.yml logs
+ - name: Upload coverage data
+ uses: actions/upload-artifact@v4
+ with:
+ name: coverage-gcs
+ path: .coverage*
+ include-hidden-files: true
+ integration-coverage-report:
+ runs-on: ubuntu-latest
+ needs: [integration-test, integration-test-s3, integration-test-adls,
integration-test-gcs]
+ steps:
+ - uses: actions/checkout@v6
+ - uses: actions/setup-python@v6
+ with:
+ python-version: '3.12'
+ - name: Install UV
+ uses: astral-sh/setup-uv@v7
+ with:
+ enable-cache: true
+ - name: Install dependencies
+ run: uv sync --group dev
+ - name: Download all coverage artifacts
+ uses: actions/download-artifact@v4
+ with:
+ pattern: coverage-*
+ merge-multiple: true
- name: Generate coverage report (75%) # Coverage threshold should only
increase over time — never decrease it!
run: COVERAGE_FAIL_UNDER=75 make coverage-report
