This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-0-test by this push:
new 982b8ad0888 [v3-0-test] Improve cache usage for prek hooks (#55328)
(#55343)
982b8ad0888 is described below
commit 982b8ad0888919deac316bde0a2866e26a2935d2
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Sep 7 09:55:03 2025 +0200
[v3-0-test] Improve cache usage for prek hooks (#55328) (#55343)
Previously we were only using prek cache from canary builds, but
since we are starting to use several separate prek runs, it makes
sense to install prek hooks only once and store them in cache and
reuse even in the same build.
This PR removes "only-canary" prek cache preparation - now all
builds including all PRs from fork preapare prek cache once and
upload them as artifacts - then restoring prek cache for every
prek run should be much faster.
(cherry picked from commit 91ac6badc2b664ca90a7a1b0a23819365f7509d5)
---
.github/actions/install-prek/action.yml | 19 ++++++++++++-
.github/workflows/basic-tests.yml | 36 +++++--------------------
.github/workflows/ci-amd.yml | 14 +++++++---
.github/workflows/ci-arm.yml | 13 ++++++---
.github/workflows/ci-image-checks.yml | 47 ++-------------------------------
5 files changed, 48 insertions(+), 81 deletions(-)
diff --git a/.github/actions/install-prek/action.yml
b/.github/actions/install-prek/action.yml
index 9f7dab1e682..a71ed48bb60 100644
--- a/.github/actions/install-prek/action.yml
+++ b/.github/actions/install-prek/action.yml
@@ -31,6 +31,9 @@ inputs:
skip-prek-hooks:
description: "Skip some prek hooks from installation"
default: ""
+ save-cache:
+ description: "Whether to save prek cache"
+ required: true
platform:
description: 'Platform for the build - linux/amd64 or linux/arm64'
required: true
@@ -61,7 +64,7 @@ runs:
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
# yamllint disable rule:line-length
- key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version
}}-${{ hashFiles('.pre-commit-config.yaml') }}
+ key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version
}}-${{inputs.skip-prek-hooks}}-${{ hashFiles('.pre-commit-config.yaml') }}
path: /tmp/
id: restore-prek-cache
- name: "Check if prek cache tarball exists"
@@ -91,3 +94,17 @@ runs:
working-directory: ${{ github.workspace }}
env:
SKIP: ${{ inputs.skip-prek-hooks }}
+ - name: "Prepare .tar file from prek cache"
+ run: |
+ tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek
+ shell: bash
+ if: inputs.save-cache == 'true'
+ - name: "Save prek cache"
+ uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
+ with:
+ # yamllint disable rule:line-length
+ key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version
}}-${{ inputs.skip-prek-hooks }}-${{ hashFiles('.pre-commit-config.yaml') }}
+ path: /tmp/cache-prek.tar.gz
+ if-no-files-found: 'error'
+ retention-days: '2'
+ if: inputs.save-cache == 'true'
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index efde755356b..c7a7d174937 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -60,6 +60,10 @@ on: # yamllint disable-line rule:truthy
description: "Whether to use uv in the image"
required: true
type: string
+ uv-version:
+ description: 'uv version to use'
+ default: "0.8.14" # Keep this comment to allow automatic replacement
of uv version
+ type: string
platform:
description: 'Platform for the build - linux/amd64 or linux/arm64'
required: true
@@ -154,32 +158,6 @@ jobs:
retention-days: '2'
if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true'
- install-prek:
- timeout-minutes: 5
- name: "Install prek for cache"
- runs-on: ${{ fromJSON(inputs.runners) }}
- env:
- PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
- SKIP: ${{ inputs.skip-prek-hooks }}
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
- with:
- persist-credentials: false
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- id: breeze
- - name: "Install prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
- skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
- platform: ${{ inputs.platform }}
-
# Those checks are run if no image needs to be built for checks. This is for
simple changes that
# Do not touch any of the python code or any of the important files that
might require building
# The CI Docker image and they can be run entirely using the prek virtual
environments on host
@@ -187,7 +165,6 @@ jobs:
timeout-minutes: 30
name: "Static checks: basic checks only"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
if: inputs.basic-checks-only == 'true'
steps:
- name: "Cleanup repo"
@@ -209,6 +186,7 @@ jobs:
python-version: ${{ steps.breeze.outputs.host-python-version }}
skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
with:
@@ -240,7 +218,6 @@ jobs:
timeout-minutes: 45
name: "Upgrade checks"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
if: inputs.canary-run == 'true'
@@ -261,9 +238,10 @@ jobs:
uses: ./.github/actions/install-prek
id: prek
with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
+ python-version: ${{ steps.breeze.outputs.host-python-version }}
skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: "Autoupdate all prek hooks"
run: prek autoupdate --freeze
- name: "Autoupdate Lucas-C hooks to bleeding edge"
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index 290d1e76016..e7e5a15be8c 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -166,7 +166,14 @@ jobs:
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
-
+ - name: "Install and cache prek"
+ uses: ./.github/actions/install-prek
+ id: prek
+ with:
+ python-version: ${{ steps.breeze.outputs.host-python-version }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
+ platform: "linux/amd64"
+ save-cache: true
run-pin-versions-hook:
name: "Run pin-versions hook"
needs: [build-info]
@@ -183,6 +190,7 @@ jobs:
# octopin needs python 3.11
python-version: "3.11"
platform: "linux/amd64"
+ save-cache: false
- name: "Run pin-versions"
run: >
prek -c .github/.pre-commit-config.yaml --all-files --verbose
--hook-stage manual
@@ -200,8 +208,8 @@ jobs:
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
- basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
- skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}}
+ basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
canary-run: ${{needs.build-info.outputs.canary-run}}
latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
use-uv: ${{needs.build-info.outputs.use-uv}}
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index ebd94d95bb9..4128fdfa2fd 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -158,7 +158,14 @@ jobs:
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
-
+ - name: "Install and cache prek"
+ uses: ./.github/actions/install-prek
+ id: prek
+ with:
+ python-version: ${{ steps.breeze.outputs.host-python-version }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
+ platform: "linux/arm64"
+ save-cache: true
basic-tests:
name: "Basic tests"
needs: [build-info]
@@ -169,8 +176,8 @@ jobs:
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
- basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
- skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}}
+ basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
canary-run: ${{needs.build-info.outputs.canary-run}}
latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
use-uv: ${{needs.build-info.outputs.use-uv}}
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index 32953b67416..68e4c01119d 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -122,54 +122,10 @@ on: # yamllint disable-line rule:truthy
permissions:
contents: read
jobs:
- install-prek:
- timeout-minutes: 5
- name: "Install prek for cache (only canary runs)"
- runs-on: ${{ fromJSON(inputs.runners) }}
- env:
- PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
- if: inputs.basic-checks-only == 'false'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: inputs.canary-run == 'true'
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
- with:
- persist-credentials: false
- if: inputs.canary-run == 'true'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- id: breeze
- if: inputs.canary-run == 'true'
- - name: "Install prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
- platform: ${{ inputs.platform }}
- if: inputs.canary-run == 'true'
- - name: "Prepare .tar file from prek cache"
- run: |
- tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek
- shell: bash
- if: inputs.canary-run == 'true'
- - name: "Save prek cache"
- uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
- with:
- # yamllint disable rule:line-length
- key: cache-prek-v6-${{ inputs.platform }}-${{
steps.breeze.outputs.host-python-version }}-${{
hashFiles('.pre-commit-config.yaml') }}
- path: /tmp/cache-prek.tar.gz
- if-no-files-found: 'error'
- retention-days: '2'
- if: inputs.canary-run == 'true'
-
static-checks:
timeout-minutes: 45
name: "Static checks"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies
}}"
@@ -196,6 +152,7 @@ jobs:
with:
python-version: ${{steps.breeze.outputs.host-python-version}}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: "Static checks"
run: prek --all-files --show-diff-on-failure --color always
env:
@@ -210,7 +167,6 @@ jobs:
timeout-minutes: 45
name: "MyPy checks"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
if: inputs.run-mypy == 'true'
strategy:
fail-fast: false
@@ -240,6 +196,7 @@ jobs:
with:
python-version: ${{steps.breeze.outputs.host-python-version}}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: "MyPy checks for ${{ matrix.mypy-check }}"
run: prek --color always --verbose --hook-stage manual "$MYPY_CHECK"
--all-files
env:
