This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 91ac6badc2b Improve cache usage for prek hooks (#55328)
91ac6badc2b is described below
commit 91ac6badc2b664ca90a7a1b0a23819365f7509d5
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sun Sep 7 08:25:42 2025 +0200
Improve cache usage for prek hooks (#55328)
Previously we were only using prek cache from canary builds, but
since we are starting to use several separate prek runs, it makes
sense to install prek hooks only once and store them in cache and
reuse even in the same build.
This PR removes "only-canary" prek cache preparation - now all
builds including all PRs from fork preapare prek cache once and
upload them as artifacts - then restoring prek cache for every
prek run should be much faster.
---
.github/actions/install-prek/action.yml | 19 ++++++++++-
.github/workflows/basic-tests.yml | 58 ++-------------------------------
.github/workflows/ci-amd.yml | 14 ++++++--
.github/workflows/ci-arm.yml | 13 ++++++--
.github/workflows/ci-image-checks.yml | 47 ++------------------------
5 files changed, 44 insertions(+), 107 deletions(-)
diff --git a/.github/actions/install-prek/action.yml
b/.github/actions/install-prek/action.yml
index e29294d4ede..d4be15daebc 100644
--- a/.github/actions/install-prek/action.yml
+++ b/.github/actions/install-prek/action.yml
@@ -31,6 +31,9 @@ inputs:
skip-prek-hooks:
description: "Skip some prek hooks from installation"
default: ""
+ save-cache:
+ description: "Whether to save prek cache"
+ required: true
platform:
description: 'Platform for the build - linux/amd64 or linux/arm64'
required: true
@@ -61,7 +64,7 @@ runs:
uses:
apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
with:
# yamllint disable rule:line-length
- key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version
}}-${{ hashFiles('.pre-commit-config.yaml') }}
+ key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version
}}-${{inputs.skip-prek-hooks}}-${{ hashFiles('.pre-commit-config.yaml') }}
path: /tmp/
id: restore-prek-cache
- name: "Check if prek cache tarball exists"
@@ -91,3 +94,17 @@ runs:
working-directory: ${{ github.workspace }}
env:
SKIP: ${{ inputs.skip-prek-hooks }}
+ - name: "Prepare .tar file from prek cache"
+ run: |
+ tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek
+ shell: bash
+ if: inputs.save-cache == 'true'
+ - name: "Save prek cache"
+ uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
+ with:
+ # yamllint disable rule:line-length
+ key: cache-prek-v6-${{ inputs.platform }}-${{ inputs.python-version
}}-${{ inputs.skip-prek-hooks }}-${{ hashFiles('.pre-commit-config.yaml') }}
+ path: /tmp/cache-prek.tar.gz
+ if-no-files-found: 'error'
+ retention-days: '2'
+ if: inputs.save-cache == 'true'
diff --git a/.github/workflows/basic-tests.yml
b/.github/workflows/basic-tests.yml
index af053ead94b..0f61212f468 100644
--- a/.github/workflows/basic-tests.yml
+++ b/.github/workflows/basic-tests.yml
@@ -183,58 +183,6 @@ jobs:
retention-days: '2'
if: steps.restore-eslint-cache-simple-am-ui.outputs.stash-hit != 'true'
- install-prek:
- timeout-minutes: 5
- name: "Install prek for cache"
- runs-on: ${{ fromJSON(inputs.runners) }}
- env:
- PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
- SKIP: ${{ inputs.skip-prek-hooks }}
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
- with:
- persist-credentials: false
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- id: breeze
- - name: "Install prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
- skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
- platform: ${{ inputs.platform }}
-
- check-translation-completness:
- timeout-minutes: 15
- name: "Check translation completeness"
- runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
- steps:
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
- with:
- persist-credentials: false
- - name: "Install prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
- skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
- platform: ${{ inputs.platform }}
- - name: "Check translation completeness"
- run: >
- prek --show-diff-on-failure --color always
- --hook-stage manual --verbose --all-files
- check-translations-completeness
- env:
- SKIP: ${{ inputs.skip-prek-hooks }}
- COLUMNS: "202"
-
# Those checks are run if no image needs to be built for checks. This is for
simple changes that
# Do not touch any of the python code or any of the important files that
might require building
# The CI Docker image and they can be run entirely using the prek virtual
environments on host
@@ -242,7 +190,6 @@ jobs:
timeout-minutes: 30
name: "Static checks: basic checks only"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
if: inputs.basic-checks-only == 'true'
steps:
- name: "Cleanup repo"
@@ -264,6 +211,7 @@ jobs:
python-version: ${{ steps.breeze.outputs.host-python-version }}
skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: Fetch incoming commit ${{ github.sha }} with its parent
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
with:
@@ -295,7 +243,6 @@ jobs:
timeout-minutes: 45
name: "Upgrade checks"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
if: inputs.canary-run == 'true'
@@ -316,9 +263,10 @@ jobs:
uses: ./.github/actions/install-prek
id: prek
with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
+ python-version: ${{ steps.breeze.outputs.host-python-version }}
skip-prek-hooks: ${{ inputs.skip-prek-hooks }}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: "Autoupdate all prek hooks"
run: prek autoupdate --freeze
- name: "Autoupdate Lucas-C hooks to bleeding edge"
diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml
index bebcf3a000a..584426f80b5 100644
--- a/.github/workflows/ci-amd.yml
+++ b/.github/workflows/ci-amd.yml
@@ -167,7 +167,14 @@ jobs:
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
-
+ - name: "Install and cache prek"
+ uses: ./.github/actions/install-prek
+ id: prek
+ with:
+ python-version: ${{ steps.breeze.outputs.host-python-version }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
+ platform: "linux/amd64"
+ save-cache: true
run-pin-versions-hook:
name: "Run pin-versions hook"
needs: [build-info]
@@ -184,6 +191,7 @@ jobs:
# octopin needs python 3.11
python-version: "3.11"
platform: "linux/amd64"
+ save-cache: false
- name: "Run pin-versions"
run: >
prek -c .github/.pre-commit-config.yaml --all-files --verbose
--hook-stage manual
@@ -201,8 +209,8 @@ jobs:
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
- basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
- skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}}
+ basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
canary-run: ${{needs.build-info.outputs.canary-run}}
latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
use-uv: ${{needs.build-info.outputs.use-uv}}
diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml
index bd3300ce5a6..493c55afc8c 100644
--- a/.github/workflows/ci-arm.yml
+++ b/.github/workflows/ci-arm.yml
@@ -159,7 +159,14 @@ jobs:
env:
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }}
GITHUB_CONTEXT: ${{ toJson(github) }}
-
+ - name: "Install and cache prek"
+ uses: ./.github/actions/install-prek
+ id: prek
+ with:
+ python-version: ${{ steps.breeze.outputs.host-python-version }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
+ platform: "linux/arm64"
+ save-cache: true
basic-tests:
name: "Basic tests"
needs: [build-info]
@@ -170,8 +177,8 @@ jobs:
run-www-tests: ${{needs.build-info.outputs.run-www-tests}}
run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}}
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
- basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}}
- skip-prek-hooks: ${{needs.build-info.outputs.skip-prek-hooks}}
+ basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }}
+ skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }}
canary-run: ${{needs.build-info.outputs.canary-run}}
latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}}
use-uv: ${{needs.build-info.outputs.use-uv}}
diff --git a/.github/workflows/ci-image-checks.yml
b/.github/workflows/ci-image-checks.yml
index 8bfff2f0007..02bb49ec413 100644
--- a/.github/workflows/ci-image-checks.yml
+++ b/.github/workflows/ci-image-checks.yml
@@ -122,54 +122,10 @@ on: # yamllint disable-line rule:truthy
permissions:
contents: read
jobs:
- install-prek:
- timeout-minutes: 5
- name: "Install prek for cache (only canary runs)"
- runs-on: ${{ fromJSON(inputs.runners) }}
- env:
- PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
- if: inputs.basic-checks-only == 'false'
- steps:
- - name: "Cleanup repo"
- shell: bash
- run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
- if: inputs.canary-run == 'true'
- - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 #
v4.2.2
- with:
- persist-credentials: false
- if: inputs.canary-run == 'true'
- - name: "Install Breeze"
- uses: ./.github/actions/breeze
- id: breeze
- if: inputs.canary-run == 'true'
- - name: "Install prek"
- uses: ./.github/actions/install-prek
- id: prek
- with:
- python-version: ${{steps.breeze.outputs.host-python-version}}
- platform: ${{ inputs.platform }}
- if: inputs.canary-run == 'true'
- - name: "Prepare .tar file from prek cache"
- run: |
- tar -C ~ -czf /tmp/cache-prek.tar.gz .cache/prek
- shell: bash
- if: inputs.canary-run == 'true'
- - name: "Save prek cache"
- uses:
apache/infrastructure-actions/stash/save@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468
- with:
- # yamllint disable rule:line-length
- key: cache-prek-v6-${{ inputs.platform }}-${{
steps.breeze.outputs.host-python-version }}-${{
hashFiles('.pre-commit-config.yaml') }}
- path: /tmp/cache-prek.tar.gz
- if-no-files-found: 'error'
- retention-days: '2'
- if: inputs.canary-run == 'true'
-
static-checks:
timeout-minutes: 45
name: "Static checks"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
env:
PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
UPGRADE_TO_NEWER_DEPENDENCIES: "${{ inputs.upgrade-to-newer-dependencies
}}"
@@ -196,6 +152,7 @@ jobs:
with:
python-version: ${{steps.breeze.outputs.host-python-version}}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: "Static checks"
run: prek --all-files --show-diff-on-failure --color always
env:
@@ -210,7 +167,6 @@ jobs:
timeout-minutes: 45
name: "MyPy checks"
runs-on: ${{ fromJSON(inputs.runners) }}
- needs: install-prek
if: inputs.run-mypy == 'true'
strategy:
fail-fast: false
@@ -240,6 +196,7 @@ jobs:
with:
python-version: ${{steps.breeze.outputs.host-python-version}}
platform: ${{ inputs.platform }}
+ save-cache: false
- name: "MyPy checks for ${{ matrix.mypy-check }}"
run: prek --color always --verbose --hook-stage manual "$MYPY_CHECK"
--all-files
env:
