This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new b14dbf2c8c Automatically upgrade various components in canary builds
(#37113)
b14dbf2c8c is described below
commit b14dbf2c8c83f32e3347522c8559d99889f8e0ed
Author: Jarek Potiuk <[email protected]>
AuthorDate: Wed Jan 31 10:50:27 2024 +0100
Automatically upgrade various components in canary builds (#37113)
This change attempts to automatically upgrade our dependencies in
the canary builds. This means that canary builds will fail
occasionally when there is a need to update some dependencies that
otherwise are not updated automatically. This gives an opportunity
to fix such failing main build as soon as it happns.
It does not impact generation of constraints or building the
cache, the only sign that things are wrong is failing canary build,
so impact to it should be minimal - we will either see the main
build failing after merge, or we will fix it alongside with fixing
some other reason for main failing (such us broken dependencies)
Currently, the following upgrades are done in canary build:
* Upgrading pre-commits
* Upgrading black version
* Upgrading build pinned dependencies
---
.github/workflows/ci.yml | 39 +++++++++++
.pre-commit-config.yaml | 12 +++-
contributing-docs/08_static_code_checks.rst | 4 +-
dev/breeze/doc/ci/05_workflows.md | 75 +++++++++++-----------
dev/breeze/doc/ci/06_diagrams.md | 2 +
dev/breeze/doc/images/output_static-checks.svg | 16 ++---
dev/breeze/doc/images/output_static-checks.txt | 2 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 +
pyproject.toml | 15 ++---
.../pre_commit_update_build_dependencies.py | 56 ++++++++++++++++
10 files changed, 165 insertions(+), 57 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3e7ab144c8..e0e4ca56e9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -583,6 +583,45 @@ jobs:
DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
RUFF_FORMAT: "github"
+ upgrade-check:
+ timeout-minutes: 45
+ name: "Upgrade checks"
+ runs-on: ["ubuntu-22.04"]
+ needs: [build-info]
+ env:
+ RUNS_ON: "${{needs.build-info.outputs.runs-on}}"
+ PYTHON_MAJOR_MINOR_VERSION:
"${{needs.build-info.outputs.default-python-version}}"
+ if: needs.build-info.outputs.canary-run == 'true'
+ steps:
+ - name: Cleanup repo
+ run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm
-rf /workspace/*"
+ - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+ uses: actions/checkout@v4
+ with:
+ persist-credentials: false
+ # Install python from scratch. No cache used. We always want to have
fresh version of everything
+ - uses: actions/setup-python@v5
+ with:
+ python-version:
"${{needs.build-info.outputs.default-python-version}}"
+ - name: "Install latest pre-commit"
+ run: pip install pre-commit
+ - name: "Autoupdate all pre-commits"
+ run: pre-commit autoupdate
+ - name: "Run automated upgrade for black"
+ run: >
+ pre-commit run
+ --all-files --show-diff-on-failure --color always --verbose
+ --hook-stage manual
+ update-black-version
+ if: always()
+ - name: "Run automated upgrade for build dependencies"
+ run: >
+ pre-commit run
+ --all-files --show-diff-on-failure --color always --verbose
+ --hook-stage manual
+ update-build-dependencies
+ if: always()
+
# Runs static checks for groups of files in the repository in a single
process without passing .
# List of files
mypy:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 100c8da645..cd7565f0b4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -176,13 +176,23 @@ repos:
pass_filenames: false
require_serial: true
- id: update-black-version
- name: Update black versions everywhere
+ name: Update black versions everywhere (manual)
entry: ./scripts/ci/pre_commit/pre_commit_update_black_version.py
+ stages: ['manual']
language: python
files: ^.pre-commit-config.yaml$
additional_dependencies: ['pyyaml']
pass_filenames: false
require_serial: true
+ - id: update-build-dependencies
+ name: Update build-dependencies to latest (manual)
+ entry: ./scripts/ci/pre_commit/pre_commit_update_build_dependencies.py
+ stages: ['manual']
+ language: python
+ language_version: python38
+ files:
^.pre-commit-config.yaml$|^scripts/ci/pre_commit/pre_commit_update_build_dependencies.py$
+ pass_filenames: false
+ require_serial: true
- repo: https://github.com/asottile/blacken-docs
rev: 1.16.0
hooks:
diff --git a/contributing-docs/08_static_code_checks.rst
b/contributing-docs/08_static_code_checks.rst
index c49eea4430..8b5acb0048 100644
--- a/contributing-docs/08_static_code_checks.rst
+++ b/contributing-docs/08_static_code_checks.rst
@@ -323,12 +323,14 @@ require Breeze Docker image to be built locally.
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| ts-compile-format-lint-www | TS types
generation / ESLint / Prettier against UI files | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
-| update-black-version | Update black
versions everywhere | |
+| update-black-version | Update black
versions everywhere (manual) | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| update-breeze-cmd-output | Update output of
breeze commands in Breeze documentation | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| update-breeze-readme-config-hash | Update Breeze
README.md with config files hash | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
+| update-build-dependencies | Update
build-dependencies to latest (manual) | |
++-----------------------------------------------------------+--------------------------------------------------------------+---------+
| update-common-sql-api-stubs | Check and update
common.sql API stubs | |
+-----------------------------------------------------------+--------------------------------------------------------------+---------+
| update-er-diagram | Update ER
diagram | * |
diff --git a/dev/breeze/doc/ci/05_workflows.md
b/dev/breeze/doc/ci/05_workflows.md
index 8310b9ce30..eaa2279262 100644
--- a/dev/breeze/doc/ci/05_workflows.md
+++ b/dev/breeze/doc/ci/05_workflows.md
@@ -219,43 +219,44 @@ builds use "Public" runners. For committers, this can be
overridden by setting t
This workflow is a regular workflow that performs all checks of Airflow
code.
-| Job | Description
| PR | Canary | Scheduled | Release branches |
-|----------------------------------|----------------------------------------------------------|-----------|----------|------------|-------------------|
-| Build info | Prints detailed information about the
build | Yes | Yes | Yes | Yes |
-| Push early cache & images | Pushes early cache/images to GitHub
Registry | | Yes | | |
-| Check that image builds quickly | Checks that image builds quickly
| | Yes | | Yes |
-| Build CI images | Builds images in-workflow (not in the
build images) | | Yes | Yes (1) | Yes (4) |
-| Generate constraints/CI verify | Generate constraints for the build and
verify CI image | Yes (2) | Yes (2) | Yes (2) | Yes (2) |
-| Build PROD images | Builds images in-workflow (not in the
build images) | | Yes | Yes (1) | Yes (4) |
-| Build Bullseye PROD images | Builds images based on Bullseye debian
| | Yes | Yes | Yes |
-| Run breeze tests | Run unit tests for Breeze
| Yes | Yes | Yes | Yes |
-| Test OpenAPI client gen | Tests if OpenAPIClient continues to
generate | Yes | Yes | Yes | Yes |
-| React WWW tests | React UI tests for new Airflow UI
| Yes | Yes | Yes | Yes |
-| Test examples image building | Tests if PROD image build examples work
| Yes | Yes | Yes | Yes |
-| Test git clone on Windows | Tests if Git clone for for Windows
| Yes (5) | Yes (5) | Yes (5) | Yes (5) |
-| Waits for CI Images | Waits for and verify CI Images
| Yes (2) | Yes (2) | Yes (2) | Yes (2) |
-| Static checks | Performs full static checks
| Yes (6) | Yes | Yes | Yes (7) |
-| Basic static checks | Performs basic static checks (no image)
| Yes (6) | | | |
-| Build docs | Builds and tests publishing of the
documentation | Yes | Yes | Yes | Yes |
-| Spellcheck docs | Spellcheck docs
| Yes | Yes | Yes | Yes |
-| Tests wheel provider packages | Tests if provider packages can be built
and released | Yes | Yes | Yes | |
-| Tests Airflow compatibility | Compatibility of provider packages with
older Airflow | Yes | Yes | Yes | |
-| Tests dist provider packages | Tests if dist provider packages can be
built | | Yes | Yes | |
-| Tests airflow release commands | Tests if airflow release command works
| | Yes | Yes | |
-| Tests (Backend/Python matrix) | Run the Pytest unit DB tests
(Backend/Python matrix) | Yes | Yes | Yes | Yes (8)
|
-| No DB tests | Run the Pytest unit Non-DB tests (with
pytest-xdist) | Yes | Yes | Yes | Yes (8) |
-| Integration tests | Runs integration tests (Postgres/Mysql)
| Yes | Yes | Yes | Yes (9) |
-| Quarantined tests | Runs quarantined tests (with flakiness
and side-effects) | Yes | Yes | Yes | Yes (8) |
-| Test airflow packages | Tests that Airflow package can be built
and released | Yes | Yes | Yes | Yes |
-| Helm tests | Run the Helm integration tests
| Yes | Yes | Yes | |
-| Helm release tests | Run the tests for Helm releasing
| Yes | Yes | Yes | |
-| Summarize warnings | Summarizes warnings from all other tests
| Yes | Yes | Yes | Yes |
-| Wait for PROD Images | Waits for and verify PROD Images
| Yes (2) | Yes (2) | Yes (2) | Yes (2) |
-| Docker Compose test/PROD verify | Tests quick-start Docker Compose and
verify PROD image | Yes | Yes | Yes | Yes |
-| Tests Kubernetes | Run Kubernetes test
| Yes | Yes | Yes | |
-| Update constraints | Upgrade constraints to latest ones
| Yes (3) | Yes (3) | Yes (3) | Yes (3) |
-| Push cache & images | Pushes cache/images to GitHub Registry
(3) | | Yes (3) | | Yes |
-| Build CI ARM images | Builds CI images for ARM
| Yes (10) | | Yes | |
+| Job | Description
| PR | Canary | Scheduled | Release branches |
+|---------------------------------|----------------------------------------------------------|----------|----------|------------|------------------|
+| Build info | Prints detailed information about the
build | Yes | Yes | Yes | Yes |
+| Push early cache & images | Pushes early cache/images to GitHub
Registry | | Yes | | |
+| Check that image builds quickly | Checks that image builds quickly
| | Yes | | Yes |
+| Build CI images | Builds images in-workflow (not in the
build images) | | Yes | Yes (1) | Yes (4) |
+| Generate constraints/CI verify | Generate constraints for the build and
verify CI image | Yes (2) | Yes (2) | Yes (2) | Yes (2) |
+| Build PROD images | Builds images in-workflow (not in the
build images) | | Yes | Yes (1) | Yes (4) |
+| Build Bullseye PROD images | Builds images based on Bullseye debian
| | Yes | Yes | Yes |
+| Run breeze tests | Run unit tests for Breeze
| Yes | Yes | Yes | Yes |
+| Test OpenAPI client gen | Tests if OpenAPIClient continues to
generate | Yes | Yes | Yes | Yes |
+| React WWW tests | React UI tests for new Airflow UI
| Yes | Yes | Yes | Yes |
+| Test examples image building | Tests if PROD image build examples work
| Yes | Yes | Yes | Yes |
+| Test git clone on Windows | Tests if Git clone for for Windows
| Yes (5) | Yes (5) | Yes (5) | Yes (5) |
+| Waits for CI Images | Waits for and verify CI Images
| Yes (2) | Yes (2) | Yes (2) | Yes (2) |
+| Upgrade checks | Performs checks if there are some pending
upgrades | | Yes | Yes | Yes |
+| Static checks | Performs full static checks
| Yes (6) | Yes | Yes | Yes (7) |
+| Basic static checks | Performs basic static checks (no image)
| Yes (6) | | | |
+| Build docs | Builds and tests publishing of the
documentation | Yes | Yes | Yes | Yes |
+| Spellcheck docs | Spellcheck docs
| Yes | Yes | Yes | Yes |
+| Tests wheel provider packages | Tests if provider packages can be built
and released | Yes | Yes | Yes | |
+| Tests Airflow compatibility | Compatibility of provider packages with
older Airflow | Yes | Yes | Yes | |
+| Tests dist provider packages | Tests if dist provider packages can be
built | | Yes | Yes | |
+| Tests airflow release commands | Tests if airflow release command works
| | Yes | Yes | |
+| Tests (Backend/Python matrix) | Run the Pytest unit DB tests
(Backend/Python matrix) | Yes | Yes | Yes | Yes (8)
|
+| No DB tests | Run the Pytest unit Non-DB tests (with
pytest-xdist) | Yes | Yes | Yes | Yes (8) |
+| Integration tests | Runs integration tests (Postgres/Mysql)
| Yes | Yes | Yes | Yes (9) |
+| Quarantined tests | Runs quarantined tests (with flakiness and
side-effects) | Yes | Yes | Yes | Yes (8) |
+| Test airflow packages | Tests that Airflow package can be built
and released | Yes | Yes | Yes | Yes |
+| Helm tests | Run the Helm integration tests
| Yes | Yes | Yes | |
+| Helm release tests | Run the tests for Helm releasing
| Yes | Yes | Yes | |
+| Summarize warnings | Summarizes warnings from all other tests
| Yes | Yes | Yes | Yes |
+| Wait for PROD Images | Waits for and verify PROD Images
| Yes (2) | Yes (2) | Yes (2) | Yes (2) |
+| Docker Compose test/PROD verify | Tests quick-start Docker Compose and
verify PROD image | Yes | Yes | Yes | Yes |
+| Tests Kubernetes | Run Kubernetes test
| Yes | Yes | Yes | |
+| Update constraints | Upgrade constraints to latest ones
| Yes (3) | Yes (3) | Yes (3) | Yes (3) |
+| Push cache & images | Pushes cache/images to GitHub Registry (3)
| | Yes (3) | | Yes |
+| Build CI ARM images | Builds CI images for ARM
| Yes (10) | | Yes | |
`(1)` Scheduled jobs builds images from scratch - to test if everything
works properly for clean builds
diff --git a/dev/breeze/doc/ci/06_diagrams.md b/dev/breeze/doc/ci/06_diagrams.md
index c9610ff009..89d6fc772c 100644
--- a/dev/breeze/doc/ci/06_diagrams.md
+++ b/dev/breeze/doc/ci/06_diagrams.md
@@ -363,6 +363,8 @@ sequenceDiagram
Note over Tests: React WWW tests
and
Note over Tests: Test git clone on Windows
+ and
+ Note over Tests: Run upgrade checks
end
Note over Tests: Skip waiting for CI images
par
diff --git a/dev/breeze/doc/images/output_static-checks.svg
b/dev/breeze/doc/images/output_static-checks.svg
index f05cd5435b..7c119d2cc6 100644
--- a/dev/breeze/doc/images/output_static-checks.svg
+++ b/dev/breeze/doc/images/output_static-checks.svg
@@ -347,14 +347,14 @@
</text><text class="breeze-static-checks-r5" x="0" y="1020.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-41)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1020.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-41)">python-no-log-warn | replace-bad-characters | rst-backticks | ruff | ruff-format </text><text
class="breeze-static-checks-r5" x="1451.8" y="1020.4" textLength="12.2"
clip-path="url(#breez [...]
</text><text class="breeze-static-checks-r5" x="0" y="1044.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-42)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1044.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-42)">| shellcheck | trailing-whitespace | ts-compile-format-lint-www |                </text><text
class="breeze-static-checks- [...]
</text><text class="breeze-static-checks-r5" x="0" y="1069.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-43)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1069.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-43)">update-black-version | update-breeze-cmd-output |                            
[...]
-</text><text class="breeze-static-checks-r5" x="0" y="1093.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-44)">update-breeze-readme-config-hash | update-common-sql-api-stubs |                 </text><text
class="breeze-static-checks-r5" x="145 [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-static-checks-line-45)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2"
clip-path="url(#breeze-static-checks-line-45)">update-er-diagram | update-extras | update-in-the-wild-to-be-sorted |            </text><text
class="breeze-static-checks-r5" x="1451.8" y="1118" textL [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1142.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-46)">update-inlined-dockerfile-scripts | update-installed-providers-to-be-sorted |    </text><text
class="breeze-static-checks-r5" x="1451.8" y="1142.4" textLength="12.2"
clip-path="url(#breeze-static-c [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1166.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-47)">update-local-yml-file | update-migration-references |                           &#
[...]
-</text><text class="breeze-static-checks-r5" x="0" y="1191.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-48)">update-providers-dependencies | update-reproducible-source-date-epoch |          </text><text
class="breeze-static-checks-r5" x="1451.8" y="1191.2" textLength="12.2" c [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1215.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-49)">update-spelling-wordlist-to-be-sorted | update-supported-versions |              </text><text
class="breeze-static-checks-r5" x="1451.8" y="1215.6" [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks-line-50)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2"
clip-path="url(#breeze-static-checks-line-50)">update-vendored-in-k8s-json-schema | update-version | validate-operators-init |  </text><text
class="breeze-static-checks-r5" x="1451.8" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks- [...]
-</text><text class="breeze-static-checks-r5" x="0" y="1264.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-51)">yamllint)                                     &
[...]
+</text><text class="breeze-static-checks-r5" x="0" y="1093.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-44)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1093.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-44)">update-breeze-readme-config-hash | update-build-dependencies |                   </text><text
class="breeze-static-checks- [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1118" textLength="12.2"
clip-path="url(#breeze-static-checks-line-45)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1118" textLength="988.2"
clip-path="url(#breeze-static-checks-line-45)">update-common-sql-api-stubs | update-er-diagram | update-extras |                </text><text
class="breeze-static-checks-r5" x="14 [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1142.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-46)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1142.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-46)">update-in-the-wild-to-be-sorted | update-inlined-dockerfile-scripts |            </text><text
class="breeze-static-checks-r5" x="1451.8" y="1142.4" textLengt [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1166.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-47)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1166.8" textLength="988.2"
clip-path="url(#breeze-static-checks-line-47)">update-installed-providers-to-be-sorted | update-local-yml-file |                </text><text
class="breeze-static-checks-r5" x="1451.8" [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1191.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-48)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1191.2" textLength="988.2"
clip-path="url(#breeze-static-checks-line-48)">update-migration-references | update-providers-dependencies |                    </text><text
class="breeze-static-ch [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1215.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-49)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1215.6" textLength="988.2"
clip-path="url(#breeze-static-checks-line-49)">update-reproducible-source-date-epoch | update-spelling-wordlist-to-be-sorted |  </text><text
class="breeze-static-checks-r5" x="1451.8" y="1215.6" textLength="12.2"
clip-path="url(#breeze-static-checks-line [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks-line-50)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1240" textLength="988.2"
clip-path="url(#breeze-static-checks-line-50)">update-supported-versions | update-vendored-in-k8s-json-schema | update-version |</text><text
class="breeze-static-checks-r5" x="1451.8" y="1240" textLength="12.2"
clip-path="url(#breeze-static-checks-line-50)"> [...]
+</text><text class="breeze-static-checks-r5" x="0" y="1264.4"
textLength="12.2" clip-path="url(#breeze-static-checks-line-51)">│</text><text
class="breeze-static-checks-r7" x="451.4" y="1264.4" textLength="988.2"
clip-path="url(#breeze-static-checks-line-51)">validate-operators-init | yamllint)                               &
[...]
</text><text class="breeze-static-checks-r5" x="0" y="1288.8"
textLength="12.2" clip-path="url(#breeze-static-checks-line-52)">│</text><text
class="breeze-static-checks-r4" x="24.4" y="1288.8" textLength="12.2"
clip-path="url(#breeze-static-checks-line-52)">-</text><text
class="breeze-static-checks-r4" x="36.6" y="1288.8" textLength="61"
clip-path="url(#breeze-static-checks-line-52)">-show</text><text
class="breeze-static-checks-r4" x="97.6" y="1288.8" textLength="195.2"
clip-path="url(# [...]
</text><text class="breeze-static-checks-r5" x="0" y="1313.2"
textLength="12.2" clip-path="url(#breeze-static-checks-line-53)">│</text><text
class="breeze-static-checks-r4" x="24.4" y="1313.2" textLength="12.2"
clip-path="url(#breeze-static-checks-line-53)">-</text><text
class="breeze-static-checks-r4" x="36.6" y="1313.2" textLength="134.2"
clip-path="url(#breeze-static-checks-line-53)">-initialize</text><text
class="breeze-static-checks-r4" x="170.8" y="1313.2" textLength="146.4" clip-p
[...]
</text><text class="breeze-static-checks-r5" x="0" y="1337.6"
textLength="12.2" clip-path="url(#breeze-static-checks-line-54)">│</text><text
class="breeze-static-checks-r4" x="24.4" y="1337.6" textLength="12.2"
clip-path="url(#breeze-static-checks-line-54)">-</text><text
class="breeze-static-checks-r4" x="36.6" y="1337.6" textLength="48.8"
clip-path="url(#breeze-static-checks-line-54)">-max</text><text
class="breeze-static-checks-r4" x="85.4" y="1337.6" textLength="292.8"
clip-path="url( [...]
diff --git a/dev/breeze/doc/images/output_static-checks.txt
b/dev/breeze/doc/images/output_static-checks.txt
index 7a064e6df4..c2860a4651 100644
--- a/dev/breeze/doc/images/output_static-checks.txt
+++ b/dev/breeze/doc/images/output_static-checks.txt
@@ -1 +1 @@
-42639ed45de3e438c5e9f95bf3d8b300
+406f5ff089e02996c057276244f98c1d
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index ad6614d452..3584779319 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -119,6 +119,7 @@ PRE_COMMIT_LIST = [
"update-black-version",
"update-breeze-cmd-output",
"update-breeze-readme-config-hash",
+ "update-build-dependencies",
"update-common-sql-api-stubs",
"update-er-diagram",
"update-extras",
diff --git a/pyproject.toml b/pyproject.toml
index dd181ddbdf..e4c7002c2b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -19,22 +19,19 @@
# build dependencies should be fixed - including all transitive dependencies.
This way we can ensure
# reproducibility of the build and make sure that any future releases of any
dependencies will not
# break the build of released airflow sources in the future.
-# We should periodically refresh these build dependencies by following this
procecedure:
-# 1. Create a new virtual environment
-# 2. Run `pip install hatchling gitpython
-# 3. Run `pip freeze`
-# 4. Copy versions you see here (note that tomli needs to have python_version
< '3.11').
+# The dependencies can be automatically upgraded by running:
+# pre-commit run --hook-stage manual update-build-dependencies --all-files
requires = [
+ "GitPython==3.1.41",
"editables==0.5",
"gitdb==4.0.11",
- "gitpython==3.1.40",
- "hatchling==1.21.0",
+ "hatchling==1.21.1",
"packaging==23.2",
"pathspec==0.12.1",
- "pluggy==1.3.0",
+ "pluggy==1.4.0",
"smmap==5.0.1",
"tomli==2.0.1; python_version < '3.11'",
- "trove-classifiers==2023.11.29",
+ "trove-classifiers==2024.1.8",
]
build-backend = "hatchling.build"
diff --git a/scripts/ci/pre_commit/pre_commit_update_build_dependencies.py
b/scripts/ci/pre_commit/pre_commit_update_build_dependencies.py
new file mode 100755
index 0000000000..64c96a90e4
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_update_build_dependencies.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
+PYPROJECT_TOML_FILE = AIRFLOW_SOURCES / "pyproject.toml"
+
+if __name__ == "__main__":
+ temp_dir = Path(tempfile.mkdtemp())
+ try:
+ subprocess.check_call([sys.executable, "-m", "venv",
temp_dir.as_posix()])
+ venv_python = temp_dir / "bin" / "python"
+ subprocess.check_call([venv_python, "-m", "pip", "install",
"gitpython", "hatchling"])
+ frozen_deps = subprocess.check_output([venv_python, "-m", "pip",
"freeze"], text=True)
+ deps = [dep for dep in sorted(frozen_deps.splitlines()) if not
dep.startswith("pip==")]
+ pyproject_toml_content = PYPROJECT_TOML_FILE.read_text()
+ result = []
+ skipping = False
+ for line in pyproject_toml_content.splitlines():
+ if not skipping:
+ result.append(line)
+ if line == "requires = [":
+ skipping = True
+ for dep in deps:
+ # Tomli is only needed for Python < 3.11, otherwise stdlib
tomllib is used
+ if dep.startswith("tomli=="):
+ dep = dep + "; python_version < '3.11'"
+ result.append(f' "{dep}",')
+ if skipping and line == "]":
+ skipping = False
+ result.append(line)
+ result.append("")
+ PYPROJECT_TOML_FILE.write_text("\n".join(result))
+ finally:
+ shutil.rmtree(temp_dir)