This is an automated email from the ASF dual-hosted git repository. ephraimanierobi pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push: new 09e6f072c1 Add `pre-commit` to check that `REVISION_HEADS_MAP` is up-to-date (#22860) 09e6f072c1 is described below commit 09e6f072c1a06594c4d2e8013e810e7ddd114fba Author: Ephraim Anierobi <splendidzig...@gmail.com> AuthorDate: Fri Apr 8 18:20:54 2022 +0100 Add `pre-commit` to check that `REVISION_HEADS_MAP` is up-to-date (#22860) This PR adds a `pre-commit` to make sure that the `REVISION_HEADS_MAP` is up-to-date in any release. Since this feature will be out in 2.3.0 and is very important to update it for releases, not just main, this would be helpful --- .pre-commit-config.yaml | 5 ++ BREEZE.rst | 33 ++++++------ STATIC_CODE_CHECKS.rst | 2 + breeze-complete | 1 + dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 + .../ci/pre_commit/pre_commit_version_heads_map.py | 62 ++++++++++++++++++++++ 6 files changed, 88 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c256c76036..3de7b7ad2b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -364,6 +364,11 @@ repos: files: ^scripts/ci/pre_commit/supported_versions\.py$|^README\.md$|^docs/apache-airflow/supported-versions\.rst$ pass_filenames: false additional_dependencies: ['tabulate'] + - id: check-revision-heads-map + name: Check that the REVISION_HEADS_MAP is up-to-date + language: python + entry: ./scripts/ci/pre_commit/pre_commit_version_heads_map.py + pass_filenames: false - id: update-version name: Update version to the latest version in the documentation entry: ./scripts/ci/pre_commit/pre_commit_update_versions.py diff --git a/BREEZE.rst b/BREEZE.rst index f734d821d9..03608ee6f6 100644 --- a/BREEZE.rst +++ b/BREEZE.rst @@ -2278,22 +2278,23 @@ This is the current syntax for `./breeze <./breeze>`_: build-providers-dependencies chart-schema-lint capitalized-breeze changelog-duplicates check-2-1-compatibility check-apache-license check-builtin-literals check-executables-have-shebangs check-extras-order - check-hooks-apply check-integrations check-merge-conflict check-xml - check-system-tests daysago-import-check debug-statements detect-private-key - docstring-params doctoc dont-use-safe-filter end-of-file-fixer fix-encoding-pragma - flake8 flynt codespell forbid-tabs helm-lint identity incorrect-use-of-LoggingMixin - insert-license inline-dockerfile-scripts isort json-schema language-matters - lint-dockerfile lint-openapi markdownlint mermaid migration-reference - mixed-line-ending mypy mypy-helm no-providers-in-core-examples no-relative-imports - persist-credentials-disabled pre-commit-descriptions pre-commit-hook-names - pretty-format-json provide-create-sessions providers-changelogs providers-init-file - providers-subpackages-init-file provider-yamls pydevd pydocstyle python-no-log-warn - pyupgrade restrict-start_date rst-backticks setup-order setup-extra-packages - shellcheck sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace - ui-lint update-breeze-config-hash update-breeze-file update-extras - update-local-yml-file update-setup-cfg-file update-supported-versions - update-versions vendor-k8s-json-schema verify-db-migrations-documented version-sync - www-lint yamllint yesqa + check-hooks-apply check-integrations check-merge-conflict check-revision-heads-map + check-xml check-system-tests daysago-import-check debug-statements + detect-private-key docstring-params doctoc dont-use-safe-filter end-of-file-fixer + fix-encoding-pragma flake8 flynt codespell forbid-tabs helm-lint identity + incorrect-use-of-LoggingMixin insert-license inline-dockerfile-scripts isort + json-schema language-matters lint-dockerfile lint-openapi markdownlint mermaid + migration-reference mixed-line-ending mypy mypy-helm no-providers-in-core-examples + no-relative-imports persist-credentials-disabled pre-commit-descriptions + pre-commit-hook-names pretty-format-json provide-create-sessions + providers-changelogs providers-init-file providers-subpackages-init-file + provider-yamls pydevd pydocstyle python-no-log-warn pyupgrade restrict-start_date + rst-backticks setup-order setup-extra-packages shellcheck sort-in-the-wild + sort-spelling-wordlist stylelint trailing-whitespace ui-lint + update-breeze-config-hash update-breeze-file update-extras update-local-yml-file + update-setup-cfg-file update-supported-versions update-versions + vendor-k8s-json-schema verify-db-migrations-documented version-sync www-lint + yamllint yesqa You can pass extra arguments including options to the pre-commit framework as <EXTRA_ARGS> passed after --. For example: diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst index 78b0202881..2fc0414345 100644 --- a/STATIC_CODE_CHECKS.rst +++ b/STATIC_CODE_CHECKS.rst @@ -165,6 +165,8 @@ require Breeze Docker images to be installed locally. ``check-integrations`` Checks if integration list is synchronized in code ------------------------------------ ---------------------------------------------------------------- ------------ ``check-merge-conflicts`` Checks that merge conflicts are not being committed +------------------------------------------------------------------------------------------------------------------ +``check-revision-heads-map`` Checks that REVISION_HEADS_MAP is up-to-date ------------------------------------ ---------------------------------------------------------------- ------------ ``check-xml`` Checks XML files with xmllint ------------------------------------ ---------------------------------------------------------------- ------------ diff --git a/breeze-complete b/breeze-complete index 77f6b6b6b3..b8e032aa37 100644 --- a/breeze-complete +++ b/breeze-complete @@ -98,6 +98,7 @@ check-extras-order check-hooks-apply check-integrations check-merge-conflict +check-revision-heads-map check-xml check-system-tests daysago-import-check diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py index 45957678f8..f9a9aaadbf 100644 --- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py +++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py @@ -41,6 +41,7 @@ PRE_COMMIT_LIST = [ 'check-hooks-apply', 'check-integrations', 'check-merge-conflict', + 'check-revision-heads-map', 'check-system-tests', 'check-xml', 'codespell', diff --git a/scripts/ci/pre_commit/pre_commit_version_heads_map.py b/scripts/ci/pre_commit/pre_commit_version_heads_map.py new file mode 100755 index 0000000000..7ba91ead0e --- /dev/null +++ b/scripts/ci/pre_commit/pre_commit_version_heads_map.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import ast +import sys +from pathlib import Path + +PROJECT_SOURCE_ROOT_DIR = Path(__file__).resolve().parent.parent.parent.parent + +DB_FILE = PROJECT_SOURCE_ROOT_DIR / "airflow" / "utils" / "db.py" + +SETUP_FILE = PROJECT_SOURCE_ROOT_DIR / "setup.py" + + +def read_revision_heads_map(): + revision_heads_map_ast_obj = ast.parse(open(DB_FILE).read()) + + revision_heads_map_ast = [ + a + for a in revision_heads_map_ast_obj.body + if isinstance(a, ast.Assign) and a.targets[0].id == "REVISION_HEADS_MAP" + ][0] + + revision_heads_map = ast.literal_eval(revision_heads_map_ast.value) + + return revision_heads_map.keys() + + +def read_current_airflow_version(): + + ast_obj = ast.parse(open(SETUP_FILE).read()) + assignments = [a for a in ast_obj.body if isinstance(a, ast.Assign)][:10] + + version = [x for x in assignments if x.targets[0].id == "version"][0] + + return ast.literal_eval(version.value) + + +if __name__ == '__main__': + versions = read_revision_heads_map() + airflow_version = read_current_airflow_version() + if 'dev' not in airflow_version and airflow_version not in versions: + print("Current airflow version is not in the REVISION_HEADS_MAP") + print("Current airflow version:", airflow_version) + print("Please add the version to the REVISION_HEADS_MAP at:", DB_FILE) + sys.exit(3)