This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new f790146a69 Convert mypy pre-commits to use breeze shell to run them
(#35877)
f790146a69 is described below
commit f790146a693e6d410755d2f8c0813543c7549d40
Author: Jarek Potiuk <[email protected]>
AuthorDate: Mon Nov 27 16:24:23 2023 +0100
Convert mypy pre-commits to use breeze shell to run them (#35877)
The Mypy static checks now use breze shell to execute, also they
force color so the errors are seen with coloured output even when
they are run as part of git hook.
Scripts folder needs to be added in its entirety in order to be
able to run the mypy check without mounting the volumes to
running container - but relying on the code already in the image.
---
.dockerignore | 3 +-
.pre-commit-config.yaml | 10 +--
scripts/ci/pre_commit/pre_commit_mypy.py | 101 +++++++++++--------------------
scripts/in_container/run_mypy.sh | 3 +
4 files changed, 46 insertions(+), 71 deletions(-)
diff --git a/.dockerignore b/.dockerignore
index 803b75ea80..d2a5d3e98c 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -36,8 +36,7 @@
!licenses
# Add those folders to the context so that they are available in the CI
container
-!scripts/in_container
-!scripts/docker
+!scripts
# Add tests and kubernetes_tests to context.
!tests
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 05adea077d..5b87ecdb70 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1024,17 +1024,17 @@ repos:
name: Run mypy for dev
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py
- files: ^dev/.*\.py$
+ files: ^dev/.*\.py$|^scripts/.*\.py$
require_serial: true
- additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml',
'jsonschema', 'filelock', 'markdown-it-py']
+ additional_dependencies: ['rich>=12.4.4']
- id: mypy-core
name: Run mypy for core
language: python
entry: ./scripts/ci/pre_commit/pre_commit_mypy.py --namespace-packages
files: \.py$
- exclude:
^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py
+ exclude:
^.*/.*_vendor/|^airflow/migrations|^airflow/providers|^dev|^scripts|^docs|^provider_packages|^tests/providers|^tests/system/providers|^tests/dags/test_imports.py
require_serial: true
- additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml',
'jsonschema', 'filelock', 'markdown-it-py']
+ additional_dependencies: ['rich>=12.4.4']
- id: mypy-providers
name: Run mypy for providers
language: python
@@ -1042,7 +1042,7 @@ repos:
files:
^airflow/providers/.*\.py$|^tests/providers/.*\.py$|^tests/system/providers/.*\.py$
exclude: ^.*/.*_vendor/
require_serial: true
- additional_dependencies: ['rich>=12.4.4', 'inputimeout', 'pyyaml',
'jsonschema', 'filelock', 'markdown-it-py']
+ additional_dependencies: ['rich>=12.4.4']
- id: mypy-docs
name: Run mypy for /docs/ folder
language: python
diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py
b/scripts/ci/pre_commit/pre_commit_mypy.py
index 9a51babc01..b89afa41b2 100755
--- a/scripts/ci/pre_commit/pre_commit_mypy.py
+++ b/scripts/ci/pre_commit/pre_commit_mypy.py
@@ -21,72 +21,45 @@ import os
import sys
from pathlib import Path
-if __name__ not in ("__main__", "__mp_main__"):
- raise SystemExit(
- "This file is intended to be executed as an executable program. You
cannot use it as a module."
- f"To run this script, run the ./{__file__} command"
- )
-
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
-AIRFLOW_SOURCES = Path(__file__).parents[3].resolve()
-GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "apache/airflow")
-os.environ["SKIP_GROUP_OUTPUT"] = "true"
+from common_precommit_utils import (
+ console,
+ filter_out_providers_on_non_main_branch,
+ initialize_breeze_precommit,
+ run_command_via_breeze_shell,
+)
-if __name__ == "__main__":
- sys.path.insert(0, str(Path(__file__).parent.resolve())) # make sure
common_precommit_utils is imported
+initialize_breeze_precommit(__name__, __file__)
- os.environ["SKIP_BREEZE_SELF_UPGRADE_CHECK"] = "true"
- from common_precommit_utils import filter_out_providers_on_non_main_branch
-
- sys.path.insert(0, str(AIRFLOW_SOURCES / "dev" / "breeze" / "src"))
- from airflow_breeze.global_constants import
DEFAULT_PYTHON_MAJOR_MINOR_VERSION, MOUNT_SELECTED
- from airflow_breeze.params.shell_params import ShellParams
- from airflow_breeze.utils.console import get_console # isort: skip
- from airflow_breeze.utils.docker_command_utils import
get_extra_docker_flags # isort: skip
- from airflow_breeze.utils.path_utils import create_mypy_volume_if_needed
# isort: skip
- from airflow_breeze.utils.run_utils import (
- get_ci_image_for_pre_commits,
- run_command,
- )
+files_to_test = filter_out_providers_on_non_main_branch(sys.argv[1:])
+if files_to_test == ["--namespace-packages"]:
+ print("No files to tests. Quitting")
+ sys.exit(0)
- shell_params = ShellParams(python=DEFAULT_PYTHON_MAJOR_MINOR_VERSION,
backend="none")
-
- files_to_test = filter_out_providers_on_non_main_branch(sys.argv[1:])
- if files_to_test == ["--namespace-packages"]:
- print("No files to tests. Quitting")
- sys.exit(0)
- airflow_image = get_ci_image_for_pre_commits()
- create_mypy_volume_if_needed()
- cmd_result = run_command(
- [
- "docker",
- "run",
- "-t",
- *get_extra_docker_flags(mount_sources=MOUNT_SELECTED,
include_mypy_volume=True),
- "-e",
- "SKIP_ENVIRONMENT_INITIALIZATION=true",
- "--pull",
- "never",
- airflow_image,
- "/opt/airflow/scripts/in_container/run_mypy.sh",
- *files_to_test,
- ],
- check=False,
- env=shell_params.env_variables_for_docker_commands,
- )
- if cmd_result.returncode != 0:
- upgrading = os.environ.get("UPGRADE_TO_NEWER_DEPENDENCIES", "false")
!= "false"
- if upgrading:
- get_console().print(
- "[warning]You are running mypy with the image that has
dependencies upgraded automatically."
- )
- flag = " --upgrade-to-newer-dependencies" if upgrading else ""
- get_console().print(
- "[warning]If you see strange stacktraces above, "
- f"run `breeze ci-image build --python 3.8{flag}` and try again. "
- "You can also run `breeze down --cleanup-mypy-cache` to clean up
the cache used. "
- "Still sometimes diff heuristic in mypy is behaving abnormal, to
double check you can "
- "call `breeze static-checks --type mypy-[dev|core|providers|docs]
--all-files` "
- 'and then commit via `git commit --no-verify -m "commit message"`.
CI will do a full check.'
+res = run_command_via_breeze_shell(
+ [
+ "/opt/airflow/scripts/in_container/run_mypy.sh",
+ *files_to_test,
+ ],
+ warn_image_upgrade_needed=True,
+ extra_env={
+ "INCLUDE_MYPY_VOLUME": "true",
+ },
+)
+if res.returncode != 0:
+ upgrading = os.environ.get("UPGRADE_TO_NEWER_DEPENDENCIES", "false") !=
"false"
+ if upgrading:
+ console.print(
+ "[yellow]You are running mypy with the image that has dependencies
upgraded automatically."
)
- sys.exit(cmd_result.returncode)
+ flag = " --upgrade-to-newer-dependencies" if upgrading else ""
+ console.print(
+ "[yellow]If you see strange stacktraces above, "
+ f"run `breeze ci-image build --python 3.8{flag}` and try again. "
+ "You can also run `breeze down --cleanup-mypy-cache` to clean up the
cache used. "
+ "Still sometimes diff heuristic in mypy is behaving abnormal, to
double check you can "
+ "call `breeze static-checks --type mypy-[dev|core|providers|docs]
--all-files` "
+ 'and then commit via `git commit --no-verify -m "commit message"`. CI
will do a full check.'
+ )
+sys.exit(res.returncode)
diff --git a/scripts/in_container/run_mypy.sh b/scripts/in_container/run_mypy.sh
index cee381d8cb..0245825a72 100755
--- a/scripts/in_container/run_mypy.sh
+++ b/scripts/in_container/run_mypy.sh
@@ -22,6 +22,9 @@ export PYTHONPATH=${AIRFLOW_SOURCES}
ADDITIONAL_MYPY_OPTIONS=()
+export MYPY_FORCE_COLOR=true
+export TERM=ansi
+
if [[ ${SUSPENDED_PROVIDERS_FOLDERS=} != "" ]];
then
for folder in ${SUSPENDED_PROVIDERS_FOLDERS=}
