This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v3-1-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-1-test by this push:
new 3a8a4534110 [v3-1-test] Extract prek hooks for Providers (#57250)
(#57260)
3a8a4534110 is described below
commit 3a8a453411073a05db955f0579a09ead15296b2f
Author: Jarek Potiuk <[email protected]>
AuthorDate: Sat Oct 25 22:38:03 2025 +0200
[v3-1-test] Extract prek hooks for Providers (#57250) (#57260)
* Extract prek hooks for Providers
* Ups in path
* Fix breeze script to consider running also in non repo root
* Fix prek script to consider running also in non repo root
* Fix prek script to consider running also in non repo root
(cherry picked from commit 045fbc5d80d630bf086d969dd12178601b5e2d08)
Co-authored-by: Jens Scheffler <[email protected]>
---
.pre-commit-config.yaml | 375 ++++--------------------
providers/.pre-commit-config.yaml | 232 +++++++++++++++
scripts/ci/prek/mypy.py | 7 +-
scripts/ci/prek/update_providers_build_files.py | 6 +-
4 files changed, 298 insertions(+), 322 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index b3fd8a169f0..2493f414db0 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -212,13 +212,6 @@ repos:
files:
^airflow-core/src/airflow/models/taskinstance\.py$|^airflow-core/src/airflow/models/taskinstancehistory\.py$
pass_filenames: false
require_serial: true
- - id: check-deferrable-default
- name: Check and fix default value of default_deferrable
- language: python
- entry: ./scripts/ci/prek/check_deferrable_default.py
- pass_filenames: false
- # libcst doesn't have source wheels for all PY except PY3.12,
excluding it
- files: ^(providers/.*/)?airflow/.*/(sensors|operators)/.*\.py$
- repo: https://github.com/adamchainz/blacken-docs
rev: dda8db18cfc68df532abf33b185ecd12d5b7b326 # frozen: 1.20.0
hooks:
@@ -232,7 +225,7 @@ repos:
- --target-version=py313
alias: blacken-docs
additional_dependencies:
- - 'black==25.1.0'
+ - 'black==25.9.0'
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0
hooks:
@@ -359,12 +352,6 @@ repos:
# changes quickly - especially when we want the early modifications from
the first local group
# to be applied before the non-local prek hooks are run
hooks:
- - id: update-providers-dependencies
- name: Update dependencies for providers
- entry: ./scripts/ci/prek/update_providers_dependencies.py
- language: python
- always_run: true
- pass_filenames: false
- id: check-shared-distributions-structure
name: Check shared distributions structure
entry: ./scripts/ci/prek/check_shared_distributions_structure.py
@@ -377,34 +364,6 @@ repos:
language: python
pass_filenames: false
files: ^shared/.*$|^.*/pyproject.toml$|^.*/_shared/.*$
- - id: validate-operators-init
- name: No templated field logic checks in operator __init__
- description: Prevent templated field logic checks in operators'
__init__
- language: python
- entry: ./scripts/ci/prek/validate_operators_init.py
- pass_filenames: true
- files:
^providers/.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$
- - id: update-providers-build-files
- name: Update providers build files
- entry: ./scripts/ci/prek/update_providers_build_files.py
- language: python
- pass_filenames: true
- files: |
- (?x)
- ^providers/[^\/]*/src/airflow/providers/[^\/]*/__init__\.py$|
-
^providers/[^\/]*/[^\/]*/src/airflow/providers/[^\/]*/[^\/]*/__init__\.py$|
- ^providers/.*/pyproject\.toml$|
- ^providers/.*/provider\.yaml$|
- ^airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE\.py\.jinja2$|
- ^airflow_breeze/templates/get_provider_info_TEMPLATE\.py\.jinja2$|
- ^airflow_breeze/templates/PROVIDER_README_TEMPLATE\.rst\.jinja2$
- require_serial: true
- - id: check-airflow-v-imports-in-tests
- name: Check AIRFLOW_V imports in tests
- language: python
- entry: ./scripts/ci/prek/check_airflow_v_imports_in_tests.py
- pass_filenames: true
- files: ^providers/.*/tests/.+\.py$
- id: ruff
name: Run 'ruff' for extremely fast Python linting
description: "Run 'ruff' for extremely fast Python linting"
@@ -494,12 +453,6 @@ repos:
language: python
files:
^dev/breeze/src/airflow_breeze/utils/docker_command_utils\.py$|^scripts/ci/docker_compose/local\.yml$
pass_filenames: false
- - id: check-sql-dependency-common-data-structure
- name: Check dependency of SQL providers
- description: Check dependency of SQL Providers with common data
structure
- entry: ./scripts/ci/prek/check_common_sql_dependency.py
- language: python
- files: ^providers/.*/src/airflow/providers/.*/hooks/.*\.py$
- id: check-extra-packages-references
name: Checks setup extra packages
description: Checks if all the extras defined in hatch_build.py are
listed in extra-packages-ref.rst file
@@ -520,15 +473,9 @@ repos:
files: >
(?x)
^airflow-core/docs/.*/diagram_[^/]*\.py$|
- ^docs/images/.*\.py$
+ ^docs/images/.*\.py$|
+ ^airflow-ctl/docs/images/diagrams/.*\.py$
pass_filenames: true
- - id: generate-volumes-for-sources
- name: Generate volumes for docker compose
- entry: ./scripts/ci/prek/generate_volumes_for_sources.py
- language: python
- files: ^providers/.*/provider\.yaml$
- pass_filenames: false
- require_serial: true
- id: prevent-deprecated-sqlalchemy-usage
name: Prevent deprecated sqlalchemy usage
entry: ./scripts/ci/prek/prevent_deprecated_sqlalchemy_usage.py
@@ -537,7 +484,6 @@ repos:
(?x)
^airflow-ctl.*\.py$|
^airflow-core/src/airflow/models/.*\.py$|
- ^providers/fab/.*\.py$|
^task_sdk.*\.py$
pass_filenames: true
- id: update-supported-versions
@@ -554,10 +500,9 @@ repos:
files: >
(?x)
^scripts/ci/prek/version_heads_map\.py$|
-
^airflow-core/src/airflow/migrations/versions/.*$|^airflow-core/src/airflow/migrations/versions|
-
^providers/fab/src/airflow/providers/fab/migrations/versions/.*$|^providers/fab/src/airflow/providers/fab/migrations/versions|
- ^airflow-core/src/airflow/utils/db\.py$|
- ^providers/fab/src/airflow/providers/fab/auth_manager/models/db\.py$
+ ^airflow-core/src/airflow/migrations/versions/.*$|
+ ^airflow-core/src/airflow/migrations/versions|
+ ^airflow-core/src/airflow/utils/db\.py$
- id: update-version
name: Update versions in docs
entry: ./scripts/ci/prek/update_versions.py
@@ -570,75 +515,6 @@ repos:
entry: "pydevd.*settrace\\("
pass_filenames: true
files: \.py$
- - id: check-pytest-mark-db-test-in-providers
- language: pygrep
- name: Check pytest.mark.db_test use in providers
- entry: pytest\.mark\.db_test
- pass_filenames: true
- # Here we should add providers that are already free from the
pytest.mark.db_test
- # and we want to keep them clean and only use non-db-tests
- files: >
- (?x)
- ^providers/airbyte/.*\.py$|
- ^providers/apache/beam/.*\.py$|
- ^providers/apache/flink/.*\.py$|
- ^providers/apache/iceberg/.*\.py$|
- ^providers/apache/kafka/.*\.py$|
- ^providers/apprise/.*\.py$|
- ^providers/arangodb/.*\.py$|
- ^providers/asana/.*\.py$|
- ^providers/atlassian/jira/.*\.py$|
- ^providers/cloudant/.*\.py$|
- ^providers/cohere/.*\.py$|
- ^providers/common/compat/.*\.py$|
- ^providers/common/messaging/.*\.py$|
- ^providers/datadog/.*\.py$|
- ^providers/dingding/.*\.py$|
- ^providers/discord/.*\.py$|
- ^providers/exasol/.*\.py$|
- ^providers/facebook/.*\.py$|
- ^providers/ftp/.*\.py$|
- ^providers/grpc/.*\.py$|
- ^providers/hashicorp/.*\.py$|
- ^providers/imap/.*\.py$|
- ^providers/influxdb/.*\.py$|
- ^providers/jdbc/.*\.py$|
- ^providers/jenkins/.*\.py$|
- ^providers/mongo/.*\.py$|
- ^providers/microsoft/psrp/.*\.py$|
- ^providers/microsoft/winrm/.*\.py$|
- ^providers/neo4j/.*\.py$|
- ^providers/odbc/.*\.py$|
- ^providers/openai/.*\.py$|
- ^providers/openfaas/.*\.py$|
- ^providers/opsgenie/.*\.py$|
- ^providers/oracle/.*\.py$|
- ^providers/pagerduty/.*\.py$|
- ^providers/pgvector/.*\.py$|
- ^providers/pinecone/.*\.py$|
- ^providers/postgres/.*\.py$|
- ^providers/presto/.*\.py$|
- ^providers/segment/.*\.py$|
- ^providers/sendgrid/.*\.py$|
- ^providers/singularity/.*\.py$|
- ^providers/slack/.*\.py$|
- ^providers/smtp/.*\.py$|
- ^providers/tableau/.*\.py$|
- ^providers/teradata/.*\.py$|
- ^providers/trino/.*\.py$|
- ^providers/vertica/.*\.py$|
- ^providers/yandex/.*\.py$|
- ^providers/zendesk/.*\.py$
- - id: check-links-to-example-dags-do-not-use-hardcoded-versions
- name: Verify no hard-coded version in example dags
- description: The links to example dags should use |version| as version
specification
- language: pygrep
- entry: >
- (?i)
-
.*https://github.*/main/providers/.*/src/airflow/providers/.*/example_dags/|
-
.*https://github.*/master/providers/.*/src/airflow/providers/.*/example_dags/
- pass_filenames: true
- files: ^providers/.*/docs/.*\.rst
- id: check-safe-filter-usage-in-html
language: pygrep
name: Don't use safe in templates
@@ -653,13 +529,6 @@ repos:
entry: "^\\s*from airflow\\.providers.(?!standard.)"
pass_filenames: true
files: ^airflow-core/src/airflow/example_dags/.*\.py$
- - id: check-no-airflow-deprecation-in-providers
- language: pygrep
- name: Do not use DeprecationWarning in providers
- description: Use AirflowProviderDeprecationWarning in providers
- entry: "^\\s*DeprecationWarning*"
- pass_filenames: true
- files: ^providers/.*/src/airflow/providers/.*\.py$
- id: check-urlparse-usage-in-code
language: pygrep
name: Don't use urlparse in code
@@ -692,80 +561,81 @@ repos:
pass_filenames: true
exclude: >
(?x)
- ^airflow-core/src/airflow/ui/src/i18n/config\.ts$|
- ^airflow-core/src/airflow/ui/openapi-gen/|
- ^airflow-core/src/airflow/ui/public/i18n/locales/de/README\.md$|
+ ^airflow-core/docs/.*commits\.rst$|
+ ^airflow-core/newsfragments/41368\.significant\.rst$|
+ ^airflow-core/newsfragments/41761.significant\.rst$|
+ ^airflow-core/newsfragments/43349\.significant\.rst$|
+
^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock\.yaml$|
^airflow-core/src/airflow/cli/commands/local_commands/fastapi_api_command\.py$|
^airflow-core/src/airflow/config_templates/|
^airflow-core/src/airflow/models/baseoperator\.py$|
^airflow-core/src/airflow/operators/__init__\.py$|
-
^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute\.py$|
+ ^airflow-core/src/airflow/serialization/serialized_objects\.py$|
+ ^airflow-core/src/airflow/ui/openapi-gen/|
+ ^airflow-core/src/airflow/ui/pnpm-lock\.yaml$|
+ ^airflow-core/src/airflow/ui/public/i18n/locales/de/README\.md$|
+ ^airflow-core/src/airflow/ui/src/i18n/config\.ts$|
+ ^airflow-core/src/airflow/utils/db\.py$|
+ ^airflow-core/src/airflow/utils/trigger_rule\.py$|
+ ^airflow-core/tests/|
+ ^.*changelog\.(rst|txt)$|
+ ^.*CHANGELOG\.(rst|txt)$|
+ ^chart/values.schema\.json$|
+ ^.*commits\.(rst|txt)$|
+ ^.*/conf_constants\.py$|
+ ^.*/conf\.py$|
+ ^contributing-docs/03_contributors_quick_start\.rst$|
+ ^dev/|
+ ^devel-common/src/docs/README\.rst$|
+ ^devel-common/src/sphinx_exts/removemarktransform\.py|
+ ^devel-common/src/tests_common/test_utils/db\.py|
+ .*/dist/.*|
+
^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store\.rst$|
+ git|
+ ^helm-tests/tests/chart_utils/helm_template_generator\.py$|
+ ^helm-tests/tests/chart_utils/ingress-networking-v1beta1\.json$|
+ package-lock\.json$|
+ ^.*\.(png|gif|jp[e]?g|svg|tgz|lock)$|
+ ^\.pre-commit-config\.yaml$|
+ ^.*/provider_conf\.py$|
+ ^providers/\.pre-commit-config\.yaml$|
^providers/amazon/src/airflow/providers/amazon/aws/hooks/emr\.py$|
^providers/amazon/src/airflow/providers/amazon/aws/operators/emr\.py$|
^providers/apache/cassandra/src/airflow/providers/apache/cassandra/hooks/cassandra\.py$|
+ ^providers/apache/hdfs/docs/connections\.rst$|
^providers/apache/hive/src/airflow/providers/apache/hive/operators/hive_stats\.py$|
^providers/apache/hive/src/airflow/providers/apache/hive/transfers/vertica_to_hive\.py$|
+ ^providers/apache/kafka/docs/connections/kafka\.rst$|
+ ^providers/apache/spark/docs/decorators/pyspark\.rst$|
^providers/apache/spark/src/airflow/providers/apache/spark/decorators/|
^providers/apache/spark/src/airflow/providers/apache/spark/hooks/|
^providers/apache/spark/src/airflow/providers/apache/spark/operators/|
+ ^providers/cncf/kubernetes/docs/operators\.rst$|
+
^providers/common/sql/tests/provider_tests/common/sql/operators/test_sql_execute\.py$|
+
^providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml$|
^providers/exasol/src/airflow/providers/exasol/hooks/exasol\.py$|
+ ^providers/fab/docs/auth-manager/webserver-authentication\.rst$|
^providers/fab/src/airflow/providers/fab/auth_manager/security_manager/|
^providers/fab/src/airflow/providers/fab/www/static/|
^providers/fab/src/airflow/providers/fab/www/templates/|
+ ^providers/google/docs/operators/cloud/kubernetes_engine\.rst$|
^providers/google/src/airflow/providers/google/cloud/hooks/bigquery\.py$|
^providers/google/src/airflow/providers/google/cloud/operators/cloud_build\.py$|
^providers/google/src/airflow/providers/google/cloud/operators/dataproc\.py$|
^providers/google/src/airflow/providers/google/cloud/operators/mlengine\.py$|
^providers/keycloak/src/airflow/providers/keycloak/auth_manager/cli/definition.py|
+ ^providers/microsoft/azure/docs/connections/azure_cosmos\.rst$|
^providers/microsoft/azure/src/airflow/providers/microsoft/azure/hooks/cosmos\.py$|
^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/hooks/winrm\.py$|
- ^airflow-core/docs/.*commits\.rst$|
^providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm\.py$|
^providers/opsgenie/src/airflow/providers/opsgenie/hooks/opsgenie\.py$|
^providers/redis/src/airflow/providers/redis/provider\.yaml$|
- ^airflow-core/src/airflow/serialization/serialized_objects\.py$|
-
^airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui/pnpm-lock\.yaml$|
- ^airflow-core/src/airflow/ui/pnpm-lock\.yaml$|
- ^airflow-core/src/airflow/utils/db\.py$|
- ^airflow-core/src/airflow/utils/trigger_rule\.py$|
- ^chart/values.schema\.json$|
- ^helm-tests/tests/chart_utils/helm_template_generator\.py$|
- ^helm-tests/tests/chart_utils/ingress-networking-v1beta1\.json$|
- ^dev/|
- ^devel-common/src/docs/README\.rst$|
-
^docs/apache-airflow-providers-amazon/secrets-backends/aws-ssm-parameter-store\.rst$|
- ^providers/apache/kafka/docs/connections/kafka\.rst$|
- ^providers/apache/hdfs/docs/connections\.rst$|
- ^providers/apache/spark/docs/decorators/pyspark\.rst$|
- ^providers/microsoft/azure/docs/connections/azure_cosmos\.rst$|
- ^providers/fab/docs/auth-manager/webserver-authentication\.rst$|
- ^providers/google/docs/operators/cloud/kubernetes_engine\.rst$|
- ^providers/cncf/kubernetes/docs/operators\.rst$|
- ^.*/conf\.py$|
- ^.*/conf_constants\.py$|
- ^.*/provider_conf\.py$|
- ^devel-common/src/sphinx_exts/removemarktransform\.py|
- ^devel-common/src/tests_common/test_utils/db\.py|
- ^airflow-core/newsfragments/41761.significant\.rst$|
- ^scripts/ci/prek/vendor_k8s_json_schema\.py$|
- ^scripts/ci/docker-compose/integration-keycloak\.yml$|
- ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$|
- ^airflow-core/tests/|
^providers/.*/tests/|
- ^\.pre-commit-config\.yaml$|
- ^.*CHANGELOG\.(rst|txt)$|
- ^.*changelog\.(rst|txt)$|
- ^.*commits\.(rst|txt)$|
- ^.*RELEASE_NOTES\.rst$|
- ^contributing-docs/03_contributors_quick_start\.rst$|
- ^.*\.(png|gif|jp[e]?g|svg|tgz|lock)$|
- git|
- ^airflow-core/newsfragments/43349\.significant\.rst$|
- ^airflow-core/newsfragments/41368\.significant\.rst$|
- .*/dist/.*|
.rat-excludes|
- package-lock\.json$|
-
^providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml$
+ ^.*RELEASE_NOTES\.rst$|
+ ^scripts/ci/docker-compose/integration-keycloak\.yml$|
+ ^scripts/ci/docker-compose/keycloak/keycloak-entrypoint\.sh$|
+ ^scripts/ci/prek/vendor_k8s_json_schema\.py$
- id: check-base-operator-partial-arguments
name: Check BaseOperator and partial() arguments
language: python
@@ -810,27 +680,6 @@ repos:
^airflow-core/src/airflow/operators/.*$|
^providers/.*/src/airflow/providers/.*$|
^providers/.*/src/airflow/providers/standard/sensors/.*$
- - id: check-base-operator-usage
- language: pygrep
- name: Check BaseOperator other imports
- description: Make sure BaseOperator is imported from airflow.models
outside of core
- entry: "from airflow\\.models\\.baseoperator import.* BaseOperator"
- pass_filenames: true
- files: >
- (?x)
- ^providers/.*/src/airflow/providers/.*\.py$
- exclude: ^providers/standard/.*/.*\.py$
- - id: check-get-lineage-collector-providers
- language: python
- name: Check providers import hook lineage code from compat
- description: Make sure you import from
airflow.provider.common.compat.lineage.hook instead of
- airflow.lineage.hook.
- entry: ./scripts/ci/prek/check_airflow_imports.py
- --pattern '^airflow\.lineage\.hook'
- --message "Only TYPE_CHECKING imports from `airflow.lineage.hook`
are allowed in providers."
- --only_top_level
- files: ^providers/.*/src/airflow/providers/.*\.py$
- exclude:
^providers/common/compat/src/airflow/providers/common/compat/.*\.py$
- id: check-decorated-operator-implements-custom-name
name: Check @task decorator implements custom_operator_name
language: python
@@ -869,13 +718,6 @@ repos:
language: python
files: ^LICENSE$
pass_filenames: false
- - id: check-aiobotocore-optional
- name: Check if aiobotocore is an optional dependency only
- entry: ./scripts/ci/prek/check_aiobotocore_optional.py
- language: python
- files: ^providers/.*/provider\.yaml$
- pass_filenames: true
- require_serial: true
- id: check-boring-cyborg-configuration
name: Checks for Boring Cyborg configuration consistency
language: python
@@ -938,20 +780,6 @@ repos:
entry: ./scripts/ci/prek/compile_ui_assets.py
pass_filenames: false
additional_dependencies: ['[email protected]']
- - id: compile-fab-assets
- name: Compile FAB provider assets
- language: node
- files: ^providers/fab/.*/www/
- entry: ./scripts/ci/prek/compile_provider_assets.py fab
- pass_filenames: false
- additional_dependencies: ['[email protected]']
- - id: compile-edge-assets
- name: Compile Edge provider assets
- language: node
- files: ^providers/edge3/.*/www/
- entry: ./scripts/ci/prek/compile_provider_assets.py edge
- pass_filenames: false
- additional_dependencies: ['[email protected]']
- id: compile-ui-assets-dev
name: Compile ui assets in dev mode (manual)
language: node
@@ -961,13 +789,6 @@ repos:
entry: ./scripts/ci/prek/compile_ui_assets_dev.py
pass_filenames: false
additional_dependencies: ['[email protected]']
- - id: check-providers-subpackages-init-file-exist
- name: Provider subpackage init files are there
- pass_filenames: false
- always_run: true
- entry: ./scripts/ci/prek/check_providers_subpackages_all_have_init.py
- language: python
- require_serial: true
- id: check-integrations-list-consistent
name: Sync integrations list with docs
entry: ./scripts/ci/prek/check_integrations_list.py
@@ -1193,12 +1014,6 @@ repos:
pass_filenames: true
files:
^airflow-core/docs/.*example-dags\.rst$|^docs/.*index\.rst$^airflow-core/docs/.*index\.rst$
always_run: true
- - id: check-system-tests-tocs
- name: Check that system tests is properly added
- entry: ./scripts/ci/prek/check_system_tests_hidden_in_index.py
- language: python
- pass_filenames: true
- files: ^providers/.*/docs/index\.rst$
- id: check-lazy-logging
name: Check that all logging methods are lazy
entry: ./scripts/ci/prek/check_lazy_logging.py
@@ -1218,13 +1033,6 @@ repos:
language: python
pass_filenames: true
files: ^airflow-core/tests/.*\.py$
- - id: check-provider-docs-valid
- name: Validate provider doc files
- entry: ./scripts/ci/prek/check_provider_docs.py
- language: python
- files: ^providers/.*/provider\.yaml$|^.*/docs/.*
- require_serial: true
- pass_filenames: false
- id: bandit
name: bandit
description: "Bandit is a tool for finding common security issues in
Python code"
@@ -1260,7 +1068,7 @@ repos:
# W0133: "Exception statement has no effect"
# see: https://github.com/astral-sh/ruff/issues/10145
- "--enable=W0133"
- - id: check-fab-migrations
+ - id: check-no-fab-migrations
language: pygrep
name: Check no migration is done on FAB related table
description: >
@@ -1318,22 +1126,6 @@ repos:
additional_dependencies: ['[email protected]']
pass_filenames: true
require_serial: true
- - id: ts-compile-lint-edge-ui
- name: Compile / format / lint edge UI
- description: TS types generation / ESLint / Prettier new UI files in
Edge Provider
- language: node
- files: |
- (?x)
-
^providers/edge3/src/airflow/providers/edge3/plugins/www/.*\.(js|ts|tsx|yaml|css|json)$|
-
^providers/edge3/src/airflow/providers/edge3/openapi/v2-edge-generated.yaml$
- exclude: |
- (?x)
-
^providers/edge3/src/airflow/providers/edge3/plugins/www/node-modules/.*|
- ^providers/edge3/src/airflow/providers/edge3/plugins/www/.pnpm-store
- entry: ./scripts/ci/prek/ts_compile_lint_edge.py
- additional_dependencies: ['[email protected]']
- pass_filenames: true
- require_serial: true
## ADD MOST PREK HOOK ABOVE THAT LINE
# The below prek hooks are those requiring CI image to be built
- id: mypy-dev
@@ -1366,21 +1158,6 @@ repos:
pass_filenames: false
files: ^airflow-core/.*\.py$
require_serial: true
- - id: mypy-providers
- stages: ['pre-push']
- name: Run mypy for providers
- language: python
- entry: ./scripts/ci/prek/mypy.py
- files: ^providers/.*\.py$
- require_serial: true
- - id: mypy-providers
- stages: ['manual']
- name: Run mypy for providers (manual)
- language: python
- entry: ./scripts/ci/prek/mypy_folder.py providers
- pass_filenames: false
- files: ^.*\.py$
- require_serial: true
- id: mypy-task-sdk
stages: ['pre-push']
name: Run mypy for task-sdk
@@ -1432,26 +1209,8 @@ repos:
language: python
entry: ./scripts/ci/prek/generate_openapi_spec.py
pass_filenames: false
- files:
^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$|^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$
+ files:
^airflow-core/src/airflow/api_fastapi/.*\.py$|^airflow-core/src/airflow/api_fastapi/auth/managers/simple/.*\.py$
exclude: ^airflow-core/src/airflow/api_fastapi/execution_api/.*
- - id: generate-openapi-spec-fab
- name: Generate the FastAPI API spec for FAB
- language: python
- entry: ./scripts/ci/prek/generate_openapi_spec_providers.py fab
- pass_filenames: false
- files:
^providers/fab/src/airflow/providers/fab/auth_manager/api_fastapi/.*\.py$
- - id: generate-openapi-spec-edge
- name: Generate the FastAPI API spec for Edge
- language: python
- entry: ./scripts/ci/prek/generate_openapi_spec_providers.py edge
- pass_filenames: false
- files: ^providers/edge3/src/airflow/providers/edge3/worker_api/.*\.py$
- - id: generate-openapi-spec-keycloak
- name: Generate the FastAPI API spec for Keycloak
- language: python
- entry: ./scripts/ci/prek/generate_openapi_spec_providers.py keycloak
- pass_filenames: false
- files:
^providers/keycloak/src/airflow/providers/keycloak/auth_manager/.*\.py$
- id: check-i18n-json
name: Check i18n files validity
description: Check i18n files are valid json, have no TODOs, and
auto-format them
@@ -1459,13 +1218,6 @@ repos:
files: ^airflow-core/src/airflow/ui/public/i18n/locales/.*\.json$
entry: ./scripts/ci/prek/check_i18n_json.py
pass_filenames: false
- - id: check-provider-yaml-valid
- name: Validate provider.yaml files
- entry: ./scripts/ci/prek/check_provider_yaml_files.py
- language: python
- files: ^providers/.*/provider\.yaml$
- exclude: ^providers/.*/.venv/.*$
- require_serial: true
- id: check-template-fields-valid
name: Check templated fields mapped in operators/sensors
language: python
@@ -1515,19 +1267,6 @@ repos:
pass_filenames: false
files:
^airflow-ctl/src/airflowctl/ctl/cli_config.py$|airflow-ctl/src/airflowctl/api/operations.py|airflow-ctl/src/airflowctl/ctl/commands/.*\.py
- - id: check-imports-in-providers
- name: Check imports in providers
- entry: ./scripts/ci/prek/check_imports_in_providers.py
- language: python
- files: ^providers/.*/src/airflow/providers/.*version_compat.*\.py$
- require_serial: true
- - id: provider-version-compat
- name: Check for correct version_compat imports in providers
- entry: ./scripts/ci/prek/check_provider_version_compat.py
- language: python
- types: [python]
- files: ^providers/.*/src/airflow/providers/.*\.py$
- require_serial: true
- id: check-airflow-version-checks-in-core
language: pygrep
name: No AIRFLOW_V_* imports in airflow-core
@@ -1543,7 +1282,6 @@ repos:
^airflow-core/tests/unit/core/test_configuration\.py$|
^airflow-core/tests/unit/models/test_renderedtifields\.py$|
^airflow-core/tests/unit/models/test_variable\.py$
-
- id: check-sdk-imports
name: Check for SDK imports in core files
entry: ./scripts/ci/prek/check_sdk_imports.py
@@ -1580,6 +1318,7 @@ repos:
^airflow-core/src/airflow/dag_processing/collection\.py$|
^airflow-core/src/airflow/dag_processing/manager\.py$|
^airflow-core/src/airflow/dag_processing/processor\.py$|
+ ^airflow-core/src/airflow/dag_processing/dagbag\.py$|
^airflow-core/src/airflow/datasets/metadata\.py$|
^airflow-core/src/airflow/exceptions\.py$|
^airflow-core/src/airflow/executors/local_executor\.py$|
@@ -1591,9 +1330,9 @@ repos:
^airflow-core/src/airflow/models/__init__\.py$|
^airflow-core/src/airflow/models/asset\.py$|
^airflow-core/src/airflow/models/baseoperator\.py$|
+ ^airflow-core/src/airflow/models/callback\.py$|
^airflow-core/src/airflow/models/connection\.py$|
^airflow-core/src/airflow/models/dag\.py$|
- ^airflow-core/src/airflow/models/dagbag\.py$|
^airflow-core/src/airflow/models/dagrun\.py$|
^airflow-core/src/airflow/models/deadline\.py$|
^airflow-core/src/airflow/models/expandinput\.py$|
diff --git a/providers/.pre-commit-config.yaml
b/providers/.pre-commit-config.yaml
new file mode 100644
index 00000000000..cb30c8d7d48
--- /dev/null
+++ b/providers/.pre-commit-config.yaml
@@ -0,0 +1,232 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+---
+default_stages: [pre-commit, pre-push]
+minimum_prek_version: '0.0.28'
+repos:
+ - repo: local
+ hooks:
+ - id: check-deferrable-default
+ name: Check and fix default value of default_deferrable
+ language: python
+ entry: ../scripts/ci/prek/check_deferrable_default.py
+ pass_filenames: false
+ files: ^(.*/)?airflow/.*/(sensors|operators)/.*\.py$
+ - id: update-providers-dependencies
+ name: Update dependencies for providers
+ entry: ../scripts/ci/prek/update_providers_dependencies.py
+ language: python
+ always_run: true
+ pass_filenames: false
+ - id: validate-operators-init
+ name: No templated field logic checks in operator __init__
+ description: Prevent templated field logic checks in operators'
__init__
+ language: python
+ entry: ../scripts/ci/prek/validate_operators_init.py
+ pass_filenames: true
+ files:
^.*/src/airflow/providers/.*/(operators|transfers|sensors)/.*\.py$
+ - id: update-providers-build-files
+ name: Update providers build files
+ entry: ../scripts/ci/prek/update_providers_build_files.py
+ language: python
+ pass_filenames: true
+ files: |
+ (?x)
+ ^[^\/]*/src/airflow/providers/[^\/]*/__init__\.py$|
+ ^[^\/]*/[^\/]*/src/airflow/providers/[^\/]*/[^\/]*/__init__\.py$|
+ ^.*/pyproject\.toml$|
+ ^.*/provider\.yaml$|
+
^../dev/breeze/src/airflow_breeze/templates/PROVIDER__INIT__PY_TEMPLATE\.py\.jinja2$|
+
^../dev/breeze/src/airflow_breeze/templates/get_provider_info_TEMPLATE\.py\.jinja2$|
+
^../dev/breeze/src/airflow_breeze/templates/PROVIDER_README_TEMPLATE\.rst\.jinja2$
+ require_serial: true
+ - id: check-airflow-v-imports-in-tests
+ name: Check AIRFLOW_V imports in tests
+ language: python
+ entry: ../scripts/ci/prek/check_airflow_v_imports_in_tests.py
+ pass_filenames: true
+ files: ^.*/tests/.+\.py$
+ - id: check-sql-dependency-common-data-structure
+ name: Check dependency of SQL providers
+ description: Check dependency of SQL Providers with common data
structure
+ entry: ../scripts/ci/prek/check_common_sql_dependency.py
+ language: python
+ files: ^.*/src/airflow/providers/.*/hooks/.*\.py$
+ - id: generate-volumes-for-sources
+ name: Generate volumes for docker compose
+ entry: ../scripts/ci/prek/generate_volumes_for_sources.py
+ language: python
+ files: ^.*/provider\.yaml$
+ pass_filenames: false
+ require_serial: true
+ - id: check-pytest-mark-db-test-in-providers
+ language: pygrep
+ name: Check pytest.mark.db_test use in providers
+ entry: pytest\.mark\.db_test
+ pass_filenames: true
+ # Here we should add providers that are already free from the
pytest.mark.db_test
+ # and we want to keep them clean and only use non-db-tests
+ files: >
+ (?x)
+ ^airbyte/.*\.py$|
+ ^apache/beam/.*\.py$|
+ ^apache/flink/.*\.py$|
+ ^apache/iceberg/.*\.py$|
+ ^apache/kafka/.*\.py$|
+ ^apprise/.*\.py$|
+ ^arangodb/.*\.py$|
+ ^asana/.*\.py$|
+ ^atlassian/jira/.*\.py$|
+ ^cloudant/.*\.py$|
+ ^cohere/.*\.py$|
+ ^common/compat/.*\.py$|
+ ^common/messaging/.*\.py$|
+ ^datadog/.*\.py$|
+ ^dingding/.*\.py$|
+ ^discord/.*\.py$|
+ ^exasol/.*\.py$|
+ ^facebook/.*\.py$|
+ ^ftp/.*\.py$|
+ ^grpc/.*\.py$|
+ ^hashicorp/.*\.py$|
+ ^imap/.*\.py$|
+ ^influxdb/.*\.py$|
+ ^jdbc/.*\.py$|
+ ^jenkins/.*\.py$|
+ ^mongo/.*\.py$|
+ ^microsoft/psrp/.*\.py$|
+ ^microsoft/winrm/.*\.py$|
+ ^neo4j/.*\.py$|
+ ^odbc/.*\.py$|
+ ^openai/.*\.py$|
+ ^openfaas/.*\.py$|
+ ^opsgenie/.*\.py$|
+ ^oracle/.*\.py$|
+ ^pagerduty/.*\.py$|
+ ^pgvector/.*\.py$|
+ ^pinecone/.*\.py$|
+ ^postgres/.*\.py$|
+ ^presto/.*\.py$|
+ ^segment/.*\.py$|
+ ^sendgrid/.*\.py$|
+ ^singularity/.*\.py$|
+ ^slack/.*\.py$|
+ ^smtp/.*\.py$|
+ ^tableau/.*\.py$|
+ ^teradata/.*\.py$|
+ ^trino/.*\.py$|
+ ^vertica/.*\.py$|
+ ^yandex/.*\.py$|
+ ^zendesk/.*\.py$
+ - id: check-links-to-example-dags-do-not-use-hardcoded-versions
+ name: Verify no hard-coded version in example dags
+ description: The links to example dags should use |version| as version
specification
+ language: pygrep
+ entry: >
+ (?i)
+
.*https://github.*/main/providers/.*/src/airflow/providers/.*/example_dags/|
+
.*https://github.*/master/providers/.*/src/airflow/providers/.*/example_dags/
+ pass_filenames: true
+ files: ^.*/docs/.*\.rst
+ - id: check-no-airflow-deprecation-in-providers
+ language: pygrep
+ name: Do not use DeprecationWarning in providers
+ description: Use AirflowProviderDeprecationWarning in providers
+ entry: "^\\s*DeprecationWarning*"
+ pass_filenames: true
+ files: ^.*/src/airflow/providers/.*\.py$
+ - id: check-base-operator-usage
+ language: pygrep
+ name: Check BaseOperator other imports
+ description: Make sure BaseOperator is imported from airflow.models
outside of core
+ entry: "from airflow\\.models\\.baseoperator import.* BaseOperator"
+ pass_filenames: true
+ files: ^.*/src/airflow/providers/.*\.py$
+ exclude: ^standard/.*/.*\.py$
+ - id: check-get-lineage-collector-providers
+ language: python
+ name: Check providers import hook lineage code from compat
+ description: Make sure you import from
airflow.provider.common.compat.lineage.hook instead of
+ airflow.lineage.hook.
+ entry: ../scripts/ci/prek/check_airflow_imports.py
+ --pattern '^airflow\.lineage\.hook'
+ --message "Only TYPE_CHECKING imports from `airflow.lineage.hook`
are allowed in providers."
+ --only_top_level
+ files: ^.*/src/airflow/providers/.*\.py$
+ exclude: ^common/compat/src/airflow/providers/common/compat/.*\.py$
+ - id: check-aiobotocore-optional
+ name: Check if aiobotocore is an optional dependency only
+ entry: ../scripts/ci/prek/check_aiobotocore_optional.py
+ language: python
+ files: ^.*/provider\.yaml$
+ pass_filenames: true
+ require_serial: true
+ - id: check-providers-subpackages-init-file-exist
+ name: Provider subpackage init files are there
+ pass_filenames: false
+ always_run: true
+ entry: ../scripts/ci/prek/check_providers_subpackages_all_have_init.py
+ language: python
+ require_serial: true
+ - id: check-system-tests-tocs
+ name: Check that system tests is properly added
+ entry: ../scripts/ci/prek/check_system_tests_hidden_in_index.py
+ language: python
+ pass_filenames: true
+ files: ^.*/docs/index\.rst$
+ - id: check-provider-docs-valid
+ name: Validate provider doc files
+ entry: ../scripts/ci/prek/check_provider_docs.py
+ language: python
+ files: ^.*/provider\.yaml$|^.*/docs/.*
+ require_serial: true
+ pass_filenames: false
+ - id: mypy-providers
+ stages: ['pre-push']
+ name: Run mypy for providers
+ language: python
+ entry: ../scripts/ci/prek/mypy.py
+ files: ^.*\.py$
+ require_serial: true
+ - id: mypy-providers
+ stages: ['manual']
+ name: Run mypy for providers (manual)
+ language: python
+ entry: ../scripts/ci/prek/mypy_folder.py providers
+ pass_filenames: false
+ files: ^.*\.py$
+ require_serial: true
+ - id: check-provider-yaml-valid
+ name: Validate provider.yaml files
+ entry: ../scripts/ci/prek/check_provider_yaml_files.py
+ language: python
+ files: ^.*/provider\.yaml$
+ exclude: ^.*/.venv/.*$
+ require_serial: true
+ - id: check-imports-in-providers
+ name: Check imports in providers
+ entry: ../scripts/ci/prek/check_imports_in_providers.py
+ language: python
+ files: ^.*/src/airflow/providers/.*version_compat.*\.py$
+ require_serial: true
+ - id: provider-version-compat
+ name: Check for correct version_compat imports in providers
+ entry: ../scripts/ci/prek/check_provider_version_compat.py
+ language: python
+ types: [python]
+ files: ^.*/src/airflow/providers/.*\.py$
+ require_serial: true
diff --git a/scripts/ci/prek/mypy.py b/scripts/ci/prek/mypy.py
index 24cec5afa57..d92ffd85e31 100755
--- a/scripts/ci/prek/mypy.py
+++ b/scripts/ci/prek/mypy.py
@@ -46,7 +46,12 @@ if not files_to_test:
# TODO(potiuk): add suspended providers exclusion
-cmd = ["bash", "-c", f"TERM=ansi mypy {' '.join([shlex.quote(file) for file in
files_to_test])}"]
+repo_root = Path(__file__).parents[3].resolve()
+cmd = [
+ "bash",
+ "-c",
+ f"TERM=ansi mypy {'
'.join([shlex.quote(str(Path(file).absolute().relative_to(repo_root))) for file
in files_to_test])}",
+]
res = run_command_via_breeze_shell(
cmd=cmd,
diff --git a/scripts/ci/prek/update_providers_build_files.py
b/scripts/ci/prek/update_providers_build_files.py
index e612a7545d4..00b26c03cdc 100755
--- a/scripts/ci/prek/update_providers_build_files.py
+++ b/scripts/ci/prek/update_providers_build_files.py
@@ -40,7 +40,7 @@ console.print(f"[bright_blue]Determining providers to
regenerate from: {file_lis
def _find_all_providers(examined_file: Path) -> None:
- console.print(f"[bright_blue]Looking at {examined_file} for new structure
provider.yaml")
+ console.print(f"[bright_blue]Looking at {examined_file} for structure
provider.yaml")
# find the folder where provider.yaml is
for parent in Path(examined_file).parents:
console.print(f"[bright_blue]Checking {parent} for provider.yaml")
@@ -58,7 +58,7 @@ def _find_all_providers(examined_file: Path) -> None:
console.print(f"[bright_blue]Found base folder {base_folder}")
break
else:
- console.print(f"[red]\nCould not find new structure base folder for
{provider_folder}")
+ console.print(f"[red]\nCould not find structure base folder for
{provider_folder}")
sys.exit(1)
provider_name =
".".join(provider_folder.relative_to(base_folder).as_posix().split("/"))
providers.add(provider_name)
@@ -66,7 +66,7 @@ def _find_all_providers(examined_file: Path) -> None:
# get all folders from arguments
for examined_file in file_list:
- _find_all_providers(Path(examined_file))
+ _find_all_providers(Path(examined_file).absolute())
console.print(f"[bright_blue]Regenerating build files for providers:
{providers}[/]")
