This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 578cbb24fb Skip python version check in some pre-commit when loading
setup.py (#31965)
578cbb24fb is described below
commit 578cbb24fbd7e4bbeef2e61078c5d333aeda0448
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Fri Jun 16 19:24:52 2023 +0200
Skip python version check in some pre-commit when loading setup.py (#31965)
Some providers are excluded for some versions of Python (for example
the `apache.hive` provider is excluded on Python 3.11. However
this might make some pre-commits confused when running on Python 3.11
because the list of generated extras will be different for them
than those on other Python versions.
We do not want to set python used in pre-commits to, say, python3.8,
because that would mean that would require our users to have python3.8
as binary available on their system, which is not always setup. Python3
on the other hand is going to be universally available.
In order to mitigate the problem we will run the precommits with
_SKIP_PYTHON_VERSION_CHECK variable set before import of setup is done,
so that the python version check is skipped and all python versions
will produce the same set of extras.
---
scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py | 2 ++
scripts/ci/pre_commit/pre_commit_insert_extras.py | 3 +++
setup.py | 6 +++++-
3 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
index a384c958de..a523ac62c0 100755
--- a/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
+++ b/scripts/ci/pre_commit/pre_commit_check_setup_extra_packages_ref.py
@@ -37,6 +37,8 @@ PY_IDENTIFIER = r"[a-zA-Z_][a-zA-Z0-9_\.]*"
sys.path.insert(0, AIRFLOW_SOURCES_DIR)
+os.environ["_SKIP_PYTHON_VERSION_CHECK"] = "true"
+
from setup import ( # noqa # isort:skip
add_all_provider_packages,
EXTRAS_DEPRECATED_ALIASES,
diff --git a/scripts/ci/pre_commit/pre_commit_insert_extras.py
b/scripts/ci/pre_commit/pre_commit_insert_extras.py
index fac926f611..2750e9dbae 100755
--- a/scripts/ci/pre_commit/pre_commit_insert_extras.py
+++ b/scripts/ci/pre_commit/pre_commit_insert_extras.py
@@ -17,6 +17,7 @@
# under the License.
from __future__ import annotations
+import os
import sys
from pathlib import Path
from textwrap import wrap
@@ -27,6 +28,8 @@ sys.path.insert(0, str(Path(__file__).parent.resolve())) #
make sure common_pre
sys.path.insert(0, str(AIRFLOW_SOURCES_DIR)) # make sure setup is imported
from Airflow
# flake8: noqa: F401
+os.environ["_SKIP_PYTHON_VERSION_CHECK"] = "true"
+
from common_precommit_utils import insert_documentation # isort: skip
from setup import EXTRAS_DEPENDENCIES # isort:skip
diff --git a/setup.py b/setup.py
index 6c7ae2ec22..547e1221d3 100644
--- a/setup.py
+++ b/setup.py
@@ -67,13 +67,17 @@ CURRENT_PYTHON_VERSION =
f"{sys.version_info.major}.{sys.version_info.minor}"
# corresponding provider.yaml file.
#
def fill_provider_dependencies() -> dict[str, dict[str, list[str]]]:
+ # in case we are loading setup from pre-commits, we want to skip the check
for python version
+ # because if someone uses a version of Python where providers are
excluded, the setup will fail
+ # to see the extras for those providers
+ skip_python_version_check = os.environ.get("_SKIP_PYTHON_VERSION_CHECK")
try:
with AIRFLOW_SOURCES_ROOT.joinpath("generated",
"provider_dependencies.json").open() as f:
dependencies = json.load(f)
return {
key: value
for key, value in dependencies.items()
- if CURRENT_PYTHON_VERSION not in value["excluded-python-versions"]
+ if CURRENT_PYTHON_VERSION not in value["excluded-python-versions"]
or skip_python_version_check
}
except Exception as e:
print(f"Exception while loading provider dependencies {e}")
