This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 564adcfb7a Deprecate PY* constants into the airflow module (#37575)
564adcfb7a is described below
commit 564adcfb7a41c4ed20a63f39f132194465400245
Author: Andrey Anshin <andrey.ans...@taragol.is>
AuthorDate: Wed Feb 21 23:24:07 2024 +0400
Deprecate PY* constants into the airflow module (#37575)
---
airflow/__init__.py | 19 ++++----
airflow/utils/hashlib_wrapper.py | 5 +-
pyproject.toml | 9 ++++
tests/core/test_airflow_module.py | 54 ++++++++++++++++++++++
tests/dag_processing/test_processor.py | 4 +-
tests/decorators/test_python.py | 3 +-
tests/operators/test_python.py | 2 +-
tests/providers/apache/hive/hooks/test_hive.py | 11 +----
.../serialization/serializers/test_serializers.py | 4 +-
9 files changed, 85 insertions(+), 26 deletions(-)
diff --git a/airflow/__init__.py b/airflow/__init__.py
index 3d9ea828e8..6464f0bc6e 100644
--- a/airflow/__init__.py
+++ b/airflow/__init__.py
@@ -39,7 +39,7 @@ if os.environ.get("_AIRFLOW_PATCH_GEVENT"):
# configuration is therefore initted early here, simply by importing it.
from airflow import configuration, settings
-__all__ = ["__version__", "DAG", "PY36", "PY37", "PY38", "PY39", "PY310",
"PY311", "XComArg"]
+__all__ = ["__version__", "DAG", "Dataset", "XComArg"]
# Make `airflow` a namespace package, supporting installing
# airflow.providers.* in different locations (i.e. one in site, and one in user
@@ -55,13 +55,6 @@ __path__ = __import__("pkgutil").extend_path(__path__,
__name__) # type: ignore
if not os.environ.get("_AIRFLOW__AS_LIBRARY", None):
settings.initialize()
-PY36 = sys.version_info >= (3, 6)
-PY37 = sys.version_info >= (3, 7)
-PY38 = sys.version_info >= (3, 8)
-PY39 = sys.version_info >= (3, 9)
-PY310 = sys.version_info >= (3, 10)
-PY311 = sys.version_info >= (3, 11)
-
# Things to lazy import in form {local_name: ('target_module', 'target_name',
'deprecated')}
__lazy_imports: dict[str, tuple[str, str, bool]] = {
"DAG": (".models.dag", "DAG", False),
@@ -77,6 +70,16 @@ def __getattr__(name: str):
# PEP-562: Lazy loaded attributes on python modules
module_path, attr_name, deprecated = __lazy_imports.get(name, ("", "",
False))
if not module_path:
+ if name.startswith("PY3") and (py_minor := name[3:]) in ("6", "7",
"8", "9", "10", "11", "12"):
+ import warnings
+
+ warnings.warn(
+ f"Python version constraint {name!r} is deprecated and will be
removed in the future. "
+ f"Please get version info from the 'sys.version_info'.",
+ DeprecationWarning,
+ )
+ return sys.version_info >= (3, int(py_minor))
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
elif deprecated:
import warnings
diff --git a/airflow/utils/hashlib_wrapper.py b/airflow/utils/hashlib_wrapper.py
index 1390ada334..f48a093bb2 100644
--- a/airflow/utils/hashlib_wrapper.py
+++ b/airflow/utils/hashlib_wrapper.py
@@ -18,13 +18,12 @@
from __future__ import annotations
import hashlib
+import sys
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from _typeshed import ReadableBuffer
-from airflow import PY39
-
def md5(__string: ReadableBuffer = b"") -> hashlib._Hash:
"""
@@ -33,6 +32,6 @@ def md5(__string: ReadableBuffer = b"") -> hashlib._Hash:
:param __string: The data to hash. Default to empty str byte.
:return: The hashed value.
"""
- if PY39:
+ if sys.version_info >= (3, 9):
return hashlib.md5(__string, usedforsecurity=False) # type: ignore
return hashlib.md5(__string)
diff --git a/pyproject.toml b/pyproject.toml
index 54041c3d90..7dfeb83389 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1462,8 +1462,17 @@ combine-as-imports = true
banned-module-level-imports = ["numpy", "pandas"]
[tool.ruff.lint.flake8-tidy-imports.banned-api]
+# Direct import from the airflow package modules and constraints
"airflow.AirflowException".msg = "Use airflow.exceptions.AirflowException
instead."
"airflow.Dataset".msg = "Use airflow.datasets.Dataset instead."
+"airflow.PY36".msg = "Use sys.version_info >= (3, 6) instead."
+"airflow.PY37".msg = "Use sys.version_info >= (3, 7) instead."
+"airflow.PY38".msg = "Use sys.version_info >= (3, 8) instead."
+"airflow.PY39".msg = "Use sys.version_info >= (3, 9) instead."
+"airflow.PY310".msg = "Use sys.version_info >= (3, 10) instead."
+"airflow.PY311".msg = "Use sys.version_info >= (3, 11) instead."
+"airflow.PY312".msg = "Use sys.version_info >= (3, 12) instead."
+# Deprecated imports
"airflow.models.baseoperator.BaseOperatorLink".msg = "Use
airflow.models.baseoperatorlink.BaseOperatorLink"
# Deprecated in Python 3.11, Pending Removal in Python 3.15:
https://github.com/python/cpython/issues/90817
# Deprecation warning in Python 3.11 also recommends using locale.getencoding
but it available in Python 3.11
diff --git a/tests/core/test_airflow_module.py
b/tests/core/test_airflow_module.py
new file mode 100644
index 0000000000..d448f4bc93
--- /dev/null
+++ b/tests/core/test_airflow_module.py
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from __future__ import annotations
+
+import sys
+
+import pytest
+
+import airflow
+from airflow.exceptions import AirflowException
+
+TEST_CASES = {
+ "PY36": sys.version_info >= (3, 6),
+ "PY37": sys.version_info >= (3, 7),
+ "PY38": sys.version_info >= (3, 8),
+ "PY39": sys.version_info >= (3, 9),
+ "PY310": sys.version_info >= (3, 10),
+ "PY311": sys.version_info >= (3, 11),
+ "PY312": sys.version_info >= (3, 12),
+}
+
+
+@pytest.mark.parametrize("py_attr, expected", TEST_CASES.items())
+def test_lazy_load_py_versions(py_attr, expected):
+ with pytest.warns(DeprecationWarning, match=f"Python version constraint
'{py_attr}' is deprecated"):
+ # If there is no warning, then most possible it imported somewhere
else.
+ assert getattr(airflow, py_attr) is expected
+
+
+@pytest.mark.parametrize("py_attr", ["PY35", "PY313"])
+def test_wrong_py_version(py_attr):
+ with pytest.raises(AttributeError, match=f"'airflow' has no attribute
'{py_attr}'"):
+ getattr(airflow, py_attr)
+
+
+def test_deprecated_exception():
+ warning_pattern = "Import 'AirflowException' directly from the airflow
module is deprecated"
+ with pytest.warns(DeprecationWarning, match=warning_pattern):
+ # If there is no warning, then most possible it imported somewhere
else.
+ assert getattr(airflow, "AirflowException") is AirflowException
diff --git a/tests/dag_processing/test_processor.py
b/tests/dag_processing/test_processor.py
index ac94edc510..6a3c51fd10 100644
--- a/tests/dag_processing/test_processor.py
+++ b/tests/dag_processing/test_processor.py
@@ -19,13 +19,14 @@ from __future__ import annotations
import datetime
import os
+import sys
from unittest import mock
from unittest.mock import MagicMock, patch
from zipfile import ZipFile
import pytest
-from airflow import PY311, settings
+from airflow import settings
from airflow.callbacks.callback_requests import TaskCallbackRequest
from airflow.configuration import TEST_DAGS_FOLDER, conf
from airflow.dag_processing.manager import DagFileProcessorAgent
@@ -53,6 +54,7 @@ from tests.test_utils.mock_executor import MockExecutor
pytestmark = pytest.mark.db_test
DEFAULT_DATE = timezone.datetime(2016, 1, 1)
+PY311 = sys.version_info >= (3, 11)
# Include the words "airflow" and "dag" in the file contents,
# tricking airflow into thinking these
diff --git a/tests/decorators/test_python.py b/tests/decorators/test_python.py
index 69b52d7724..f78d71e05f 100644
--- a/tests/decorators/test_python.py
+++ b/tests/decorators/test_python.py
@@ -22,7 +22,6 @@ from typing import TYPE_CHECKING, Dict, Tuple, Union
import pytest
-from airflow import PY38, PY311
from airflow.decorators import setup, task as task_decorator, teardown
from airflow.decorators.base import DecoratedMappedOperator
from airflow.exceptions import AirflowException, XComNotFound
@@ -49,6 +48,8 @@ if TYPE_CHECKING:
from airflow.models.dagrun import DagRun
DEFAULT_DATE = timezone.datetime(2016, 1, 1)
+PY38 = sys.version_info >= (3, 8)
+PY311 = sys.version_info >= (3, 11)
class TestAirflowTaskDecorator(BasePythonTest):
diff --git a/tests/operators/test_python.py b/tests/operators/test_python.py
index f1a297aab3..a4de8510a1 100644
--- a/tests/operators/test_python.py
+++ b/tests/operators/test_python.py
@@ -37,7 +37,6 @@ from unittest.mock import MagicMock
import pytest
from slugify import slugify
-from airflow import PY311
from airflow.decorators import task_group
from airflow.exceptions import AirflowException, DeserializingResultError,
RemovedInAirflow3Warning
from airflow.models.baseoperator import BaseOperator
@@ -75,6 +74,7 @@ DEFAULT_DATE = timezone.datetime(2016, 1, 1)
TEMPLATE_SEARCHPATH = os.path.join(AIRFLOW_MAIN_FOLDER, "tests",
"config_templates")
LOGGER_NAME = "airflow.task.operators"
DEFAULT_PYTHON_VERSION = f"{sys.version_info[0]}.{sys.version_info[1]}"
+PY311 = sys.version_info >= (3, 11)
class BasePythonTest:
diff --git a/tests/providers/apache/hive/hooks/test_hive.py
b/tests/providers/apache/hive/hooks/test_hive.py
index a137364767..027ad8c688 100644
--- a/tests/providers/apache/hive/hooks/test_hive.py
+++ b/tests/providers/apache/hive/hooks/test_hive.py
@@ -17,22 +17,13 @@
# under the License.
from __future__ import annotations
-import pytest
-
-from airflow import PY311
-
-if PY311:
- pytest.skip(
- "The tests are skipped because Apache Hive provider is not supported
on Python 3.11",
- allow_module_level=True,
- )
-
import datetime
import itertools
from collections import namedtuple
from unittest import mock
import pandas as pd
+import pytest
from hmsclient import HMSClient
from airflow.exceptions import AirflowException
diff --git a/tests/serialization/serializers/test_serializers.py
b/tests/serialization/serializers/test_serializers.py
index cb0b03b324..1f64f90baf 100644
--- a/tests/serialization/serializers/test_serializers.py
+++ b/tests/serialization/serializers/test_serializers.py
@@ -18,6 +18,7 @@ from __future__ import annotations
import datetime
import decimal
+import sys
from importlib import metadata
from unittest.mock import patch
@@ -31,11 +32,10 @@ from packaging import version
from pendulum import DateTime
from pendulum.tz.timezone import FixedTimezone, Timezone
-from airflow import PY39
from airflow.models.param import Param, ParamsDict
from airflow.serialization.serde import DATA, deserialize, serialize
-if PY39:
+if sys.version_info >= (3, 9):
from zoneinfo import ZoneInfo
else:
from backports.zoneinfo import ZoneInfo
