This is an automated email from the ASF dual-hosted git repository.

Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 41619107e37 Fix import airflow slowdown from reading every package's 
metadata (#76890)
41619107e37 is described below

commit 41619107e376d9307b84f4dd9400ff101cd0702b
Author: Andrew Chang <[email protected]>
AuthorDate: Tue Nov 17 12:29:16 2026 +0800

    Fix import airflow slowdown from reading every package's metadata (#76890)
---
 .../src/airflow_shared/module_loading/__init__.py  | 21 +++++-----
 .../tests/module_loading/test_entry_points.py      | 45 ++++++++++++++++++----
 .../tests/plugins_manager/test_plugins_manager.py  | 10 ++++-
 3 files changed, 57 insertions(+), 19 deletions(-)

diff --git 
a/shared/module_loading/src/airflow_shared/module_loading/__init__.py 
b/shared/module_loading/src/airflow_shared/module_loading/__init__.py
index adfbae8acf4..3a65d9fc250 100644
--- a/shared/module_loading/src/airflow_shared/module_loading/__init__.py
+++ b/shared/module_loading/src/airflow_shared/module_loading/__init__.py
@@ -164,11 +164,6 @@ def is_valid_dotpath(path: str) -> bool:
     return bool(re.match(pattern, path))
 
 
-def _normalize_dist_name(name: str) -> str:
-    # PEP 503 normalization, the same key importlib.metadata.entry_points() 
deduplicates on.
-    return re.sub(r"[-_.]+", "-", name).lower()
-
-
 @functools.cache
 def _get_grouped_entry_points() -> dict[str, list[EPnD]]:
     mapping: dict[str, list[EPnD]] = defaultdict(list)
@@ -178,12 +173,18 @@ def _get_grouped_entry_points() -> dict[str, list[EPnD]]:
             # A distribution is yielded once per sys.path entry it is found on 
(e.g. RHEL/Fedora
             # venvs list both lib64/ and lib/ site-packages), possibly in 
different versions.
             # Only the first one is importable, so it is the only one whose 
entry points count.
-            dist_metadata = dist.metadata
-            if dist_metadata and (name := dist_metadata.get("Name")):
-                normalized_name = _normalize_dist_name(name)
-                if normalized_name in seen_dist_names:
+            # No public API returns the name without parsing METADATA. 
_normalized_name is private,
+            # but Python's own importlib.metadata.entry_points() deduplicates 
on it too, and it comes
+            # from the .dist-info directory name. If it cannot be read, the 
distribution is still
+            # loaded, only without deduplication. The exception type differs 
between versions.
+            try:
+                name = dist._normalized_name  # type: ignore[attr-defined]
+            except Exception:
+                name = None
+            if name:
+                if name in seen_dist_names:
                     continue
-                seen_dist_names.add(normalized_name)
+                seen_dist_names.add(name)
             for e in dist.entry_points:
                 # importlib_metadata >= 10 types this as its IDistribution 
protocol, which older
                 # releases and the stdlib lack; a cast would be redundant on 
those.
diff --git a/shared/module_loading/tests/module_loading/test_entry_points.py 
b/shared/module_loading/tests/module_loading/test_entry_points.py
index ff22042aefe..a9a65798bd7 100644
--- a/shared/module_loading/tests/module_loading/test_entry_points.py
+++ b/shared/module_loading/tests/module_loading/test_entry_points.py
@@ -17,6 +17,7 @@
 # under the License.
 from __future__ import annotations
 
+import re
 from collections.abc import Iterable
 from unittest import mock
 
@@ -29,7 +30,8 @@ class MockDistribution:
     def __init__(
         self, name: str, entry_points: Iterable[metadata.EntryPoint], version: 
str = "1.0.0"
     ) -> None:
-        self.metadata: dict[str, str] | None = {"Name": name}
+        self.metadata = {"Name": name}
+        self._normalized_name = re.sub(r"[-_.]+", "_", name).lower()
         self.entry_points = entry_points
         self.version = version
 
@@ -97,17 +99,46 @@ def 
test_entry_points_with_dist_same_dist_in_two_versions_returns_first_only():
     ]
 
 
-# No metadata file at all, or metadata without a Name field (the key is 
absent, not None).
[email protected]("dist_metadata", [None, {"Version": "1.0.0"}], 
ids=["no-metadata", "no-name"])
-def 
test_entry_points_with_dist_without_name_are_not_deduplicated(dist_metadata):
+def test_entry_points_with_dist_deduplicates_without_reading_metadata():
+    distributions = [
+        MockDistribution("apache-airflow-providers-foo", 
[metadata.EntryPoint("a", "b", "group_x")]),
+        MockDistribution("apache_airflow_providers_foo", 
[metadata.EntryPoint("c", "d", "group_x")]),
+        MockDistribution("apache-airflow-providers-bar", 
[metadata.EntryPoint("e", "f", "group_x")]),
+    ]
+
+    with (
+        mock.patch("airflow_shared.module_loading.metadata", 
MockMetadata(distributions)),
+        mock.patch.object(
+            MockDistribution, "metadata", create=True, 
new_callable=mock.PropertyMock
+        ) as metadata_property,
+    ):
+        entries = list(entry_points_with_dist("group_x"))
+
+    assert [ep.name for ep, _ in entries] == ["a", "e"]
+    metadata_property.assert_not_called()
+
+
[email protected](
+    "error",
+    [TypeError, KeyError, FileNotFoundError, AttributeError],
+    ids=["stdlib", "importlib-metadata-8", "importlib-metadata-9-no-file", 
"no-attribute"],
+)
+def test_entry_points_with_dist_without_name_are_not_deduplicated(error):
     distributions = [
         MockDistribution("unnamed", [metadata.EntryPoint("a", "b", 
"group_x")]),
         MockDistribution("unnamed", [metadata.EntryPoint("c", "d", 
"group_x")]),
     ]
-    for dist in distributions:
-        dist.metadata = dist_metadata
 
-    with mock.patch("airflow_shared.module_loading.metadata", 
MockMetadata(distributions)):
+    with (
+        mock.patch("airflow_shared.module_loading.metadata", 
MockMetadata(distributions)),
+        mock.patch.object(
+            MockDistribution,
+            "_normalized_name",
+            create=True,
+            new_callable=mock.PropertyMock,
+            side_effect=error,
+        ),
+    ):
         entries = list(entry_points_with_dist("group_x"))
 
     assert [ep.name for ep, _ in entries] == ["a", "c"]
diff --git 
a/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py 
b/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py
index 414259908ae..16e89829eaf 100644
--- a/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py
+++ b/shared/plugins_manager/tests/plugins_manager/test_plugins_manager.py
@@ -24,6 +24,7 @@ from unittest import mock
 
 import pytest
 
+from airflow_shared.module_loading import _get_grouped_entry_points
 from airflow_shared.plugins_manager import (
     AirflowPlugin,
     EntryPointSource,
@@ -41,8 +42,12 @@ def mock_metadata_distribution(mocker):
         else:
             patch_fq = "importlib.metadata.distributions"
 
-        with mock.patch(patch_fq, *args, **kwargs) as m:
-            yield m
+        _get_grouped_entry_points.cache_clear()
+        try:
+            with mock.patch(patch_fq, *args, **kwargs) as m:
+                yield m
+        finally:
+            _get_grouped_entry_points.cache_clear()
 
     return wrapper
 
@@ -87,6 +92,7 @@ class TestPluginsManager:
         """
         mock_dist = mock.Mock()
         mock_dist.metadata = {"Name": "test-dist"}
+        mock_dist._normalized_name = "test_dist"
 
         mock_entrypoint = mock.Mock()
         mock_entrypoint.name = "test-entrypoint"

Reply via email to