This is an automated email from the ASF dual-hosted git repository.
Lee-W pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 50c5629b41c Run OpenLineage e2e compat tests when the compat setup
changes (#69866)
50c5629b41c is described below
commit 50c5629b41cdaaabf1bce9ba557335377ed74ddf
Author: PoAn Yang <[email protected]>
AuthorDate: Tue Jul 14 22:56:29 2026 +0900
Run OpenLineage e2e compat tests when the compat setup changes (#69866)
Co-authored-by: Wei Lee <[email protected]>
Signed-off-by: PoAn Yang <[email protected]>
---
dev/breeze/doc/ci/04_selective_checks.md | 10 +++++--
.../src/airflow_breeze/utils/selective_checks.py | 21 +++++++++----
dev/breeze/tests/test_selective_checks.py | 35 +++++++++++++++++++++-
3 files changed, 57 insertions(+), 9 deletions(-)
diff --git a/dev/breeze/doc/ci/04_selective_checks.md
b/dev/breeze/doc/ci/04_selective_checks.md
index 99c384ef874..dd47f24b19a 100644
--- a/dev/breeze/doc/ci/04_selective_checks.md
+++ b/dev/breeze/doc/ci/04_selective_checks.md
@@ -435,9 +435,13 @@ together using `pytest-xdist` (pytest-xdist distributes
the tests among parallel
`PROD Image building`.
* `OpenLineage E2E compat tests` (the same suite rerun against older released
Airflow versions with
current provider code, exposed as the `run-openlineage-e2e-compat-tests`
output) are costly, so
- they do NOT run on every OpenLineage PR: only on `canary` runs or when the
`full tests needed`
- label is explicitly set. Deliberately not tied to *derived* `full tests
needed` (large PRs, env
- changes, pushes) — the same rationale as `run-ui-e2e-tests`.
+ they do NOT run on every OpenLineage PR: on `canary` runs, when the `full
tests needed` label is
+ explicitly set, or when a file that drives the compat setup but does not
itself force the full
+ matrix changes — the shared e2e harness
+ (`airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py` / `constants.py`)
or the compat Dockerfile
+ (`airflow-e2e-tests/docker/openlineage-compat.Dockerfile`). The compat
workflow
+ (`.github/workflows/openlineage-e2e-compat-tests.yml`) matches
`ENVIRONMENT_FILES` and so already
+ forces the full matrix — the same rationale as `run-ui-e2e-tests`.
* The specific unit test type is enabled only if changed files match the
expected patterns for each type
(`API`, `CLI`, `WWW`, `Providers` etc.). The `Always` test type is added
always if any unit
tests are run. `Providers` tests are removed if current branch is different
than `main`
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 1edf4b4bfd7..17e3fca8e2a 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -133,6 +133,7 @@ class FileGroupForCi(Enum):
JAVA_SDK_E2E_FILES = auto()
GO_SDK_E2E_FILES = auto()
OPENLINEAGE_E2E_FILES = auto()
+ OPENLINEAGE_E2E_COMPAT_FILES = auto()
ALL_PYPROJECT_TOML_FILES = auto()
ALL_PYTHON_FILES = auto()
ALL_SOURCE_FILES = auto()
@@ -260,6 +261,14 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] =
HashableDict(
r"^providers/common/io/.*",
r"^providers/common/sql/.*",
],
+ FileGroupForCi.OPENLINEAGE_E2E_COMPAT_FILES: [
+ # Only add files that affect the compat setup and do NOT already
trigger the full matrix
+ # here. The compat workflow
(.github/workflows/openlineage-e2e-compat-tests.yml) is
+ # intentionally absent: it matches ENVIRONMENT_FILES and so
already forces full_tests.
+ r"^airflow-e2e-tests/tests/airflow_e2e_tests/conftest\.py$",
+ r"^airflow-e2e-tests/tests/airflow_e2e_tests/constants\.py$",
+ r"^airflow-e2e-tests/docker/openlineage-compat\.Dockerfile$",
+ ],
FileGroupForCi.PYTHON_PRODUCTION_FILES: [
# Production Python source the runtime ships — excludes tests,
docs,
# dev tooling, and generated files within those trees. Used by
@@ -1053,13 +1062,14 @@ class SelectiveChecks:
@cached_property
def run_openlineage_e2e_compat_tests(self) -> bool:
- # The older-Airflow compat matrix is costly, so it does not run on
every OpenLineage PR:
- # only on canary (scheduled / main) or when a maintainer explicitly
asks via the label.
- # Deliberately not tied to derived full_tests_needed (large PRs, env
changes, pushes) — same
- # rationale as run_ui_e2e_tests.
+ # Costly older-Airflow matrix. Like run_ui_e2e_tests it is not
triggered by *derived*
+ # full_tests_needed (pushes, env changes, large PRs) — only by canary,
an explicit label, or
+ # an actual change to a file that drives the compat setup but does not
itself force the full
+ # matrix: the shared e2e harness (conftest / constants) or the compat
Dockerfile. The compat
+ # workflow already forces full_tests_needed via ENVIRONMENT_FILES.
if self._is_canary_run() or FULL_TESTS_NEEDED_LABEL in self._pr_labels:
return True
- return False
+ return self._should_be_run(FileGroupForCi.OPENLINEAGE_E2E_COMPAT_FILES)
@cached_property
def run_amazon_tests(self) -> bool:
@@ -1204,6 +1214,7 @@ class SelectiveChecks:
or self.run_java_sdk_e2e_tests
or self.run_go_sdk_e2e_tests
or self.run_openlineage_e2e_tests
+ or self.run_openlineage_e2e_compat_tests
or self.run_ui_e2e_tests
)
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 530e6ad74aa..25da64bd517 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1514,18 +1514,51 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
("airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/harness.py",),
{
"run-openlineage-e2e-tests": "true",
+ "run-openlineage-e2e-compat-tests": "false",
"prod-image-build": "true",
},
- id="Run OpenLineage e2e tests when the e2e harness changes",
+ id="Run OpenLineage e2e tests (not the costly compat matrix) when
the OL e2e suite changes",
),
pytest.param(
("providers/ftp/src/airflow/providers/ftp/hooks/ftp.py",),
{
"run-openlineage-e2e-tests": "false",
+ "run-openlineage-e2e-compat-tests": "false",
"prod-image-build": "false",
},
id="Do not run OpenLineage e2e tests for unrelated provider
change",
),
+ pytest.param(
+ ("airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py",),
+ {
+ "run-openlineage-e2e-tests": "false",
+ "run-openlineage-e2e-compat-tests": "true",
+ "full-tests-needed": "false",
+ "prod-image-build": "true",
+ },
+ id="Run OpenLineage e2e compat tests when the shared e2e harness
(conftest) changes",
+ ),
+ pytest.param(
+ ("airflow-e2e-tests/docker/openlineage-compat.Dockerfile",),
+ {
+ "run-openlineage-e2e-tests": "false",
+ "run-openlineage-e2e-compat-tests": "true",
+ "full-tests-needed": "false",
+ "prod-image-build": "true",
+ },
+ id="Run OpenLineage e2e compat tests when the compat Dockerfile
changes",
+ ),
+ pytest.param(
+ (".github/workflows/openlineage-e2e-compat-tests.yml",),
+ {
+ # The compat workflow matches ENVIRONMENT_FILES, so it forces
the full matrix.
+ # The compat therefore runs via full_tests_needed, not via
OPENLINEAGE_E2E_COMPAT_FILES.
+ "run-openlineage-e2e-tests": "true",
+ "run-openlineage-e2e-compat-tests": "true",
+ "full-tests-needed": "true",
+ },
+ id="Compat workflow change forces the full matrix (compat runs via
full_tests_needed)",
+ ),
(
pytest.param(
("devel-common/pyproject.toml",),
