This is an automated email from the ASF dual-hosted git repository. jason810496 pushed a commit to branch jason/lang-sdk-e2e/08-native-e2e in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 9e4160d7169b5323a54c4c238f44e90310d59aa1 Author: ZHE YOU LIU <[email protected]> AuthorDate: Mon Sep 28 08:02:53 2026 +0000 Run the Lang-SDK e2e jobs when native Dag parsing changes The Java and TypeScript e2e suites now parse native Dags through the Dag processor, the Dag bag and the coordinators' Dag importers, so changes to those sources select both jobs. The Java native-Dag bundle selects the Java job. --- dev/breeze/doc/ci/04_selective_checks.md | 16 ++++++--- .../src/airflow_breeze/utils/selective_checks.py | 13 ++++++++ dev/breeze/tests/test_selective_checks.py | 38 ++++++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 2c3f81b506e..fa7c675a4e6 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -446,11 +446,17 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel or tests), then tests for all Providers are run * `Java SDK E2E tests` (the `java_sdk` mode of the deployed-stack tests, exposed as the `run-java-sdk-e2e-tests` output) run when the Java SDK sources (`java-sdk/`, excluding `.md`), the - Java test-fixture bundle (`airflow-e2e-tests/java-test-bundle/`), the Java e2e suite or its Docker + Java test-fixture and native-Dag bundles (`airflow-e2e-tests/java-test-bundle/`, + `airflow-e2e-tests/java-native-bundle/`), the Java e2e suite or its Docker files (`airflow-e2e-tests/tests/airflow_e2e_tests/java_sdk_tests/`, - `airflow-e2e-tests/docker/java.yml`, `airflow-e2e-tests/docker/Dockerfile.java`), or the Java - coordinator (`task-sdk/src/airflow/sdk/coordinators/java/`, `_subprocess.py`) change. Like the - other deployed e2e suites, enabling them forces `PROD Image building`. + `airflow-e2e-tests/docker/java.yml`, `airflow-e2e-tests/docker/Dockerfile.java`), the Java + coordinator (`task-sdk/src/airflow/sdk/coordinators/java/`, `_subprocess.py`), or the native + Lang-SDK Dag parsing sources change. Those are the Dag processing files `dagbag.py`, + `importer_routing.py`, `manager.py` and `processor.py`, `models/dagcode.py`, the Task SDK importers + (`task-sdk/src/airflow/sdk/importers/`), the coordinator registry + (`task-sdk/src/airflow/sdk/execution_time/coordinator.py`), and `coordinators/_dag_importer.py` + and `coordinators/_materialize.py`; they trigger the TypeScript SDK E2E tests too. Like the other + deployed e2e suites, enabling them forces `PROD Image building`. * `OpenLineage E2E tests` (the `openlineage` mode of the deployed-stack tests under `airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests`, exposed as the `run-openlineage-e2e-tests` output) run when the `openlineage` or `common` providers or the @@ -610,7 +616,7 @@ GitHub Actions to pass the list of parameters to a command to execute | run-system-tests | Whether system tests should be run ("true"/"false") | true | | | run-task-sdk-tests | Whether Task SDK tests should be run ("true"/"false") | true | | | run-ts-sdk-docs | Whether the TypeScript SDK API reference should be built — on `ts-sdk/api-docs/`, `ts-sdk/docs/`, or `ts-sdk/src/` changes, including Markdown ("true"/"false") | true | | -| run-ts-sdk-e2e-tests | Whether TypeScript SDK e2e tests should be run — on runtime-affecting `ts-sdk/`, TS e2e test, or Node coordinator changes ("true"/"false") | true | | +| run-ts-sdk-e2e-tests | Whether TypeScript SDK e2e tests should be run — on runtime-affecting `ts-sdk/`, TS e2e test, Node coordinator, or native Lang-SDK Dag parsing changes ("true"/"false") | true | | | run-ui-tests | Whether UI tests should be run ("true"/"false") | true | | | run-unit-tests | Whether unit tests should be run ("true"/"false") | true | | | run-www-tests | Whether Legacy WWW tests should be run ("true"/"false") | true | | diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 842d3a017c9..b95ab843fa2 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -180,6 +180,16 @@ class HashableDict(dict[T, list[str]]): return hash(frozenset(self)) +# Core and Task SDK sources on the native Lang-SDK Dag path: discovery, parsing through a +# coordinator's runtime, rebuilding the Dag for a Dag bag, and reading its source. +LANG_SDK_NATIVE_DAG_PARSING_FILES = ( + r"^airflow-core/src/airflow/dag_processing/(dagbag|importer_routing|manager|processor)\.py$", + r"^airflow-core/src/airflow/models/dagcode\.py$", + r"^task-sdk/src/airflow/sdk/coordinators/_(dag_importer|materialize)\.py$", + r"^task-sdk/src/airflow/sdk/execution_time/coordinator\.py$", + r"^task-sdk/src/airflow/sdk/importers/.*", +) + CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] = HashableDict( { FileGroupForCi.ENVIRONMENT_FILES: [ @@ -246,11 +256,13 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] = HashableDict( # `.md` excluded — doc-only edits do not affect the Gradle build. r"^java-sdk/(?!.*\.md$).*", r"^airflow-e2e-tests/java-test-bundle/.*", + r"^airflow-e2e-tests/java-native-bundle/.*", r"^airflow-e2e-tests/tests/airflow_e2e_tests/java_sdk_tests/.*", r"^airflow-e2e-tests/docker/java\.yml$", r"^airflow-e2e-tests/docker/Dockerfile\.java$", r"^task-sdk/src/airflow/sdk/coordinators/_subprocess\.py$", r"^task-sdk/src/airflow/sdk/coordinators/java/.*", + *LANG_SDK_NATIVE_DAG_PARSING_FILES, ], FileGroupForCi.GO_SDK_E2E_FILES: [ # `.md` excluded — doc-only edits do not affect the Go build or e2e tests. @@ -283,6 +295,7 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] = HashableDict( r"^airflow-e2e-tests/docker/ts\.yml$", r"^task-sdk/src/airflow/sdk/coordinators/_subprocess\.py$", r"^task-sdk/src/airflow/sdk/coordinators/node/.*", + *LANG_SDK_NATIVE_DAG_PARSING_FILES, ], FileGroupForCi.PYTHON_PRODUCTION_FILES: [ # Production Python source the runtime ships — excludes tests, docs, diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index be57fd40b15..72327a6348b 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1560,6 +1560,44 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): }, id="Run java e2e tests when JavaCoordinator changes", ), + pytest.param( + ("task-sdk/src/airflow/sdk/coordinators/_dag_importer.py",), + { + "run-java-sdk-e2e-tests": "true", + "run-ts-sdk-e2e-tests": "true", + "prod-image-build": "true", + }, + id="Run java and ts e2e tests when the coordinator Dag importer changes", + ), + pytest.param( + ("airflow-core/src/airflow/dag_processing/manager.py",), + { + "run-java-sdk-e2e-tests": "true", + "run-ts-sdk-e2e-tests": "true", + "prod-image-build": "true", + }, + id="Run java and ts e2e tests when the Dag processor manager changes", + ), + pytest.param( + ( + "airflow-e2e-tests/java-native-bundle/src/java/org/apache/airflow/e2e/NativeBundleBuilder.java", + ), + { + "run-java-sdk-tests": "false", + "run-java-sdk-e2e-tests": "true", + "run-ts-sdk-e2e-tests": "false", + "prod-image-build": "true", + }, + id="Run only java e2e tests when the java native-Dag bundle changes", + ), + pytest.param( + ("airflow-core/src/airflow/dag_processing/bundles/base.py",), + { + "run-java-sdk-e2e-tests": "false", + "run-ts-sdk-e2e-tests": "false", + }, + id="Skip java and ts e2e tests for a Dag processing change off the native Dag path", + ), pytest.param( ("go-sdk/sdk/variable.go",), {
