This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-12474-446ff6266886fe1406b2abc2a3553b2ea070a21a in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit 866079361a2d8fb747361e889eec4c12e1f3aa10 Author: Goutam Adwant <[email protected]> AuthorDate: Mon Sep 28 13:21:43 2026 +0000 [Fix][CI] Route engine, UDF and standalone-module changes to their tests (#12474) --- .github/workflows/backend.yml | 31 +++++++-- .../test_update_modules_check.py | 76 ++++++++++++++++++++++ tools/update_modules_check/update_modules_check.py | 23 +++++++ 3 files changed, 124 insertions(+), 6 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index acd1042a52..1745584921 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -258,6 +258,13 @@ jobs: echo "dist=$true_or_false" >> $GITHUB_OUTPUT echo "dist_files=$file_list" >> $GITHUB_OUTPUT + # Modules outside every other route whose tests only run in the unit-test job. + standalone_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-trace/**" "seatunnel-e2e/seatunnel-core-e2e/**"` + true_or_false=${standalone_files%%$'\n'*} + file_list=${standalone_files#*$'\n'} + echo "standalone=$true_or_false" >> $GITHUB_OUTPUT + echo "standalone_files=$file_list" >> $GITHUB_OUTPUT + api_files=`python tools/update_modules_check/check_file_updates.py ua $workspace apache/dev origin/$current_branch "seatunnel-api/**" "seatunnel-common/**" "seatunnel-config/**" "seatunnel-core/**" "seatunnel-e2e/seatunnel-e2e-common/**" "seatunnel-formats/**" "seatunnel-plugin-discovery/**" "seatunnel-transforms-v2/**" "seatunnel-translation/**" "seatunnel-e2e/seatunnel-transforms-v2-e2e/**" "pom.xml" "**/workflows/**" "tools/**" "seatunnel-dist/**"` true_or_false=${api_files%%$'\n'*} file_list=${api_files#*$'\n'} @@ -312,6 +319,16 @@ jobs: echo $modules echo "modules=$modules" >> $GITHUB_OUTPUT + - name: Check Standalone Module Update + id: standalone-modules + if: ${{ steps.filter.outputs.standalone == 'true' }} + env: + STANDALONE_FILES: ${{ steps.filter.outputs.standalone_files }} + run: | + modules=`python tools/update_modules_check/update_modules_check.py standalone "$STANDALONE_FILES"` + echo $modules + echo "modules=$modules" >> $GITHUB_OUTPUT + - name: Check Deleted Modules id: deleted-modules if: ${{ steps.filter.outputs.deleted-poms == 'true' }} @@ -323,13 +340,14 @@ jobs: - name: Make unit test modules id: ut-modules timeout-minutes: 60 - if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.filter.outputs.dist == 'true') }} + if: ${{ steps.filter.outputs.api == 'false' && (steps.engine-modules.outputs.modules != '' || steps.cv2-modules.outputs.modules != '' || steps.filter.outputs.dist == 'true' || steps.standalone-modules.outputs.modules != '') }} run: | dist_modules='' if [[ '${{ steps.filter.outputs.dist }}' == 'true' ]]; then dist_modules=',seatunnel-dist' fi - modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'$dist_modules + standalone_modules='${{ steps.standalone-modules.outputs.modules }}' + modules='${{ steps.engine-modules.outputs.modules }}${{ steps.cv2-modules.outputs.modules }}'$dist_modules$standalone_modules modules=${modules: 1} pl_modules=`python tools/update_modules_check/update_modules_check.py replace "$modules"` # remove deleted modules @@ -344,7 +362,7 @@ jobs: engine_modules='${{ steps.engine-modules.outputs.modules }}' connector_modules='${{ steps.cv2-modules.outputs.modules }}' - if [[ "zz${engine_modules}${connector_modules}" == "zz" && "zz"$dist_modules != "zz" ]];then + if [[ "zz${engine_modules}${connector_modules}" == "zz" && "zz${dist_modules}${standalone_modules}" != "zz" ]];then echo $pl_modules modules_json=$(python tools/update_modules_check/update_modules_check.py json "$pl_modules") echo "modules=$modules_json" >> "$GITHUB_OUTPUT" @@ -892,7 +910,7 @@ jobs: engine-k8s-it: needs: [ changes, sanity-check ] - if: needs.changes.outputs.api == 'true' || contains(fromJSON(needs.changes.outputs.it-modules), 'seatunnel-engine-k8s-e2e') + if: needs.changes.outputs.api == 'true' || needs.changes.outputs.engine == 'true' || contains(fromJSON(needs.changes.outputs.it-modules), 'seatunnel-engine-k8s-e2e') runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -937,7 +955,8 @@ jobs: matrix: java: [ '8', '11' ] os: [ 'ubuntu-latest' ] - timeout-minutes: 120 + # Part-1 alone takes 99-107 minutes on ubuntu-latest; the UDF E2E module adds its own build and tests. + timeout-minutes: 135 steps: - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.java }} @@ -950,7 +969,7 @@ jobs: run: tools/github/free_disk_space.sh - name: run transform-v2 integration test (part-1) run: | - ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1 -am -Pci + ./mvnw -B -T 1 verify -DskipUT=true -DskipIT=false -D"license.skipAddThirdParty"=true -D"skip.ui"=true --no-snapshot-updates -pl :seatunnel-transforms-v2-e2e-part-1,:seatunnel-transforms-v2-e2e-udf -am -Pci env: MAVEN_OPTS: -Xmx4096m diff --git a/tools/update_modules_check/test_update_modules_check.py b/tools/update_modules_check/test_update_modules_check.py index 341979243d..f6b277e47c 100644 --- a/tools/update_modules_check/test_update_modules_check.py +++ b/tools/update_modules_check/test_update_modules_check.py @@ -17,6 +17,7 @@ """Regression tests for connector E2E module sharding.""" import io +import json import re import unittest from collections import Counter @@ -27,6 +28,8 @@ from update_modules_check import ( ALL_CONNECTORS_DEDICATED_SHARD_MODULES, ALL_CONNECTORS_OPTIONAL_DEDICATED_SHARD_MODULES, ALL_CONNECTORS_REQUIRED_DEDICATED_SHARD_MODULES, + STANDALONE_MODULE_PATHS, + build_standalone_modules, build_sub_it_modules, get_sub_it_modules, get_sub_update_it_modules, @@ -275,5 +278,78 @@ class ConnectorItShardingTest(unittest.TestCase): ): build_sub_it_modules("connector-normal-e2e", 7, current_num) + @staticmethod + def repo_root(): + return Path(__file__).resolve().parents[2] + + @staticmethod + def pom_artifact_id(pom): + text = re.sub(r"<parent>.*?</parent>", "", pom.read_text(encoding="utf-8"), flags=re.S) + return re.search(r"<artifactId>([^<]+)</artifactId>", text).group(1) + + def workflow_pl_modules(self): + modules = set() + for pl in re.findall( + r"-pl\s+(:[A-Za-z0-9._-]+(?:,:[A-Za-z0-9._-]+)*)", self.workflow_text() + ): + modules.update(module.lstrip(":") for module in pl.split(",") if module) + return modules + + def test_standalone_paths_map_to_their_test_modules(self) -> None: + self.assertEqual( + ["seatunnel-trace-analyzer", "seatunnel-starter-e2e"], + build_standalone_modules( + json.dumps( + [ + "seatunnel-trace/seatunnel-trace-analyzer/src/main/java/A.java", + "seatunnel-trace/pom.xml", + "seatunnel-e2e/seatunnel-core-e2e/seatunnel-starter-e2e/pom.xml", + ] + ) + ), + ) + self.assertEqual([], build_standalone_modules("[]")) + + def test_standalone_modules_exist_and_match_workflow_filter(self) -> None: + workflow = self.workflow_text() + filter_line = next( + line for line in workflow.splitlines() if line.strip().startswith("standalone_files=") + ) + self.assertEqual( + [prefix + "**" for prefix, _ in STANDALONE_MODULE_PATHS], + re.findall(r'"([^"]+)"', filter_line), + ) + for path_prefix, module in STANDALONE_MODULE_PATHS: + with self.subTest(module=module): + poms = [ + pom + for pom in (self.repo_root() / path_prefix).rglob("pom.xml") + if "target" not in pom.parts + ] + self.assertIn(module, {self.pom_artifact_id(pom) for pom in poms}) + + def test_engine_changes_run_the_k8s_integration_test(self) -> None: + job = re.search( + r"^ engine-k8s-it:\n(.*?)(?=^ \S)", self.workflow_text(), re.M | re.S + ).group(1) + self.assertTrue( + "needs.changes.outputs.engine == 'true'" in job, + "engine-k8s-it must run for engine changes", + ) + + def test_every_non_connector_e2e_it_module_has_a_workflow_job(self) -> None: + e2e_root = self.repo_root() / "seatunnel-e2e" + workflow_modules = self.workflow_pl_modules() + for pom in sorted(e2e_root.rglob("pom.xml")): + relative = pom.relative_to(e2e_root).parts + if relative[0] in ("seatunnel-connector-v2-e2e", "seatunnel-e2e-common") or "target" in relative: + continue + test_root = pom.parent / "src" / "test" / "java" + if not test_root.is_dir() or not any(test_root.rglob("*IT.java")): + continue + module = self.pom_artifact_id(pom) + with self.subTest(module=module): + self.assertIn(module, workflow_modules) + if __name__ == "__main__": unittest.main() diff --git a/tools/update_modules_check/update_modules_check.py b/tools/update_modules_check/update_modules_check.py index e9217d5bf4..aa662e14c3 100644 --- a/tools/update_modules_check/update_modules_check.py +++ b/tools/update_modules_check/update_modules_check.py @@ -75,6 +75,27 @@ _CONNECTOR_IT_MODULES_WITH_DEDICATED_JOB = set( ) - {"connector-jdbc-e2e"} +# Paths that no other change filter in backend.yml covers, mapped to the module whose +# tests exercise them. These tests only run in the unit-test job. +STANDALONE_MODULE_PATHS = ( + ("seatunnel-trace/", "seatunnel-trace-analyzer"), + ("seatunnel-e2e/seatunnel-core-e2e/", "seatunnel-starter-e2e"), +) + + +def build_standalone_modules(files): + modules = [] + for file in json.loads(files): + for path_prefix, module in STANDALONE_MODULE_PATHS: + if file.startswith(path_prefix) and module not in modules: + modules.append(module) + return modules + + +def get_standalone_modules(files): + print("".join("," + module for module in build_standalone_modules(files))) + + def get_cv2_modules(files): get_modules(files, 1, "connector-", "seatunnel-connectors-v2") @@ -295,6 +316,8 @@ def main(argv): get_cv2_e2e_modules(argv[2]) elif argv[1] == "engine": get_engine_modules(argv[2]) + elif argv[1] == "standalone": + get_standalone_modules(argv[2]) elif argv[1] == "engine-e2e": get_engine_e2e_modules(argv[2]) elif argv[1] == "tree":
