This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-19919-47958f5309ffb46cb6ffda3b9e9d7fc512c004ad in repository https://gitbox.apache.org/repos/asf/hudi.git
commit a40cbd09dc4931e70c7ebe001e0ccefa52a676fc Author: voonhous <[email protected]> AuthorDate: Tue Nov 17 08:11:56 2026 +0000 chore(ci): build and verify only the IT modules in integration-tests (#19919) * chore(ci): build and verify only the IT modules in integration-tests Part of #19524. IT_MODULES names the four modules with integration-test classes outside hudi-flink: hudi-cli, hudi-integ-test, hudi-aws and hudi-azure. The build passes them with -am, which Maven resolves to 30 modules; the IT step verifies just the four instead of the whole reactor minus hudi-flink. Every module no longer built here is built, and where a bundle shaded and run, in the job that owns it. Measured: the job from 37 to 28 minutes, same 25 test classes. * chore(ci): check IT_MODULES against the IT classes The integration-tests job lists its IT modules by hand. A node test now walks the source tree for IT*.java test classes and fails when the modules holding them, hudi-flink aside, differ from the list. --- .github/ci/scripts/test-fork-ci-status.cjs | 28 ++++++++++++++++++++++++++++ .github/workflows/bot.yml | 11 +++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/ci/scripts/test-fork-ci-status.cjs b/.github/ci/scripts/test-fork-ci-status.cjs index 79aa181d1500..55d27fbc91c7 100644 --- a/.github/ci/scripts/test-fork-ci-status.cjs +++ b/.github/ci/scripts/test-fork-ci-status.cjs @@ -466,6 +466,34 @@ test('BUNDLE_MODULES lists every bundle of the default reactor', () => { assert.deepEqual([...listed].sort(), [...bundles].sort()); }); +test('IT_MODULES lists every module with integration-test classes but hudi-flink', () => { + const root = path.join(__dirname, '../../..'); + // The integration-tests profile of the root pom runs **/IT*.java; hudi-flink's ITs run in the + // flink jobs. + const modules = new Set(); + const walk = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + if (entry.name.startsWith('.') || entry.name === 'target' || entry.name === 'node_modules') { + continue; + } + const file = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(file); + } else if (/^IT.*\.java$/.test(entry.name)) { + const relative = path.relative(root, file).split(path.sep).join('/'); + const module = /^(.+?)\/src\/test\//.exec(relative); + if (module) { + modules.add(module[1]); + } + } + } + }; + walk(root); + assert.ok(modules.delete('hudi-flink-datasource/hudi-flink')); + const listed = ci.workflow.jobs['integration-tests'].env.IT_MODULES.split(','); + assert.deepEqual([...listed].sort(), [...modules].sort()); +}); + // [case, context, whether the test jobs run, whether the aggregate job runs, the `changes` job] for (const [name, github, tests, aggregate, needs] of [ ['fork PR to master', forkPr, false, true], diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index d8b4206192ae..f9fbd1fafdfd 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -1875,6 +1875,13 @@ jobs: - sparkProfile: 'spark3.5' flinkProfile: 'flink2.2' sparkArchive: 'spark-3.5.9/spark-3.5.9-bin-hadoop3.tgz' + env: + # The only modules with integration-test classes besides hudi-flink, whose ITs run in the + # flink jobs. The build below builds these and what they depend on (-am) and the IT step + # verifies just these, instead of building and re-packaging all 66 modules. A module that + # adds an IT class must be listed here, or its ITs stop running in CI; + # .github/ci/scripts/test-fork-ci-status.cjs checks the list against the IT classes. + IT_MODULES: hudi-cli,hudi-integ-test,hudi-aws,hudi-azure,packaging/hudi-gcp-bundle steps: - uses: actions/checkout@v5 - name: Set up JDK 11 @@ -1901,7 +1908,7 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} SCALA_PROFILE: '-Dscala-2.12 -Dscala.binary.version=2.12' run: - mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true $MVN_ARGS -Ddocker.compose.skip=true + mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true $MVN_ARGS -Ddocker.compose.skip=true -pl "$IT_MODULES" -am - name: 'UT integ-test' env: SPARK_PROFILE: ${{ matrix.sparkProfile }} @@ -1943,7 +1950,7 @@ jobs: SPARK_ARCHIVE_BASENAME=$(basename $SPARK_ARCHIVE) export SPARK_HOME=$GITHUB_WORKSPACE/${SPARK_ARCHIVE_BASENAME%.*} rm -f $GITHUB_WORKSPACE/$SPARK_ARCHIVE - mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl !hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false + mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl "$IT_MODULES" $MVN_ARGS -Djacoco.skip=false - name: Test timing summary if: always() run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}"
