This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a40cbd09dc49 chore(ci): build and verify only the IT modules in
integration-tests (#19919)
a40cbd09dc49 is described below
commit a40cbd09dc4931e70c7ebe001e0ccefa52a676fc
Author: voonhous <[email protected]>
AuthorDate: Tue Nov 17 08:11:56 2026 +0000
chore(ci): build and verify only the IT modules in integration-tests
(#19919)
* chore(ci): build and verify only the IT modules in integration-tests
Part of #19524.
IT_MODULES names the four modules with integration-test classes
outside hudi-flink: hudi-cli, hudi-integ-test, hudi-aws and
hudi-azure. The build passes them with -am, which Maven resolves to
30 modules; the IT step verifies just the four instead of the whole
reactor minus hudi-flink. Every module no longer built here is
built, and where a bundle shaded and run, in the job that owns it.
Measured: the job from 37 to 28 minutes, same 25 test classes.
* chore(ci): check IT_MODULES against the IT classes
The integration-tests job lists its IT modules by hand. A node test
now walks the source tree for IT*.java test classes and fails when
the modules holding them, hudi-flink aside, differ from the list.
---
.github/ci/scripts/test-fork-ci-status.cjs | 28 ++++++++++++++++++++++++++++
.github/workflows/bot.yml | 11 +++++++++--
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/.github/ci/scripts/test-fork-ci-status.cjs
b/.github/ci/scripts/test-fork-ci-status.cjs
index 79aa181d1500..55d27fbc91c7 100644
--- a/.github/ci/scripts/test-fork-ci-status.cjs
+++ b/.github/ci/scripts/test-fork-ci-status.cjs
@@ -466,6 +466,34 @@ test('BUNDLE_MODULES lists every bundle of the default
reactor', () => {
assert.deepEqual([...listed].sort(), [...bundles].sort());
});
+test('IT_MODULES lists every module with integration-test classes but
hudi-flink', () => {
+ const root = path.join(__dirname, '../../..');
+ // The integration-tests profile of the root pom runs **/IT*.java;
hudi-flink's ITs run in the
+ // flink jobs.
+ const modules = new Set();
+ const walk = (dir) => {
+ for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+ if (entry.name.startsWith('.') || entry.name === 'target' || entry.name
=== 'node_modules') {
+ continue;
+ }
+ const file = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ walk(file);
+ } else if (/^IT.*\.java$/.test(entry.name)) {
+ const relative = path.relative(root, file).split(path.sep).join('/');
+ const module = /^(.+?)\/src\/test\//.exec(relative);
+ if (module) {
+ modules.add(module[1]);
+ }
+ }
+ }
+ };
+ walk(root);
+ assert.ok(modules.delete('hudi-flink-datasource/hudi-flink'));
+ const listed =
ci.workflow.jobs['integration-tests'].env.IT_MODULES.split(',');
+ assert.deepEqual([...listed].sort(), [...modules].sort());
+});
+
// [case, context, whether the test jobs run, whether the aggregate job runs,
the `changes` job]
for (const [name, github, tests, aggregate, needs] of [
['fork PR to master', forkPr, false, true],
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index d8b4206192ae..f9fbd1fafdfd 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -1875,6 +1875,13 @@ jobs:
- sparkProfile: 'spark3.5'
flinkProfile: 'flink2.2'
sparkArchive: 'spark-3.5.9/spark-3.5.9-bin-hadoop3.tgz'
+ env:
+ # The only modules with integration-test classes besides hudi-flink,
whose ITs run in the
+ # flink jobs. The build below builds these and what they depend on (-am)
and the IT step
+ # verifies just these, instead of building and re-packaging all 66
modules. A module that
+ # adds an IT class must be listed here, or its ITs stop running in CI;
+ # .github/ci/scripts/test-fork-ci-status.cjs checks the list against the
IT classes.
+ IT_MODULES:
hudi-cli,hudi-integ-test,hudi-aws,hudi-azure,packaging/hudi-gcp-bundle
steps:
- uses: actions/checkout@v5
- name: Set up JDK 11
@@ -1901,7 +1908,7 @@ jobs:
FLINK_PROFILE: ${{ matrix.flinkProfile }}
SCALA_PROFILE: '-Dscala-2.12 -Dscala.binary.version=2.12'
run:
- mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true $MVN_ARGS
-Ddocker.compose.skip=true
+ mvn clean install -T 2 $SCALA_PROFILE -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -Pintegration-tests -DskipTests=true $MVN_ARGS
-Ddocker.compose.skip=true -pl "$IT_MODULES" -am
- name: 'UT integ-test'
env:
SPARK_PROFILE: ${{ matrix.sparkProfile }}
@@ -1943,7 +1950,7 @@ jobs:
SPARK_ARCHIVE_BASENAME=$(basename $SPARK_ARCHIVE)
export SPARK_HOME=$GITHUB_WORKSPACE/${SPARK_ARCHIVE_BASENAME%.*}
rm -f $GITHUB_WORKSPACE/$SPARK_ARCHIVE
- mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl
!hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false
+ mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl
"$IT_MODULES" $MVN_ARGS -Djacoco.skip=false
- name: Test timing summary
if: always()
run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job
}}"