This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-19909-1d2e87269cf3981f91043e1891a7492e71e34ed1 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit e9e589796533ba08902ddee3dd039f0c3acdeba7 Author: voonhous <[email protected]> AuthorDate: Mon Nov 16 11:14:07 2026 +0000 chore: trim CI builds to one full chain per module and lane (#19909) * chore(ci): one full build per module and lane Part of #19524. Workflow files only. - flink-2's failsafe filter -Dit.test=!ITTestHoodieDataSource dropped failsafe's IT* include and ran 170 unit-test classes the docker job already runs. New filter: ITTest*,!ITTestHoodieDataSource. - The docker job built the whole reactor, 7.8 minutes, of which the 15 bundles and the examples are 6. The build, UT and FT reactors now skip the bundles, the examples and hudi-cli, each of which is built, tested or validated in the job that owns it. - hudi-examples-spark is built only in the jobs that run its quickstart. Measured over three runs: 577 to a mean of 548 runner-minutes, flink-2 39 to 23 minutes, the docker job 37 to about 30. * chore(ci): match flink-2's IT filter to the failsafe include ITTest* is narrower than the integration-tests profile's **/IT*.java include, so a future hudi-flink IT not named ITTest* would run in no job. Use IT*,!ITTestHoodieDataSource. Today both select the same 19 classes. * chore(ci): check BUNDLE_MODULES against the reactor's bundles A bundle missing from BUNDLE_MODULES stays in the docker job's reactor and fails to resolve the modules the trimmed build no longer installs, as hudi-spark-native-bundle would have. The comment now says every new packaging/ module must be listed, and a node test in validate-pr-and-source compares the list with the packaging/ modules of the root pom's default <modules>. --- .github/ci/scripts/test-fork-ci-status.cjs | 15 +++++++++ .github/workflows/bot.yml | 52 +++++++++++++++++++----------- 2 files changed, 49 insertions(+), 18 deletions(-) diff --git a/.github/ci/scripts/test-fork-ci-status.cjs b/.github/ci/scripts/test-fork-ci-status.cjs index bbfd556dd12a..cc2c6d6ecf8e 100644 --- a/.github/ci/scripts/test-fork-ci-status.cjs +++ b/.github/ci/scripts/test-fork-ci-status.cjs @@ -452,6 +452,21 @@ test('the matrix-expanded test jobs stay within the ASF cap of 20 jobs at once', assert.equal(Number(stated?.[1]), total); }); +// test-common-and-other-modules builds and tests the default reactor minus its bundles. A bundle +// missing from BUNDLE_MODULES stays in the reactor and fails to resolve the modules the trimmed +// build no longer installs. +test('BUNDLE_MODULES lists every bundle of the default reactor', () => { + const pom = readFileSync(path.join(__dirname, '../../../pom.xml'), 'utf8'); + // The first <modules> block is the default reactor; profiles declare theirs later in the file. + const modules = [.../<modules>([\s\S]*?)<\/modules>/.exec(pom)[1].matchAll(/<module>([^<]+)<\/module>/g)] + .map((match) => match[1]); + const bundles = modules.filter((module) => module.startsWith('packaging/')); + assert.ok(bundles.length > 0); + const listed = ci.workflow.jobs['test-common-and-other-modules'].env.BUNDLE_MODULES.split(',') + .map((module) => module.replace(/^!/, '')); + assert.deepEqual([...listed].sort(), [...bundles].sort()); +}); + // [case, context, whether the test jobs run, whether the aggregate jobs run, the `changes` job] for (const [name, github, tests, aggregate, needs] of [ ['fork PR to master', forkPr, false, true], diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index f9d919797179..4372831bd867 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -76,7 +76,10 @@ env: SCALA_TEST_OTHERS1_FILTER: -DwildcardSuites=org.apache.spark.sql.hudi.procedure,org.apache.spark.sql.hudi.ddl SCALA_TEST_OTHERS2_FILTER: -DwildcardSuites=org.apache.hudi,org.apache.spark.hudi,org.apache.spark.sql.avro,org.apache.spark.sql.execution,org.apache.spark.sql.hive,org.apache.spark.sql.hudi.analysis,org.apache.spark.sql.hudi.blob,org.apache.spark.sql.hudi.catalog,org.apache.spark.sql.hudi.command,org.apache.spark.sql.hudi.common,org.apache.spark.sql.hudi.feature FLINK_IT_FILTER1: -Dit.test=ITTestHoodieDataSource - FLINK_IT_FILTER2: -Dit.test=!ITTestHoodieDataSource + # An include is required next to the exclusion: a bare exclusion replaces failsafe's IT* + # include pattern with every test class, which ran hudi-flink's unit tests a second time here. + # The include repeats the integration-tests profile's **/IT*.java, so no IT class is left out. + FLINK_IT_FILTER2: -Dit.test=IT*,!ITTestHoodieDataSource # Three conditions route the jobs. GitHub expressions cannot be named, so the first test job # spells them out in full under the YAML anchor `route`, every other test job reuses it through @@ -340,9 +343,22 @@ jobs: env: UT_MODULES: >- - !hudi-hadoop-common,!hudi-hadoop-mr,!hudi-client/hudi-java-client,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource,!hudi-spark-datasource/hudi-spark,!hudi-spark-datasource/hudi-spark3.5.x,!hudi-spark-datasource/hudi-spark3-common,!hudi-spark-datasource/hudi-spark-common,!hudi-utilities + !hudi-hadoop-common,!hudi-hadoop-mr,!hudi-client/hudi-java-client,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource,!hudi-spark-datasource/hudi-spark,!hudi-spark-datasource/hudi-spark3.5.x,!hudi-spark-datasource/hudi-spark3-common,!hudi-spark-datasource/hudi-spark-common,!hudi-utilities FT_MODULES: >- - !hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource/hudi-spark,!hudi-utilities + !hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource/hudi-spark,!hudi-utilities + # Every bundle of the default reactor. None has tests, validate-bundles shades and exercises + # every one, and nothing this job tests depends on them, so the build and the two test steps + # below all leave them out (a bundle left in a test reactor would try to resolve the modules + # the trimmed build no longer installs). A new packaging/ module in the root pom's <modules> + # must be added here; .github/ci/scripts/test-fork-ci-status.cjs checks the two match. + BUNDLE_MODULES: >- + !packaging/hudi-hadoop-mr-bundle,!packaging/hudi-datahub-sync-bundle,!packaging/hudi-hive-sync-bundle,!packaging/hudi-aws-bundle,!packaging/hudi-gcp-bundle,!packaging/hudi-azure-bundle,!packaging/hudi-spark-bundle,!packaging/hudi-spark-native-bundle,!packaging/hudi-presto-bundle,!packaging/hudi-utilities-slim-bundle,!packaging/hudi-timeline-server-bundle,!packaging/hudi-flink-bundle,!packaging/hudi-kafka-connect-bundle,!packaging/hudi-cli-bundle,!packaging/hudi-utilities-bundle + # Also skipped by the build: the metaserver bundle, which only exists under + # -Phudi-platform-service and so cannot be listed for the test steps; the examples, which + # this job never tests; and hudi-cli, which depends on the utilities bundle and is built + # and tested in test-spark-client-and-hadoop-common. + BUILD_SKIP_MODULES: >- + !packaging/hudi-metaserver-server-bundle,!hudi-cli,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-spark,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-k8s steps: - uses: actions/checkout@v5 @@ -367,7 +383,7 @@ jobs: -v /var/run/docker.sock:/var/run/docker.sock \ -w /hudi \ hudi-ci \ - /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 2 -D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -Phudi-platform-service -Pthrift-gen-source -DskipTests=true -Dmaven.compiler.maxmem=8192m -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS" + /bin/bash -c "MAVEN_OPTS='-Xmx8g' mvn clean install -T 2 -D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -Phudi-platform-service -Pthrift-gen-source -DskipTests=true -Dmaven.compiler.maxmem=8192m -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -pl $BUILD_SKIP_MODULES,$BUNDLE_MODULES" - name: UT - common and other modules env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -380,7 +396,7 @@ jobs: -v /var/run/docker.sock:/var/run/docker.sock \ -w /hudi \ hudi-ci \ - /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests -D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $UT_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false" + /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests -D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $UT_MODULES,$BUNDLE_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false" - name: FT - common and other modules env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -393,7 +409,7 @@ jobs: -v /var/run/docker.sock:/var/run/docker.sock \ -w /hudi \ hudi-ci \ - /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests -D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $FT_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false" + /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests -D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $FT_MODULES,$BUNDLE_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -437,7 +453,7 @@ jobs: # SPARK_PROFILE: ${{ matrix.sparkProfile }} # SPARK_MODULES: ${{ matrix.sparkModules }} # run: -# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES" +# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" # - name: Java UT 1 - Common & Spark # env: # SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -556,7 +572,7 @@ jobs: # SPARK_PROFILE: ${{ matrix.sparkProfile }} # SPARK_MODULES: ${{ matrix.sparkModules }} # run: -# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES" +# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" # - name: Java FTB - Spark # env: # SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -616,7 +632,7 @@ jobs: # SPARK_PROFILE: ${{ matrix.sparkProfile }} # SPARK_MODULES: ${{ matrix.sparkModules }} # run: -# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES" +# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" # - name: Scala UT - Common & Spark # env: # SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -676,7 +692,7 @@ jobs: # SPARK_PROFILE: ${{ matrix.sparkProfile }} # SPARK_MODULES: ${{ matrix.sparkModules }} # run: -# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,$SPARK_COMMON_MODULES,$SPARK_MODULES" +# mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" # - name: Scala UT - Common & Spark # env: # SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -803,7 +819,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Java UT - functional package env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -921,7 +937,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Java UT 2 - Common & Spark env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -982,7 +998,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Java FTA - Spark env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -1036,7 +1052,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Java FTB - Spark env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -1090,7 +1106,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Scala UT - Common & Spark (dml.others) env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -1151,7 +1167,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Scala UT - Common & Spark (dml.insert, dml.schema) env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -1248,7 +1264,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Scala UT - Common & Spark (procedure, ddl) env: SCALA_PROFILE: ${{ matrix.scalaProfile }} @@ -1309,7 +1325,7 @@ jobs: SPARK_PROFILE: ${{ matrix.sparkProfile }} SPARK_MODULES: ${{ matrix.sparkModules }} run: - mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-examples/hudi-examples-spark,hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" + mvn clean install -T 2 -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DskipTests=true $MVN_ARGS -am -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" - name: Scala UT - Common & Spark (remaining packages) env: SCALA_PROFILE: ${{ matrix.scalaProfile }}
