This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-19918-e9e589796533ba08902ddee3dd039f0c3acdeba7 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit b2ee4f665b585d6c324ef110a04e7c6de675a516 Author: voonhous <[email protected]> AuthorDate: Tue Nov 17 02:04:59 2026 +0000 chore(ci): list the slowest tests of each job on the run page (#19918) * chore(ci): list the slowest tests of each job on the run page Closes #19528. Part of #19524. scripts/ci/test_timing_summary.py walks every surefire-reports directory, which surefire and the scalatest plugin both fill with JUnit XML, and prints the slowest classes and the slowest tests. Every job that runs tests, 17 in bot.yml, gets one step after its test steps with if: always(), so the tables appear on the run summary for green and red jobs alike. Under a second of runner time per job. * chore(ci): read failsafe reports in the test timing summary test-flink-1, test-flink-2 and integration-tests run mostly failsafe ITs, whose TEST-*.xml land in target/failsafe-reports, so their tables read 0 tests. Scan failsafe-reports next to surefire-reports. --- .github/workflows/bot.yml | 66 +++++++++++++++++++++ scripts/ci/test_timing_summary.py | 121 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 4372831bd867..847ae151097e 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -222,6 +222,9 @@ jobs: # No coverage on merge-queue runs. Codecov files an upload from a gh-readonly-queue ref under # master, for a commit that may never land, and a group that succeeds would upload the same # commit twice: once from the queue run, once from the master push run. + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -267,6 +270,9 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} run: mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -fae -Dtest="TestHoodieDeltaStreamer*" -Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -319,6 +325,9 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} run: mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -fae -Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -410,6 +419,9 @@ jobs: -w /hudi \ hudi-ci \ /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests -D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $FT_MODULES,$BUNDLE_MODULES -Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false" + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -461,6 +473,9 @@ jobs: # SPARK_MODULES: ${{ matrix.sparkModules }} # run: # mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests $JAVA_UT_FILTER1 -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false +# - name: Test timing summary +# if: always() +# run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" # - name: Generate merged coverage report # if: always() # run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -527,6 +542,9 @@ jobs: # SPARK_MODULES: ${{ matrix.sparkModules }} # run: # mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false +# - name: Test timing summary +# if: always() +# run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" # - name: Generate merged coverage report # if: always() # run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -587,6 +605,9 @@ jobs: # SPARK_MODULES: ${{ matrix.sparkModules }} # run: # mvn test -Pfunctional-tests-c -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false +# - name: Test timing summary +# if: always() +# run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" # - name: Generate merged coverage report # if: always() # run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -647,6 +668,9 @@ jobs: # SPARK_MODULES: ${{ matrix.sparkModules }} # run: # mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_DML_FILTER -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false +# - name: Test timing summary +# if: always() +# run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" # - name: Generate merged coverage report # if: always() # run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -707,6 +731,9 @@ jobs: # SPARK_MODULES: ${{ matrix.sparkModules }} # run: # mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_OTHERS_FILTER -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false +# - name: Test timing summary +# if: always() +# run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" # - name: Generate merged coverage report # if: always() # run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -754,6 +781,9 @@ jobs: FLINK_PROFILE: ${{ matrix.flinkProfile }} run: mvn test -Punit-tests -fae -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -pl hudi-hadoop-mr,hudi-client/hudi-java-client $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -827,6 +857,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests "$JAVA_UT_FUNCTIONAL_FILTER" -Dsurefire.failIfNoSpecifiedTests=false -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -891,6 +924,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Punit-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -DwildcardSuites=skipScalaTests "$JAVA_UT_NON_FUNCTIONAL_FILTER" -Dsurefire.failIfNoSpecifiedTests=false -pl "hudi-common,$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -952,6 +988,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Pfunctional-tests-c -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1006,6 +1045,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1060,6 +1102,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Pfunctional-tests-b -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1121,6 +1166,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_DML1_FILTER -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1182,6 +1230,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_DML2_FILTER -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1279,6 +1330,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_OTHERS1_FILTER -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1340,6 +1394,9 @@ jobs: SPARK_MODULES: ${{ matrix.sparkModules }} run: mvn test -Pfunctional-tests -Pjava17 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -Dtest=skipJavaTests $SCALA_TEST_OTHERS2_FILTER -Dsurefire.failIfNoSpecifiedTests=false -pl "$SPARK_COMMON_MODULES,$SPARK_MODULES" $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1486,6 +1543,9 @@ jobs: # No coverage on merge-queue runs. Codecov files an upload from a gh-readonly-queue ref under # master, for a commit that may never land, and a group that succeeds would upload the same # commit twice: once from the queue run, once from the master push run. + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && endsWith(matrix.flinkProfile, '2.2') && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1533,6 +1593,9 @@ jobs: run: | mvn clean install -T 2 -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -pl hudi-flink-datasource/hudi-flink -am -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" -DskipTests=true $MVN_ARGS mvn verify -Pintegration-tests -D"$SCALA_PROFILE" -D"$FLINK_PROFILE" -Davro.version="$FLINK_AVRO_VERSION" -Dparquet.version="$FLINK_PARQUET_VERSION" $FLINK_IT_FILTER2 -pl hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && endsWith(matrix.flinkProfile, '2.2') && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE @@ -1881,6 +1944,9 @@ jobs: export SPARK_HOME=$GITHUB_WORKSPACE/${SPARK_ARCHIVE_BASENAME%.*} rm -f $GITHUB_WORKSPACE/$SPARK_ARCHIVE mvn verify $SCALA_PROFILE -D"$SPARK_PROFILE" -Pintegration-tests -pl !hudi-flink-datasource/hudi-flink $MVN_ARGS -Djacoco.skip=false + - name: Test timing summary + if: always() + run: python3 scripts/ci/test_timing_summary.py --title "${{ github.job }}" - name: Generate merged coverage report if: always() && github.event_name != 'merge_group' run: ./scripts/jacoco/generate_merged_coverage_report.sh $GITHUB_WORKSPACE diff --git a/scripts/ci/test_timing_summary.py b/scripts/ci/test_timing_summary.py new file mode 100755 index 000000000000..5528c23294ae --- /dev/null +++ b/scripts/ci/test_timing_summary.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Summarise test timings from surefire, failsafe and scalatest XML reports. + +Scans every target/surefire-reports/TEST-*.xml and target/failsafe-reports/TEST-*.xml under the +given root (surefire and the scalatest-maven-plugin write the first, failsafe's integration tests +the second) and prints, as Markdown, the slowest test classes and +the slowest individual tests. CI appends the output to the job's step summary so the long tail +of a job is visible on the run page without opening logs; run it locally after `mvn test` to +see the same for a module. + +Usage: test_timing_summary.py [--root DIR] [--classes N] [--tests N] [--title TEXT] +""" +import argparse +import collections +import os +import sys +import xml.etree.ElementTree as ET + +REPORT_DIRS = ("surefire-reports", "failsafe-reports") + + +def scan(root): + classes = collections.defaultdict(lambda: [0.0, 0, 0]) # time, tests, failed + tests = [] + files = 0 + for dirpath, dirnames, filenames in os.walk(root): + # Reports live at <module>/target/{surefire,failsafe}-reports. Prune everything else so the + # walk stays cheap on a built checkout, where target/ holds thousands of class directories. + base = os.path.basename(dirpath) + if base == "target": + dirnames[:] = [d for d in dirnames if d in REPORT_DIRS] + elif base not in REPORT_DIRS: + dirnames[:] = [d for d in dirnames if d not in (".git", "node_modules", "src")] + if base not in REPORT_DIRS: + continue + for name in filenames: + if not (name.startswith("TEST-") and name.endswith(".xml")): + continue + try: + suite = ET.parse(os.path.join(dirpath, name)).getroot() + except ET.ParseError: + continue + files += 1 + for case in suite.iter("testcase"): + cls = case.get("classname") or suite.get("name") or "?" + if cls.startswith("org.scalatest.tools."): + continue + t = float(case.get("time") or 0) + failed = any(child.tag in ("failure", "error") for child in case) + entry = classes[cls] + entry[0] += t + entry[1] += 1 + entry[2] += failed + tests.append((t, cls, case.get("name") or "?", failed)) + return files, classes, tests + + +def short(cls): + return cls.rsplit(".", 1)[-1] + + +def fmt(seconds): + return f"{seconds / 60:.1f} min" if seconds >= 120 else f"{seconds:.1f} s" + + +def render(title, files, classes, tests, n_classes, n_tests): + total = sum(v[0] for v in classes.values()) + count = sum(v[1] for v in classes.values()) + out = [f"### {title}", "", + f"{count} tests in {len(classes)} classes from {files} report files, " + f"{fmt(total)} of test time.", ""] + if not classes: + return "\n".join(out) + out += [f"Slowest {min(n_classes, len(classes))} classes:", "", + "| class | time | tests | share |", "|---|---|---|---|"] + for cls, (t, n, failed) in sorted(classes.items(), key=lambda kv: -kv[1][0])[:n_classes]: + flag = f" ({failed} failed)" if failed else "" + out.append(f"| {short(cls)}{flag} | {fmt(t)} | {n} | {t / total * 100:.0f}% |") + out += ["", f"Slowest {min(n_tests, len(tests))} tests:", "", + "| test | time |", "|---|---|"] + for t, cls, name, failed in sorted(tests, key=lambda x: -x[0])[:n_tests]: + flag = " (failed)" if failed else "" + out.append(f"| {short(cls)}#{name}{flag} | {fmt(t)} |") + return "\n".join(out) + + +def main(): + p = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + p.add_argument("--root", default=".") + p.add_argument("--classes", type=int, default=15) + p.add_argument("--tests", type=int, default=15) + p.add_argument("--title", default="Test timing") + args = p.parse_args() + files, classes, tests = scan(args.root) + text = render(args.title, files, classes, tests, args.classes, args.tests) + print(text) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a") as fh: + fh.write(text + "\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main())
