This is an automated email from the ASF dual-hosted git repository. jonnybot pushed a commit to branch INDY-PERF-EXPLORATION in repository https://gitbox.apache.org/repos/asf/groovy.git
commit a7e426f3fbdfec749dd19f98af5930e2aea8b0f2 Author: Jonny Carter <[email protected]> AuthorDate: Fri Feb 20 13:18:14 2026 -0600 Dynamically group benchmarks for CI --- .github/benchmark-groups.json | 34 -------------- .github/workflows/groovy-performance.yml | 52 +++++++++------------ subprojects/performance/build.gradle | 78 ++++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 64 deletions(-) diff --git a/.github/benchmark-groups.json b/.github/benchmark-groups.json deleted file mode 100644 index cd95eae5a3..0000000000 --- a/.github/benchmark-groups.json +++ /dev/null @@ -1,34 +0,0 @@ -[ - { - "group": "dispatch", - "pattern": ".*bench\\.dispatch\\..*" - }, - { - "group": "grailslike", - "pattern": ".*bench\\.grailslike\\..*" - }, - { - "group": "indy", - "pattern": ".*bench\\.indy\\..*" - }, - { - "group": "memory", - "pattern": ".*bench\\.memory\\..*" - }, - { - "group": "orm", - "pattern": ".*bench\\.orm\\..*" - }, - { - "group": "profiling", - "pattern": ".*bench\\.profiling\\..*" - }, - { - "group": "core", - "pattern": ".*(Ackermann|Ary|Fibo|GeneratedHashCode)Bench.*" - }, - { - "group": "plugin", - "pattern": ".*plugin\\..*" - } -] diff --git a/.github/workflows/groovy-performance.yml b/.github/workflows/groovy-performance.yml index 1e2650f2ce..42915a978b 100644 --- a/.github/workflows/groovy-performance.yml +++ b/.github/workflows/groovy-performance.yml @@ -20,7 +20,7 @@ on: workflow_dispatch: inputs: benchmark_filter: - description: 'Benchmark group to run (e.g., dispatch, orm). Leave empty for all groups.' + description: 'Filter by subpackage name (e.g., dispatch, orm, indy). Leave empty for all.' required: false default: '' compare_baseline: @@ -77,10 +77,12 @@ jobs: # Full benchmark suite: build once, fan out into parallel matrix jobs # ============================================================================ - # Step 1: Build the JMH fat jar once + # Step 1: Build the JMH fat jar and discover benchmark groups build-jmh-jar: if: github.event_name != 'pull_request' runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -89,8 +91,19 @@ jobs: java-version: '17' - uses: gradle/actions/setup-gradle@v4 - - name: Build JMH fat jar - run: ./gradlew :performance:jmhJar + - name: Build JMH fat jar and discover groups + run: ./gradlew :performance:jmhGroups + + - name: Build matrix from groups + id: set-matrix + run: | + GROUPS=$(cat subprojects/performance/build/jmh-groups.json) + FILTER="${{ github.event.inputs.benchmark_filter }}" + if [ -n "$FILTER" ]; then + GROUPS=$(echo "$GROUPS" | jq -c --arg f "$FILTER" '[.[] | select(.group | contains($f))]') + fi + MATRIX=$(echo "$GROUPS" | jq -c '{include: [.[] | {group, pattern, indy: true}, {group, pattern, indy: false}]}') + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - name: Upload JMH jar uses: actions/upload-artifact@v4 @@ -99,36 +112,15 @@ jobs: path: subprojects/performance/build/libs/*-jmh.jar retention-days: 1 - # Step 2: Read benchmark groups and output matrix JSON - discover-groups: - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - uses: actions/checkout@v4 - - - name: Build matrix from benchmark groups - id: set-matrix - run: | - FILTER="${{ github.event.inputs.benchmark_filter }}" - if [ -n "$FILTER" ]; then - # Run only the requested group - MATRIX=$(jq -c --arg f "$FILTER" '[.[] | select(.group == $f)]' .github/benchmark-groups.json) - else - MATRIX=$(jq -c '.' .github/benchmark-groups.json) - fi - echo "matrix={\"include\":$(echo "$MATRIX" | jq -c '[.[] | {group: .group, pattern: .pattern, indy: true}] + [.[] | {group: .group, pattern: .pattern, indy: false}]')}" >> "$GITHUB_OUTPUT" - - # Step 3: Run benchmarks in parallel (groups x indy modes) + # Step 2: Run benchmarks in parallel (groups x indy modes) benchmark-matrix: - needs: [build-jmh-jar, discover-groups] + needs: build-jmh-jar if: github.event_name != 'pull_request' runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false - matrix: ${{ fromJson(needs.discover-groups.outputs.matrix) }} + matrix: ${{ fromJson(needs.build-jmh-jar.outputs.matrix) }} steps: - uses: actions/setup-java@v4 with: @@ -148,7 +140,7 @@ jobs: -Dgroovy.target.indy=${{ matrix.indy }} \ -jar "$JAR" \ "${{ matrix.pattern }}" \ - -wi 1 \ + -f 1 -wi 1 \ -rf json -rff results.json timeout-minutes: 45 @@ -297,7 +289,7 @@ jobs: -Dgroovy.target.indy=true \ -jar "$JAR" \ ".*bench\.memory\..*" \ - -f 1 -wi 1 -i 1 -r 2s -w 2s \ + -f 1 -wi 1 \ -prof gc \ -rf json -rff gc-profile-results.json diff --git a/subprojects/performance/build.gradle b/subprojects/performance/build.gradle index 2405d91011..820f717c8a 100644 --- a/subprojects/performance/build.gradle +++ b/subprojects/performance/build.gradle @@ -328,3 +328,81 @@ tasks.register('jmhCompareBaseline') { } } } + +// ============================================================================ +// Dynamic Benchmark Grouping (for CI matrix) +// ============================================================================ + +/** + * Discover all JMH benchmarks from the fat jar and group them by subpackage + * into appropriately-sized chunks for parallel CI execution. + * + * Outputs build/jmh-groups.json — an array of {group, pattern} objects. + */ +tasks.register('jmhGroups') { + group = 'benchmark' + description = 'Discover JMH benchmarks and output groups as JSON for CI matrix' + dependsOn 'jmhJar' + + def outputFile = layout.buildDirectory.file("jmh-groups.json") + outputs.file(outputFile) + + doLast { + // Time estimate: 1 fork × (1 warmup @10s + 1 iteration @10s) + ~5s overhead + def secondsPerMethod = 25 + def maxSecondsPerJob = 600 // 10-minute target + def maxPerGroup = (int) (maxSecondsPerJob / secondsPerMethod) + + // List all benchmarks from the fat jar + def listing = new ByteArrayOutputStream() + project.javaexec { + mainClass = 'org.openjdk.jmh.Main' + classpath = files(tasks.named('jmhJar')) + args '-l' + standardOutput = listing + } + + def benchmarks = listing.toString().readLines() + .collect { it.trim() } + .findAll { it.startsWith('org.apache.groovy.') } + + // Group by subpackage: bench.dispatch.* → "dispatch", bench.Foo → "core", plugin.* → "plugin" + def groups = benchmarks.groupBy { fqcn -> + def parts = (fqcn - 'org.apache.groovy.').split('\\.') + parts[0] == 'bench' ? (parts.length >= 4 ? parts[1] : 'core') : parts[0] + }.sort() + + // Build entries, splitting large groups by class + def entries = [] + groups.each { name, methods -> + if (methods.size() <= maxPerGroup) { + def pattern = name == 'core' + ? ".*(" + methods.collect { it.split('\\.')[-2] }.unique().sort().join('|') + ").*" + : ".*\\.${name}\\..*" + entries << [group: name, pattern: pattern, benchmarks: methods.size()] + } else { + def byClass = methods.groupBy { it.split('\\.')[-2] } + def chunk = [] + int chunkMethods = 0 + int chunkNum = 1 + byClass.sort().each { cls, meths -> + if (chunkMethods + meths.size() > maxPerGroup && chunk) { + entries << [group: "${name}-${chunkNum}", pattern: ".*(" + chunk.join('|') + ").*", benchmarks: chunkMethods] + chunk = [] + chunkMethods = 0 + chunkNum++ + } + chunk << cls + chunkMethods += meths.size() + } + if (chunk) { + entries << [group: "${name}-${chunkNum}", pattern: ".*(" + chunk.join('|') + ").*", benchmarks: chunkMethods] + } + } + } + + def json = groovy.json.JsonOutput.prettyPrint(groovy.json.JsonOutput.toJson(entries)) + outputFile.get().asFile.text = json + println json + } +}
