This is an automated email from the ASF dual-hosted git repository. jonnybot pushed a commit to branch INDY-PERF-EXPLORATION in repository https://gitbox.apache.org/repos/asf/groovy.git
commit 04e00b8c5ec59a7c5982a3728af20ca5fd296b04 Author: Jonny Carter <[email protected]> AuthorDate: Mon Feb 2 03:19:20 2026 -0600 Remove indy distinction for benchmarks --- subprojects/performance/README.adoc | 87 +++++++------------ subprojects/performance/build.gradle | 163 +---------------------------------- 2 files changed, 33 insertions(+), 217 deletions(-) diff --git a/subprojects/performance/README.adoc b/subprojects/performance/README.adoc index 1e2c15e2eb..4413ade130 100644 --- a/subprojects/performance/README.adoc +++ b/subprojects/performance/README.adoc @@ -28,14 +28,11 @@ with a particular focus on investigating invokedynamic performance characteristi [source,bash] ---- -# Run all benchmarks (indy mode) -./gradlew -Pindy=true :performance:jmh +# Run all benchmarks +./gradlew :performance:jmh # Run specific benchmark suite -./gradlew -Pindy=true -PbenchInclude=ColdCall :performance:jmh - -# Compare indy vs non-indy -./gradlew :performance:jmhCompare +./gradlew -PbenchInclude=ColdCall :performance:jmh ---- == Compiler Performance Tests @@ -53,11 +50,6 @@ JMH Benchmarks can be run using: ./gradlew :performance:jmh -In order to run the benchmarks against InvokeDynamic generated classes use -the `indy` property: - - ./gradlew -Pindy=true :performance:jmh - To run a single benchmark or a matched set of benchmarks, use the `benchInclude` property: @@ -69,7 +61,7 @@ names or class names. It is equivalent to `.*${benchInclude}.*`. == InvokeDynamic Performance Benchmarks A comprehensive set of benchmarks designed to investigate and isolate performance -issues with Groovy's invokedynamic implementation. See `INDY_PERFORMANCE_PLAN.md` +characteristics of Groovy's invokedynamic implementation. See `INDY_PERFORMANCE_PLAN.md` for the full testing plan and background. === Phase 1: Core Performance Benchmarks @@ -79,34 +71,34 @@ for the full testing plan and background. Measures the cost of method invocations before callsites are optimized. Critical for web applications where objects are created per-request. - ./gradlew -Pindy=true -PbenchInclude=ColdCallBench :performance:jmh + ./gradlew -PbenchInclude=ColdCallBench :performance:jmh ==== Warmup Behavior Benchmarks (`WarmupBehaviorBench`) Tests performance at different warmup levels (10, 100, 1000, 10000+ calls) to understand how the optimization threshold affects performance. - ./gradlew -Pindy=true -PbenchInclude=WarmupBehavior :performance:jmh + ./gradlew -PbenchInclude=WarmupBehavior :performance:jmh ==== Threshold Sensitivity Benchmarks (`ThresholdSensitivityBench`) Tests different usage patterns (web request, batch processing, mixed) to understand which patterns benefit from different threshold configurations. - ./gradlew -Pindy=true -PbenchInclude=ThresholdSensitivity :performance:jmh + ./gradlew -PbenchInclude=ThresholdSensitivity :performance:jmh ==== Cache Invalidation Benchmarks (`CacheInvalidationBench`) Tests polymorphic dispatch scenarios that cause inline cache invalidation, addressing issues described in GROOVY-8298. - ./gradlew -Pindy=true -PbenchInclude=CacheInvalidation :performance:jmh + ./gradlew -PbenchInclude=CacheInvalidation :performance:jmh ==== Property Access Benchmarks (`PropertyAccessBench`) Tests GORM-like property access patterns common in Grails applications. - ./gradlew -Pindy=true -PbenchInclude=PropertyAccess :performance:jmh + ./gradlew -PbenchInclude=PropertyAccess :performance:jmh ==== Memory Allocation Benchmarks (`MemoryAllocationBench`) @@ -114,11 +106,11 @@ Measures heap growth and allocation rates during Groovy operations. Quantifies the memory overhead of MethodHandleWrapper AtomicLong objects and CacheableCallSite LRU cache entries. - ./gradlew -Pindy=true -PbenchInclude=MemoryAllocation :performance:jmh + ./gradlew -PbenchInclude=MemoryAllocation :performance:jmh For detailed allocation profiling with GC metrics: - ./gradlew -Pindy=true -PbenchInclude=MemoryAllocation :performance:jmh -Pjmh.profilers=gc + ./gradlew -PbenchInclude=MemoryAllocation :performance:jmh -Pjmh.profilers=gc ==== Callsite Growth Benchmarks (`CallsiteGrowthBench`) @@ -126,14 +118,14 @@ Measures how memory grows as unique callsites accumulate. Tests with parameterized callsite counts (100, 1000, 10000) to quantify the ~32 bytes per cached method handle overhead. - ./gradlew -Pindy=true -PbenchInclude=CallsiteGrowth :performance:jmh + ./gradlew -PbenchInclude=CallsiteGrowth :performance:jmh ==== Long-Running Session Benchmarks (`LongRunningSessionBench`) Simulates web application memory patterns over time: request cycles, sustained load, and memory recovery after GC. - ./gradlew -Pindy=true -PbenchInclude=LongRunningSession :performance:jmh + ./gradlew -PbenchInclude=LongRunningSession :performance:jmh === Phase 2: Grails-Realistic Scenario Tests @@ -148,7 +140,7 @@ Simulates full HTTP request lifecycle in a Grails-like application: * Domain object manipulation (property access, validation) * Model building for view - ./gradlew -Pindy=true -PbenchInclude=RequestLifecycle :performance:jmh + ./gradlew -PbenchInclude=RequestLifecycle :performance:jmh ==== Template Render Benchmarks (`TemplateRenderBench`) @@ -159,7 +151,7 @@ Simulates GSP-like template rendering with heavy property access: * GString interpolation * Spread operators (`people*.name`) - ./gradlew -Pindy=true -PbenchInclude=TemplateRender :performance:jmh + ./gradlew -PbenchInclude=TemplateRender :performance:jmh ==== Mini-Grails Domain Model @@ -173,23 +165,7 @@ The `grailslike` package contains realistic domain classes: * `ControllerSimulator` - Request handling patterns * `TemplateSimulator` - GSP-like rendering -=== Phase 3: Comparison Infrastructure - -==== Automated Indy vs Non-Indy Comparison - -[source,bash] ----- -# Run both modes and generate comparison report -./gradlew :performance:jmhCompare - -# Or run individually -./gradlew :performance:jmhIndy -./gradlew :performance:jmhNonIndy ----- - -The comparison report is saved to `build/results/jmh-compare/comparison-report.txt`. - -==== Threshold Parameter Sweep +=== Threshold Parameter Sweep Test multiple threshold values to find optimal settings: @@ -200,9 +176,9 @@ Test multiple threshold values to find optimal settings: Tests thresholds: 0, 10, 100, 1000, 10000, 100000 -Results saved to `build/results/jmh-compare/threshold-sweep/threshold-summary.txt`. +Results saved to `build/results/jmh-threshold-sweep/threshold-summary.txt`. -=== Phase 4: Profiling Integration +=== Profiling Integration ==== JFR Profiling @@ -210,7 +186,7 @@ Run benchmarks with Java Flight Recorder: [source,bash] ---- -./gradlew -Pindy=true :performance:jmhProfile +./gradlew :performance:jmhProfile ---- JFR output saved to `build/results/jmh-profile/benchmark.jfr`. @@ -229,7 +205,7 @@ Run with JMH's GC profiler for detailed memory analysis: [source,bash] ---- -./gradlew -Pindy=true :performance:jmhGcProfile +./gradlew :performance:jmhGcProfile ---- ==== Memory Tracking Benchmarks @@ -238,20 +214,20 @@ Use `MemoryTrackingState` in benchmarks for detailed heap tracking: [source,bash] ---- -./gradlew -Pindy=true -PbenchInclude=MemoryProfile :performance:jmh +./gradlew -PbenchInclude=MemoryProfile :performance:jmh ---- -=== Phase 5: Baseline Management & CI +=== Baseline Management & CI ==== Saving Baselines [source,bash] ---- # Run benchmarks -./gradlew -Pindy=true :performance:jmh +./gradlew :performance:jmh # Save as baseline -./gradlew :performance:jmhSaveBaseline -PbaselineName=groovy-4.0.x-indy +./gradlew :performance:jmhSaveBaseline -PbaselineName=groovy-4.0.x ---- ==== Comparing Against Baselines @@ -263,7 +239,7 @@ Use `MemoryTrackingState` in benchmarks for detailed heap tracking: ./gradlew :performance:jmhCompareBaseline # Compare against specific baseline -./gradlew :performance:jmhCompareBaseline -PbaselineName=groovy-4.0.x-indy +./gradlew :performance:jmhCompareBaseline -PbaselineName=groovy-4.0.x ---- ==== CI Integration @@ -271,7 +247,7 @@ Use `MemoryTrackingState` in benchmarks for detailed heap tracking: Performance benchmarks run automatically via GitHub Actions: * **On PRs**: Smoke tests for performance-related changes -* **Weekly**: Full benchmark suite with indy/non-indy comparison +* **Weekly**: Full benchmark suite * **On demand**: Via workflow_dispatch with customizable filters See `.github/workflows/groovy-performance.yml` for details. @@ -287,7 +263,7 @@ Test with different thresholds: [source,bash] ---- -./gradlew -Pindy=true -PbenchInclude=ThresholdSensitivity :performance:jmh \ +./gradlew -PbenchInclude=ThresholdSensitivity :performance:jmh \ --jvmArgs="-Dgroovy.indy.optimize.threshold=100" ---- @@ -299,26 +275,27 @@ Focus on cold call and request lifecycle: [source,bash] ---- -./gradlew -Pindy=true -PbenchInclude="ColdCall|RequestLifecycle|Template" :performance:jmh +./gradlew -PbenchInclude="ColdCall|RequestLifecycle|Template" :performance:jmh ---- ==== For Memory Investigation [source,bash] ---- -./gradlew -Pindy=true -PbenchInclude=Memory :performance:jmh -Pjmh.profilers=gc +./gradlew -PbenchInclude=Memory :performance:jmh -Pjmh.profilers=gc ---- ==== For Polymorphic Dispatch Issues [source,bash] ---- -./gradlew -Pindy=true -PbenchInclude="CacheInvalidation|Dispatch" :performance:jmh +./gradlew -PbenchInclude="CacheInvalidation|Dispatch" :performance:jmh ---- ==== For Full Regression Testing [source,bash] ---- -./gradlew :performance:jmhCompare +./gradlew :performance:jmh +./gradlew :performance:jmhCompareBaseline ---- diff --git a/subprojects/performance/build.gradle b/subprojects/performance/build.gradle index 88cfd5dc04..2405d91011 100644 --- a/subprojects/performance/build.gradle +++ b/subprojects/performance/build.gradle @@ -32,159 +32,7 @@ performanceTests { project.files('src/jmh/groovy/org/apache/groovy/bench/Fibo.groovy') } -// ============================================================================ -// Indy vs Non-Indy Comparison Infrastructure -// ============================================================================ - def jmhResultsDir = layout.buildDirectory.dir("results/jmh") -def jmhCompareDir = layout.buildDirectory.dir("results/jmh-compare") - -/** - * Run JMH benchmarks with invokedynamic enabled. - * Results are saved to build/results/jmh-compare/indy-results.txt - */ -tasks.register('jmhIndy', JavaExec) { - dependsOn 'jmhJar' - group = 'benchmark' - description = 'Run JMH benchmarks with invokedynamic enabled' - - mainClass = 'org.openjdk.jmh.Main' - classpath = files(tasks.named('jmhJar')) - - def outputFile = jmhCompareDir.map { it.file("indy-results.txt") } - - args '-rf', 'text' - args '-rff', outputFile.get().asFile.absolutePath - - // Include filter if specified - if (project.hasProperty('benchInclude')) { - args '-e', '.*' + project.benchInclude + '.*' - } - - jvmArgs '-Dgroovy.target.indy=true' - - doFirst { - jmhCompareDir.get().asFile.mkdirs() - println "Running JMH with invokedynamic ENABLED" - println "Results will be saved to: ${outputFile.get().asFile}" - } -} - -/** - * Run JMH benchmarks with invokedynamic disabled (classic bytecode). - * Results are saved to build/results/jmh-compare/noindy-results.txt - */ -tasks.register('jmhNonIndy', JavaExec) { - dependsOn 'jmhJar' - group = 'benchmark' - description = 'Run JMH benchmarks with invokedynamic disabled (classic bytecode)' - - mainClass = 'org.openjdk.jmh.Main' - classpath = files(tasks.named('jmhJar')) - - def outputFile = jmhCompareDir.map { it.file("noindy-results.txt") } - - args '-rf', 'text' - args '-rff', outputFile.get().asFile.absolutePath - - // Include filter if specified - if (project.hasProperty('benchInclude')) { - args '-e', '.*' + project.benchInclude + '.*' - } - - jvmArgs '-Dgroovy.target.indy=false' - - doFirst { - jmhCompareDir.get().asFile.mkdirs() - println "Running JMH with invokedynamic DISABLED" - println "Results will be saved to: ${outputFile.get().asFile}" - } -} - -/** - * Run both indy and non-indy benchmarks and generate comparison report. - */ -tasks.register('jmhCompare') { - dependsOn 'jmhIndy', 'jmhNonIndy' - group = 'benchmark' - description = 'Run benchmarks with and without invokedynamic and compare results' - - def indyResults = jmhCompareDir.map { it.file("indy-results.txt") } - def nonIndyResults = jmhCompareDir.map { it.file("noindy-results.txt") } - def comparisonReport = jmhCompareDir.map { it.file("comparison-report.txt") } - - doLast { - def indyFile = indyResults.get().asFile - def nonIndyFile = nonIndyResults.get().asFile - def reportFile = comparisonReport.get().asFile - - if (!indyFile.exists() || !nonIndyFile.exists()) { - println "ERROR: Results files not found. Run jmhIndy and jmhNonIndy first." - return - } - - def parseResults = { File file -> - def results = [:] - file.eachLine { line -> - // Parse JMH result lines: BenchmarkName Mode Cnt Score Error Units - def match = line =~ /^(\S+)\s+(\S+)\s+(\d+)\s+([\d.]+)\s*±?\s*([\d.]*)\s*(\S+)/ - if (match) { - def name = match[0][1] - def score = match[0][4] as double - def error = match[0][5] ? (match[0][5] as double) : 0.0 - def unit = match[0][6] - results[name] = [score: score, error: error, unit: unit] - } - } - results - } - - def indyData = parseResults(indyFile) - def nonIndyData = parseResults(nonIndyFile) - - reportFile.withWriter { writer -> - writer.writeLine "=" * 80 - writer.writeLine "INVOKEDYNAMIC PERFORMANCE COMPARISON REPORT" - writer.writeLine "Generated: ${new Date()}" - writer.writeLine "=" * 80 - writer.writeLine "" - - def allBenchmarks = (indyData.keySet() + nonIndyData.keySet()).unique().sort() - - writer.writeLine String.format("%-60s %12s %12s %12s", "Benchmark", "Indy", "Non-Indy", "Ratio") - writer.writeLine "-" * 100 - - allBenchmarks.each { name -> - def indy = indyData[name] - def nonIndy = nonIndyData[name] - - if (indy && nonIndy) { - def ratio = indy.score / nonIndy.score - def ratioStr = String.format("%.2fx", ratio) - if (ratio > 1.1) ratioStr += " SLOWER" - else if (ratio < 0.9) ratioStr += " FASTER" - - writer.writeLine String.format("%-60s %12.3f %12.3f %12s", - name.length() > 60 ? "..." + name[-57..-1] : name, - indy.score, nonIndy.score, ratioStr) - } - } - - writer.writeLine "" - writer.writeLine "=" * 80 - writer.writeLine "Notes:" - writer.writeLine "- Ratio > 1.0 means indy is SLOWER than non-indy" - writer.writeLine "- Ratio < 1.0 means indy is FASTER than non-indy" - writer.writeLine "- Use -PbenchInclude=<pattern> to filter benchmarks" - writer.writeLine "=" * 80 - } - - println "" - println "Comparison report saved to: ${reportFile}" - println "" - println reportFile.text - } -} // ============================================================================ // Threshold Parameter Sweep @@ -199,7 +47,7 @@ tasks.register('jmhThresholdSweep') { description = 'Run benchmarks with varying groovy.indy.optimize.threshold values' def thresholds = [0, 10, 100, 1000, 10000, 100000] - def sweepDir = jmhCompareDir.map { it.dir("threshold-sweep") } + def sweepDir = layout.buildDirectory.dir("results/jmh-threshold-sweep") doFirst { sweepDir.get().asFile.mkdirs() @@ -231,7 +79,6 @@ tasks.register('jmhThresholdSweep') { } jvmArgs "-Dgroovy.indy.optimize.threshold=${threshold}" - jvmArgs '-Dgroovy.target.indy=true' } // Parse results @@ -319,10 +166,6 @@ tasks.register('jmhProfile', JavaExec) { jvmArgs '-XX:+FlightRecorder' jvmArgs "-XX:StartFlightRecording=duration=60s,filename=${jfrFile.get().asFile.absolutePath}" - if (project.hasProperty('indy') && project.indy == 'true') { - jvmArgs '-Dgroovy.target.indy=true' - } - doFirst { profileDir.get().asFile.mkdirs() println "Running JMH with JFR profiling" @@ -356,10 +199,6 @@ tasks.register('jmhGcProfile', JavaExec) { args '.*' + project.benchInclude + '.*' } - if (project.hasProperty('indy') && project.indy == 'true') { - jvmArgs '-Dgroovy.target.indy=true' - } - doFirst { println "Running JMH with GC profiler" println "Results will be saved to: ${outputFile.get().asFile}"
