jamesfredley commented on code in PR #16169: URL: https://github.com/apache/grails-core/pull/16169#discussion_r3930257761
########## grails-benchmarks/src/report/groovy/org/apache/grails/benchmarks/report/AppIndyBench.groovy: ########## @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.grails.benchmarks.report + +import groovy.transform.CompileStatic + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption + +@CompileStatic +class AppIndyBench { + + static final List<App> APPS = [ + new App('latency', ':grails-test-examples-latency:integrationTest', 'latencyapp.AppBenchFastPingSpec'), + new App('app1', ':grails-test-examples-app1:integrationTest', 'functionaltests.AppBenchInterceptorDemoSpec'), + new App('gsp-layout', ':grails-test-examples-gsp-layout:integrationTest', 'org.example.grails.layout.AppBenchDemoRenderTextSpec') + ].asImmutable() + + static void main(String[] args) { + int exit = run(args, new WrapperGradleRunner()) + if (exit != 0) { + System.exit(exit) + } + } + + static int run(String[] args, GradleRunner runner) { + return run(args, runner, new GitHubComments(), System.getenv()) + } + + static int run(String[] args, GradleRunner runner, CommentPoster poster, Map<String, String> environment) { + try { + Options options = parse(args) + Path noindyDir = recreateDirectory(options.outputDir.resolve('noindy')) + Path indyDir = recreateDirectory(options.outputDir.resolve('indy')) + + ['false', 'true'].each { String indy -> + Path modeDir = indy == 'true' ? indyDir : noindyDir Review Comment: Implemented in c17dc649c3. Modes now interleave per app (even: noindy then indy, odd: reverse), so first-vs-second drift is not always on the indy side. ########## grails-testing-support-http-client/src/testFixtures/groovy/org/apache/grails/testing/http/client/bench/AppHttpBench.groovy: ########## @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.testing.http.client.bench + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic + +/** + * App-level HTTP microbench helper that emits JMH-compatible JSON so results can be compared + * with {@code :grails-benchmarks:jmhCompare} using the same methodology as the framework JMH suite. + * + * <p>Measurement model (deliberately simple and reproducible on one machine): + * <ul> + * <li>warm up the full Spring Boot stack with {@code warmup} requests (discarded)</li> + * <li>collect {@code samples} timed requests as one raw series</li> + * <li>split the series into {@code forks} equal chunks to mimic JMH multi-fork rawData shape</li> + * <li>report mean ns/op with a simple standard-error-based scoreError</li> + * </ul> + * + * <p>Enable gated specs with {@code -PappBench=true}. Optional properties: + * {@code appBenchWarmup}, {@code appBenchSamples}, {@code appBenchForks}, {@code appBenchOut}. + */ +@CompileStatic +final class AppHttpBench { + + private AppHttpBench() { + } + + static boolean enabled() { + Boolean.getBoolean('app.bench') || Boolean.parseBoolean(System.getProperty('appBench', 'false')) + } + + static int warmupCount() { + Integer.getInteger('app.bench.warmup', Integer.getInteger('appBenchWarmup', 200)) + } + + static int sampleCount() { + Integer.getInteger('app.bench.samples', Integer.getInteger('appBenchSamples', 1000)) + } + + static int forkCount() { + Integer.getInteger('app.bench.forks', Integer.getInteger('appBenchForks', 2)) + } + + static Path outputPath(String defaultFileName) { + String configured = System.getProperty('app.bench.out', System.getProperty('appBenchOut', '')) + if (configured) { + return Paths.get(configured) + } + Path dir = Paths.get('build', 'app-bench') + Files.createDirectories(dir) + return dir.resolve(defaultFileName) + } + + /** + * Time a single request body. The closure must perform the HTTP call and assert success. + * + * @return elapsed nanoseconds + */ + static long timeNanos(Closure<?> request) { + long start = System.nanoTime() + request.call() + return System.nanoTime() - start + } + + /** + * Warm up, sample, and append one JMH-shaped benchmark entry to {@code out}. + * + * @param benchmark fully-qualified-style name, e.g. {@code appbench.latency.FastPing.httpGet} + * @param request closure that performs one successful request + */ + static void measureAndWrite(String benchmark, Path out, Closure<?> request) { + int warmup = Math.max(0, warmupCount()) + int samples = sampleCount() + if (samples < 1) { + throw new IllegalArgumentException("app.bench.samples must be >= 1, was ${samples}") + } + + for (int i = 0; i < warmup; i++) { + request.call() + } + + double[] values = new double[samples] + for (int i = 0; i < samples; i++) { + values[i] = (double) timeNanos(request) + } + + Map<String, Object> entry = toJmhEntry(benchmark, values, forkCount()) + appendEntry(out, entry) + } + + static Map<String, Object> toJmhEntry(String benchmark, double[] values, int forks) { + if (values == null || values.length == 0) { + throw new IllegalArgumentException('values must contain at least one sample') + } + double mean = mean(values) + double stdev = stdev(values, mean) + double scoreError = stdev * 1.96d / Math.sqrt((double) values.length) + Review Comment: Implemented in c17dc649c3. scoreError is a 95% t-interval over fork means; JSON also emits org.apache.grails.benchmarks.ruler.AppBenchCpu.measure so runner-health gates apply. ########## grails-testing-support-http-client/src/main/groovy/org/apache/grails/testing/http/client/bench/AppHttpBench.groovy: ########## @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.testing.http.client.bench + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic + +/** + * App-level HTTP microbench helper that emits JMH-compatible JSON so results can be compared + * with {@code :grails-benchmarks:jmhCompare} using the same methodology as the framework JMH suite. + * + * <p>Measurement model (deliberately simple and reproducible on one machine): + * <ul> + * <li>warm up the full Spring Boot stack with {@code warmup} requests (discarded)</li> + * <li>collect {@code samples} timed requests as one raw series</li> + * <li>split the series into {@code forks} equal chunks to mimic JMH multi-fork rawData shape</li> + * <li>report mean ns/op with a simple standard-error-based scoreError</li> + * </ul> + * + * <p>Enable gated specs with {@code -PappBench=true}. Optional properties: + * {@code appBenchWarmup}, {@code appBenchSamples}, {@code appBenchForks}, {@code appBenchOut}. + */ +@CompileStatic +final class AppHttpBench { + + private AppHttpBench() { + } + + static boolean enabled() { + Boolean.getBoolean('app.bench') || Boolean.parseBoolean(System.getProperty('appBench', 'false')) + } + + static int warmupCount() { + Integer.getInteger('app.bench.warmup', Integer.getInteger('appBenchWarmup', 200)) + } + + static int sampleCount() { + Integer.getInteger('app.bench.samples', Integer.getInteger('appBenchSamples', 1000)) + } + + static int forkCount() { + Integer.getInteger('app.bench.forks', Integer.getInteger('appBenchForks', 2)) + } + + static Path outputPath(String defaultFileName) { + String configured = System.getProperty('app.bench.out', System.getProperty('appBenchOut', '')) + if (configured) { + return Paths.get(configured) + } + Path dir = Paths.get('build', 'app-bench') + Files.createDirectories(dir) + return dir.resolve(defaultFileName) + } + + /** + * Time a single request body. The closure must perform the HTTP call and assert success. + * + * @return elapsed nanoseconds + */ + static long timeNanos(Closure<?> request) { + long start = System.nanoTime() + request.call() + return System.nanoTime() - start + } + + /** + * Warm up, sample, and append one JMH-shaped benchmark entry to {@code out}. + * + * @param benchmark fully-qualified-style name, e.g. {@code appbench.latency.FastPing.httpGet} + * @param request closure that performs one successful request + */ + static void measureAndWrite(String benchmark, Path out, Closure<?> request) { + int warmup = Math.max(0, warmupCount()) + int samples = sampleCount() + if (samples < 1) { + throw new IllegalArgumentException("app.bench.samples must be >= 1, was ${samples}") + } + + for (int i = 0; i < warmup; i++) { + request.call() + } + + double[] values = new double[samples] + for (int i = 0; i < samples; i++) { + values[i] = (double) timeNanos(request) + } + + Map<String, Object> entry = toJmhEntry(benchmark, values, forkCount()) + appendEntry(out, entry) + } + + static Map<String, Object> toJmhEntry(String benchmark, double[] values, int forks) { + if (values == null || values.length == 0) { + throw new IllegalArgumentException('values must contain at least one sample') + } + double mean = mean(values) + double stdev = stdev(values, mean) + double scoreError = stdev * 1.96d / Math.sqrt((double) values.length) + + int forkCount = Math.min(Math.max(1, forks), values.length) + int perFork = Math.max(1, values.length.intdiv(forkCount)) + List<List<Double>> rawData = new ArrayList<>(forkCount) + int offset = 0 + for (int f = 0; f < forkCount; f++) { + int end = (f == forkCount - 1) ? values.length : Math.min(values.length, offset + perFork) + List<Double> chunk = new ArrayList<>(Math.max(0, end - offset)) + for (int i = offset; i < end; i++) { + chunk.add(values[i]) + } + rawData.add(chunk) + offset = end + } + + Map<String, Object> percentiles = new LinkedHashMap<>() + double[] sorted = Arrays.copyOf(values, values.length) + Arrays.sort(sorted) + percentiles.put('0.0', sorted[0]) + percentiles.put('50.0', percentile(sorted, 0.50d)) + percentiles.put('90.0', percentile(sorted, 0.90d)) + percentiles.put('95.0', percentile(sorted, 0.95d)) + percentiles.put('99.0', percentile(sorted, 0.99d)) + percentiles.put('100.0', sorted[sorted.length - 1]) + + Map<String, Object> primary = new LinkedHashMap<>() + primary.put('score', mean) + primary.put('scoreError', scoreError) + primary.put('scoreConfidence', [mean - scoreError, mean + scoreError]) + primary.put('scorePercentiles', percentiles) + primary.put('scoreUnit', 'ns/op') + primary.put('rawData', rawData) + + Map<String, Object> entry = new LinkedHashMap<>() + entry.put('jmhVersion', 'app-bench-1.0') + entry.put('benchmark', benchmark) + entry.put('mode', 'avgt') + entry.put('threads', 1) + entry.put('forks', forkCount) + entry.put('jdkVersion', System.getProperty('java.version', 'unknown')) + entry.put('vmName', System.getProperty('java.vm.name', 'unknown')) + entry.put('vmVersion', System.getProperty('java.vm.version', 'unknown')) + entry.put('warmupIterations', 1) + entry.put('warmupTime', "${warmupCount()} reqs") + entry.put('measurementIterations', values.length) + entry.put('measurementTime', '1 req') + entry.put('primaryMetric', primary) + entry.put('secondaryMetrics', Collections.emptyMap()) + return entry + } + + static void appendEntry(Path out, Map<String, Object> entry) { + List<Object> entries = new ArrayList<>() + if (Files.exists(out)) { + String existing = Files.readString(out, StandardCharsets.UTF_8).trim() + if (existing.startsWith('[')) { + Object parsed = new groovy.json.JsonSlurper().parseText(existing) Review Comment: Implemented in c17dc649c3. measureAndWrite replaces the output file. Truncated JSON and non-array existing content fail loudly. ########## gradle/app-bench-config.gradle: ########## @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// Experimental app-level indy A/B harness wiring. +// Enable with -PappBench=true (and usually --tests '*AppBench*'). +// Optional: -PappBenchWarmup=200 -PappBenchSamples=1000 -PappBenchForks=2 -PappBenchOut=<json> + +def appBenchEnabled = providers.gradleProperty('appBench').map { Boolean.parseBoolean(it) }.orElse(false) +def appBenchWarmup = providers.gradleProperty('appBenchWarmup').orElse('200') +def appBenchSamples = providers.gradleProperty('appBenchSamples').orElse('1000') +def appBenchForks = providers.gradleProperty('appBenchForks').orElse('2') +def appBenchOut = providers.gradleProperty('appBenchOut').orElse('') + +tasks.withType(Test).configureEach { Test test -> + test.systemProperty('app.bench', String.valueOf(appBenchEnabled.get())) + test.systemProperty('app.bench.warmup', appBenchWarmup.get()) + test.systemProperty('app.bench.samples', appBenchSamples.get()) + test.systemProperty('app.bench.forks', appBenchForks.get()) + if (appBenchOut.get()) { + test.systemProperty('app.bench.out', appBenchOut.get()) + test.outputs.file(appBenchOut.get()) + } Review Comment: Implemented in c17dc649c3. app-bench-config now wires only integrationTest and only when -PappBench=true, so ordinary tests do not share AppBenchOut or pick up app.bench.* inputs. ########## grails-testing-support-http-client/src/testFixtures/groovy/org/apache/grails/testing/http/client/bench/AppHttpBench.groovy: ########## @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.grails.testing.http.client.bench + +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths + +import groovy.json.JsonOutput +import groovy.transform.CompileStatic + +/** + * App-level HTTP microbench helper that emits JMH-compatible JSON so results can be compared + * with {@code :grails-benchmarks:jmhCompare} using the same methodology as the framework JMH suite. + * + * <p>Measurement model (deliberately simple and reproducible on one machine): + * <ul> + * <li>warm up the full Spring Boot stack with {@code warmup} requests (discarded)</li> + * <li>collect {@code samples} timed requests as one raw series</li> + * <li>split the series into {@code forks} equal chunks to mimic JMH multi-fork rawData shape</li> + * <li>report mean ns/op with a simple standard-error-based scoreError</li> + * </ul> + * + * <p>Enable gated specs with {@code -PappBench=true}. Optional properties: + * {@code appBenchWarmup}, {@code appBenchSamples}, {@code appBenchForks}, {@code appBenchOut}. + */ +@CompileStatic +final class AppHttpBench { + Review Comment: Implemented in c17dc649c3. AppHttpBench moved to grails-testing-support-http-client test fixtures; example apps depend on testFixtures(project(':grails-testing-support-http-client')). ########## .github/workflows/benchmark.yml: ########## @@ -264,6 +264,40 @@ jobs: git worktree remove --force "$WORKTREE_ROOT/base" || true git worktree remove --force "$WORKTREE_ROOT/head" || true + app-bench: + name: "App indy benchmarks" + if: >- + contains(github.event.pull_request.labels.*.name, 'performance') && + (github.event.action != 'labeled' || github.event.label.name == 'performance') + runs-on: ubuntu-24.04 + env: + RESULT_DIR: ${{ github.workspace }}/app-bench-results + steps: + - name: "Checkout repository" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: "Setup JDK" + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 + with: + distribution: liberica + java-version: 21 + - name: "Setup Gradle" + uses: gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0 + with: + cache-provider: basic # 'basic' uses the MIT-licensed, open-source cache provider; the default 'enhanced' provider (v6+) is proprietary (Gradle commercial Terms of Use) + develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} + - name: "Verify JMH comparison tool" + run: ./gradlew :grails-benchmarks:test --max-workers=4 + - name: "Run app-level indy A/B benches" + timeout-minutes: 180 + run: ./gradlew --no-daemon :grails-benchmarks:appIndyBench -PappBenchWarmup=80 -PappBenchSamples=300 -PappBenchForks=2 -PappBenchOutDir="$RESULT_DIR" --max-workers=4 Review Comment: Implemented in c17dc649c3. Nested failures no longer abort before compare. We compare whatever JSON exists, write a fallback GITHUB_STEP_SUMMARY, and the workflow measure step is continue-on-error so artifacts still upload. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
