jamesfredley commented on code in PR #16169:
URL: https://github.com/apache/grails-core/pull/16169#discussion_r3930258560
##########
.github/workflows/benchmark.yml:
##########
@@ -264,6 +264,40 @@ jobs:
git worktree remove --force "$WORKTREE_ROOT/base" || true
git worktree remove --force "$WORKTREE_ROOT/head" || true
+ app-bench:
+ name: "App indy benchmarks"
+ if: >-
+ contains(github.event.pull_request.labels.*.name, 'performance') &&
+ (github.event.action != 'labeled' || github.event.label.name ==
'performance')
+ runs-on: ubuntu-24.04
+ env:
+ RESULT_DIR: ${{ github.workspace }}/app-bench-results
+ steps:
+ - name: "Checkout repository"
+ uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd #
v6.0.2
+ - name: "Setup JDK"
+ uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 #
v5.2.0
+ with:
+ distribution: liberica
+ java-version: 21
+ - name: "Setup Gradle"
+ uses:
gradle/actions/setup-gradle@50e97c2cd7a37755bbfafc9c5b7cafaece252f6e # v6.1.0
+ with:
+ cache-provider: basic # 'basic' uses the MIT-licensed, open-source
cache provider; the default 'enhanced' provider (v6+) is proprietary (Gradle
commercial Terms of Use)
+ develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
+ - name: "Verify JMH comparison tool"
+ run: ./gradlew :grails-benchmarks:test --max-workers=4
Review Comment:
Implemented in c17dc649c3. Added a Stop Gradle daemons step before app-bench
measurement, matching the JMH job.
##########
grails-benchmarks/src/report/groovy/org/apache/grails/benchmarks/report/AppIndyBench.groovy:
##########
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.grails.benchmarks.report
+
+import groovy.transform.CompileStatic
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+
+@CompileStatic
+class AppIndyBench {
+
+ static final List<App> APPS = [
+ new App('latency',
':grails-test-examples-latency:integrationTest',
'latencyapp.AppBenchFastPingSpec'),
+ new App('app1', ':grails-test-examples-app1:integrationTest',
'functionaltests.AppBenchInterceptorDemoSpec'),
+ new App('gsp-layout',
':grails-test-examples-gsp-layout:integrationTest',
'org.example.grails.layout.AppBenchDemoRenderTextSpec')
+ ].asImmutable()
+
+ static void main(String[] args) {
+ int exit = run(args, new WrapperGradleRunner())
+ if (exit != 0) {
+ System.exit(exit)
+ }
+ }
+
+ static int run(String[] args, GradleRunner runner) {
+ return run(args, runner, new GitHubComments(), System.getenv())
+ }
+
+ static int run(String[] args, GradleRunner runner, CommentPoster poster,
Map<String, String> environment) {
+ try {
+ Options options = parse(args)
+ Path noindyDir =
recreateDirectory(options.outputDir.resolve('noindy'))
+ Path indyDir = recreateDirectory(options.outputDir.resolve('indy'))
+
+ ['false', 'true'].each { String indy ->
+ Path modeDir = indy == 'true' ? indyDir : noindyDir
+ APPS.each { App app ->
+ Path out = modeDir.resolve(app.name + '.json')
+ runner.run(options.projectDir, gradleArgs(options, app,
indy, out))
+ if (!Files.isRegularFile(out)) {
+ throw new IllegalStateException("Missing result file:
${out}")
+ }
Review Comment:
Implemented in c17dc649c3. Missing-result message now names skipTests,
skipFunctionalTests, onlyCoreTests, ignored gated spec, and --tests mismatch.
##########
grails-benchmarks/src/report/groovy/org/apache/grails/benchmarks/report/AppIndyBench.groovy:
##########
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.grails.benchmarks.report
+
+import groovy.transform.CompileStatic
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+
+@CompileStatic
+class AppIndyBench {
+
+ static final List<App> APPS = [
+ new App('latency',
':grails-test-examples-latency:integrationTest',
'latencyapp.AppBenchFastPingSpec'),
+ new App('app1', ':grails-test-examples-app1:integrationTest',
'functionaltests.AppBenchInterceptorDemoSpec'),
+ new App('gsp-layout',
':grails-test-examples-gsp-layout:integrationTest',
'org.example.grails.layout.AppBenchDemoRenderTextSpec')
+ ].asImmutable()
+
+ static void main(String[] args) {
+ int exit = run(args, new WrapperGradleRunner())
+ if (exit != 0) {
+ System.exit(exit)
+ }
+ }
+
+ static int run(String[] args, GradleRunner runner) {
+ return run(args, runner, new GitHubComments(), System.getenv())
+ }
+
+ static int run(String[] args, GradleRunner runner, CommentPoster poster,
Map<String, String> environment) {
+ try {
+ Options options = parse(args)
+ Path noindyDir =
recreateDirectory(options.outputDir.resolve('noindy'))
+ Path indyDir = recreateDirectory(options.outputDir.resolve('indy'))
+
+ ['false', 'true'].each { String indy ->
+ Path modeDir = indy == 'true' ? indyDir : noindyDir
+ APPS.each { App app ->
+ Path out = modeDir.resolve(app.name + '.json')
+ runner.run(options.projectDir, gradleArgs(options, app,
indy, out))
+ if (!Files.isRegularFile(out)) {
+ throw new IllegalStateException("Missing result file:
${out}")
+ }
+ }
+ }
+
+ Path report = options.outputDir.resolve('indy-vs-noindy.md')
+ int compareExit = JmhCompare.run(
+ ['--base', noindyDir.toString(), '--head',
indyDir.toString(), '--output', report.toString()] as String[],
+ poster,
+ environment
+ )
+ if (compareExit != 0) {
+ return compareExit
+ }
+ appendStepSummary(report, environment)
+ return 0
+ } catch (Exception error) {
+ error.printStackTrace(System.err)
+ return 2
+ }
+ }
+
+ static List<String> gradleArgs(Options options, App app, String indy, Path
out) {
+ return [
+ '--no-daemon',
+ "--max-workers=${options.maxWorkers}".toString(),
+ app.task,
+ '--tests',
+ app.tests,
+ "-PgrailsIndy=${indy}".toString(),
+ '-PappBench=true',
+ "-PappBenchWarmup=${options.warmup}".toString(),
+ "-PappBenchSamples=${options.samples}".toString(),
+ "-PappBenchForks=${options.forks}".toString(),
+ "-PappBenchOut=${out.toAbsolutePath()}".toString()
+ ]
+ }
+
+ static Options parse(String[] args) {
+ Set<String> values = ['project-dir', 'output-dir', 'warmup',
'samples', 'forks', 'max-workers'] as Set<String>
+ Map<String, String> options = new LinkedHashMap<>()
+ for (int index = 0; index < args.length; index++) {
+ String option = args[index]
+ if (!option.startsWith('--')) {
+ throw new IllegalArgumentException("unknown option: ${option}")
+ }
+ String key = option.substring(2)
+ if (!values.contains(key)) {
+ throw new IllegalArgumentException("unknown option: --${key}")
+ }
+ if (index + 1 >= args.length) {
+ throw new IllegalArgumentException("missing value for
--${key}")
+ }
+ options.put(key, args[++index])
+ }
+ String projectDirValue = options.get('project-dir')
+ if (!projectDirValue) {
+ throw new IllegalArgumentException('--project-dir is required')
+ }
+ Path projectDir = Path.of(projectDirValue).toAbsolutePath().normalize()
+ Path outputDir = options.containsKey('output-dir')
+ ?
Path.of(options.get('output-dir')).toAbsolutePath().normalize()
+ : projectDir.resolve('build').resolve('app-bench')
+ return new Options(
+ projectDir,
+ outputDir,
+ parsePositiveInt(options.getOrDefault('warmup', '200'),
'warmup', true),
+ parsePositiveInt(options.getOrDefault('samples', '1000'),
'samples', false),
+ parsePositiveInt(options.getOrDefault('forks', '2'), 'forks',
false),
+ parsePositiveInt(options.getOrDefault('max-workers', '4'),
'max-workers', false)
+ )
+ }
+
+ private static int parsePositiveInt(String raw, String name, boolean
allowZero) {
+ int value
+ try {
+ value = Integer.parseInt(raw)
+ } catch (NumberFormatException ignored) {
+ throw new IllegalArgumentException("--${name} must be an integer")
+ }
+ if (value < 0 || (!allowZero && value < 1)) {
+ throw new IllegalArgumentException("--${name} must be ${allowZero
? '>= 0' : '>= 1'}")
+ }
+ return value
+ }
+
+ private static void appendStepSummary(Path report, Map<String, String>
environment) {
+ String summary = environment.get('GITHUB_STEP_SUMMARY')
+ if (!summary || !Files.isRegularFile(report)) {
+ return
+ }
+ Files.writeString(
+ Path.of(summary),
+ Files.readString(report, StandardCharsets.UTF_8),
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.APPEND
+ )
+ }
+
+ static Path recreateDirectory(Path directory) {
+ if (Files.exists(directory)) {
+ Files.walk(directory).withCloseable { stream ->
+ stream.sorted(Comparator.reverseOrder()).forEach { Path path
-> Files.deleteIfExists(path) }
+ }
+ }
+ return Files.createDirectories(directory)
+ }
+
+ @CompileStatic
+ static final class App {
+ final String name
+ final String task
+ final String tests
+
+ App(String name, String task, String tests) {
+ this.name = name
+ this.task = task
+ this.tests = tests
+ }
+ }
+
+ @CompileStatic
+ static final class Options {
+ final Path projectDir
+ final Path outputDir
+ final int warmup
+ final int samples
+ final int forks
+ final int maxWorkers
+
+ Options(Path projectDir, Path outputDir, int warmup, int samples, int
forks, int maxWorkers) {
+ this.projectDir = projectDir
+ this.outputDir = outputDir
+ this.warmup = warmup
+ this.samples = samples
+ this.forks = forks
+ this.maxWorkers = maxWorkers
+ }
+ }
+
+ @CompileStatic
+ interface GradleRunner {
+ void run(Path projectDir, List<String> args)
+ }
+
+ @CompileStatic
+ static final class WrapperGradleRunner implements GradleRunner {
+ @Override
+ void run(Path projectDir, List<String> args) {
+ Path javaHome = Path.of(System.getProperty('java.home'))
+ List<String> command = commandLine(javaHome, projectDir, args)
+ ProcessBuilder processBuilder = new ProcessBuilder(command)
+ processBuilder.directory(projectDir.toFile())
+ processBuilder.inheritIO()
+ processBuilder.environment().put('JAVA_HOME', javaHome.toString())
+ Process process = processBuilder.start()
+ int exit = process.waitFor()
+ if (exit != 0) {
Review Comment:
Implemented in c17dc649c3. Nested Gradle waitFor now times out after 90
minutes and destroyForcibly; each nested build uses a unique
--project-cache-dir under the owned run directory.
##########
grails-benchmarks/src/report/groovy/org/apache/grails/benchmarks/report/AppIndyBench.groovy:
##########
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.grails.benchmarks.report
+
+import groovy.transform.CompileStatic
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+
+@CompileStatic
+class AppIndyBench {
+
+ static final List<App> APPS = [
+ new App('latency',
':grails-test-examples-latency:integrationTest',
'latencyapp.AppBenchFastPingSpec'),
+ new App('app1', ':grails-test-examples-app1:integrationTest',
'functionaltests.AppBenchInterceptorDemoSpec'),
+ new App('gsp-layout',
':grails-test-examples-gsp-layout:integrationTest',
'org.example.grails.layout.AppBenchDemoRenderTextSpec')
+ ].asImmutable()
+
+ static void main(String[] args) {
+ int exit = run(args, new WrapperGradleRunner())
+ if (exit != 0) {
+ System.exit(exit)
+ }
+ }
+
+ static int run(String[] args, GradleRunner runner) {
+ return run(args, runner, new GitHubComments(), System.getenv())
+ }
+
+ static int run(String[] args, GradleRunner runner, CommentPoster poster,
Map<String, String> environment) {
+ try {
+ Options options = parse(args)
+ Path noindyDir =
recreateDirectory(options.outputDir.resolve('noindy'))
+ Path indyDir = recreateDirectory(options.outputDir.resolve('indy'))
+
+ ['false', 'true'].each { String indy ->
+ Path modeDir = indy == 'true' ? indyDir : noindyDir
+ APPS.each { App app ->
+ Path out = modeDir.resolve(app.name + '.json')
+ runner.run(options.projectDir, gradleArgs(options, app,
indy, out))
+ if (!Files.isRegularFile(out)) {
+ throw new IllegalStateException("Missing result file:
${out}")
+ }
+ }
+ }
+
+ Path report = options.outputDir.resolve('indy-vs-noindy.md')
+ int compareExit = JmhCompare.run(
+ ['--base', noindyDir.toString(), '--head',
indyDir.toString(), '--output', report.toString()] as String[],
+ poster,
+ environment
+ )
+ if (compareExit != 0) {
+ return compareExit
+ }
+ appendStepSummary(report, environment)
+ return 0
+ } catch (Exception error) {
+ error.printStackTrace(System.err)
+ return 2
+ }
+ }
+
+ static List<String> gradleArgs(Options options, App app, String indy, Path
out) {
+ return [
+ '--no-daemon',
+ "--max-workers=${options.maxWorkers}".toString(),
+ app.task,
+ '--tests',
+ app.tests,
+ "-PgrailsIndy=${indy}".toString(),
+ '-PappBench=true',
+ "-PappBenchWarmup=${options.warmup}".toString(),
+ "-PappBenchSamples=${options.samples}".toString(),
+ "-PappBenchForks=${options.forks}".toString(),
+ "-PappBenchOut=${out.toAbsolutePath()}".toString()
+ ]
+ }
+
+ static Options parse(String[] args) {
+ Set<String> values = ['project-dir', 'output-dir', 'warmup',
'samples', 'forks', 'max-workers'] as Set<String>
+ Map<String, String> options = new LinkedHashMap<>()
+ for (int index = 0; index < args.length; index++) {
+ String option = args[index]
+ if (!option.startsWith('--')) {
+ throw new IllegalArgumentException("unknown option: ${option}")
+ }
+ String key = option.substring(2)
+ if (!values.contains(key)) {
+ throw new IllegalArgumentException("unknown option: --${key}")
+ }
+ if (index + 1 >= args.length) {
+ throw new IllegalArgumentException("missing value for
--${key}")
+ }
+ options.put(key, args[++index])
+ }
+ String projectDirValue = options.get('project-dir')
+ if (!projectDirValue) {
+ throw new IllegalArgumentException('--project-dir is required')
+ }
+ Path projectDir = Path.of(projectDirValue).toAbsolutePath().normalize()
+ Path outputDir = options.containsKey('output-dir')
+ ?
Path.of(options.get('output-dir')).toAbsolutePath().normalize()
+ : projectDir.resolve('build').resolve('app-bench')
+ return new Options(
+ projectDir,
+ outputDir,
+ parsePositiveInt(options.getOrDefault('warmup', '200'),
'warmup', true),
+ parsePositiveInt(options.getOrDefault('samples', '1000'),
'samples', false),
+ parsePositiveInt(options.getOrDefault('forks', '2'), 'forks',
false),
+ parsePositiveInt(options.getOrDefault('max-workers', '4'),
'max-workers', false)
+ )
+ }
+
+ private static int parsePositiveInt(String raw, String name, boolean
allowZero) {
+ int value
+ try {
+ value = Integer.parseInt(raw)
+ } catch (NumberFormatException ignored) {
+ throw new IllegalArgumentException("--${name} must be an integer")
+ }
+ if (value < 0 || (!allowZero && value < 1)) {
+ throw new IllegalArgumentException("--${name} must be ${allowZero
? '>= 0' : '>= 1'}")
+ }
+ return value
+ }
+
+ private static void appendStepSummary(Path report, Map<String, String>
environment) {
+ String summary = environment.get('GITHUB_STEP_SUMMARY')
+ if (!summary || !Files.isRegularFile(report)) {
+ return
+ }
+ Files.writeString(
+ Path.of(summary),
+ Files.readString(report, StandardCharsets.UTF_8),
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.APPEND
+ )
+ }
+
+ static Path recreateDirectory(Path directory) {
+ if (Files.exists(directory)) {
+ Files.walk(directory).withCloseable { stream ->
+ stream.sorted(Comparator.reverseOrder()).forEach { Path path
-> Files.deleteIfExists(path) }
+ }
+ }
+ return Files.createDirectories(directory)
+ }
+
+ @CompileStatic
+ static final class App {
+ final String name
+ final String task
+ final String tests
+
+ App(String name, String task, String tests) {
+ this.name = name
+ this.task = task
+ this.tests = tests
+ }
+ }
+
+ @CompileStatic
+ static final class Options {
+ final Path projectDir
+ final Path outputDir
+ final int warmup
+ final int samples
+ final int forks
+ final int maxWorkers
+
+ Options(Path projectDir, Path outputDir, int warmup, int samples, int
forks, int maxWorkers) {
+ this.projectDir = projectDir
+ this.outputDir = outputDir
+ this.warmup = warmup
+ this.samples = samples
+ this.forks = forks
+ this.maxWorkers = maxWorkers
+ }
+ }
+
+ @CompileStatic
+ interface GradleRunner {
+ void run(Path projectDir, List<String> args)
+ }
+
+ @CompileStatic
+ static final class WrapperGradleRunner implements GradleRunner {
+ @Override
+ void run(Path projectDir, List<String> args) {
+ Path javaHome = Path.of(System.getProperty('java.home'))
+ List<String> command = commandLine(javaHome, projectDir, args)
+ ProcessBuilder processBuilder = new ProcessBuilder(command)
+ processBuilder.directory(projectDir.toFile())
+ processBuilder.inheritIO()
+ processBuilder.environment().put('JAVA_HOME', javaHome.toString())
+ Process process = processBuilder.start()
+ int exit = process.waitFor()
+ if (exit != 0) {
+ throw new IllegalStateException("Nested Gradle exited ${exit}:
${command}")
+ }
+ }
+
+ static List<String> commandLine(Path javaHome, Path projectDir,
List<String> args) {
+ Path java = javaExecutable(javaHome)
+ Path wrapperJar =
projectDir.resolve('gradle').resolve('wrapper').resolve('gradle-wrapper.jar')
+ if (!Files.isRegularFile(java)) {
+ throw new IllegalStateException("Java executable not found:
${java}")
+ }
+ if (!Files.isRegularFile(wrapperJar)) {
+ throw new IllegalStateException("Gradle wrapper jar not found:
${wrapperJar}")
+ }
+ List<String> command = new ArrayList<>()
+ command.add(java.toString())
+ command.add('-cp')
+ command.add(wrapperJar.toString())
+ command.add('org.gradle.wrapper.GradleWrapperMain')
+ command.addAll(args)
Review Comment:
Implemented in c17dc649c3. DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS are
forwarded onto the wrapper JVM before -cp.
##########
grails-benchmarks/src/reportTest/groovy/org/apache/grails/benchmarks/report/AppIndyBenchSpec.groovy:
##########
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.grails.benchmarks.report
+
+import spock.lang.Specification
+import spock.lang.TempDir
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+
+class AppIndyBenchSpec extends Specification {
+
+ @TempDir
+ Path temporaryDirectory
+
+ void 'parse requires project-dir and applies defaults'() {
+ when:
+ AppIndyBench.Options options = AppIndyBench.parse(['--project-dir',
temporaryDirectory.toString()] as String[])
+
+ then:
+ options.projectDir == temporaryDirectory.toAbsolutePath().normalize()
+ options.outputDir ==
options.projectDir.resolve('build').resolve('app-bench')
+ options.warmup == 200
+ options.samples == 1000
+ options.forks == 2
+ options.maxWorkers == 4
+ }
+
+ void 'parse rejects a missing project-dir'() {
+ when:
+ AppIndyBench.parse(['--warmup', '10'] as String[])
+
+ then:
+ IllegalArgumentException error = thrown()
+ error.message.contains('--project-dir is required')
+ }
+
+ void 'run invokes six nested builds then compares directory results'() {
+ given:
+ Path outputDir = temporaryDirectory.resolve('out')
+ Path summary = temporaryDirectory.resolve('summary.md')
+ List<List<String>> invocations = []
+ AppIndyBench.GradleRunner runner = { Path projectDir, List<String>
args ->
+ invocations.add(args)
+ writeDummyResult(args)
+ } as AppIndyBench.GradleRunner
+
+ when:
+ int exit = AppIndyBench.run(
+ [
+ '--project-dir', temporaryDirectory.toString(),
+ '--output-dir', outputDir.toString(),
+ '--warmup', '80',
+ '--samples', '300',
+ '--forks', '2',
+ '--max-workers', '3'
+ ] as String[],
+ runner,
+ new GitHubComments(),
+ [GITHUB_STEP_SUMMARY: summary.toString()]
+ )
+
+ then:
+ exit == 0
+ invocations.size() == 6
+
invocations[0].contains(':grails-test-examples-latency:integrationTest')
+ invocations[0].contains('latencyapp.AppBenchFastPingSpec')
+ invocations[0].contains('-PgrailsIndy=false')
+ invocations[0].contains('-PappBench=true')
Review Comment:
Implemented in c17dc649c3. Specs now assert grailsIndy=false writes under
run/noindy and true under run/indy, with distinguishable dummy scores so an
inverted ternary fails.
##########
grails-testing-support-http-client/src/testFixtures/groovy/org/apache/grails/testing/http/client/bench/AppHttpBench.groovy:
##########
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.grails.testing.http.client.bench
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.Paths
+
+import groovy.json.JsonOutput
+import groovy.transform.CompileStatic
+
+/**
+ * App-level HTTP microbench helper that emits JMH-compatible JSON so results
can be compared
+ * with {@code :grails-benchmarks:jmhCompare} using the same methodology as
the framework JMH suite.
+ *
+ * <p>Measurement model (deliberately simple and reproducible on one machine):
+ * <ul>
+ * <li>warm up the full Spring Boot stack with {@code warmup} requests
(discarded)</li>
+ * <li>collect {@code samples} timed requests as one raw series</li>
+ * <li>split the series into {@code forks} equal chunks to mimic JMH
multi-fork rawData shape</li>
+ * <li>report mean ns/op with a simple standard-error-based scoreError</li>
+ * </ul>
+ *
+ * <p>Enable gated specs with {@code -PappBench=true}. Optional properties:
+ * {@code appBenchWarmup}, {@code appBenchSamples}, {@code appBenchForks},
{@code appBenchOut}.
+ */
+@CompileStatic
+final class AppHttpBench {
+
+ private AppHttpBench() {
+ }
+
+ static boolean enabled() {
+ Boolean.getBoolean('app.bench') ||
Boolean.parseBoolean(System.getProperty('appBench', 'false'))
+ }
+
+ static int warmupCount() {
+ Integer.getInteger('app.bench.warmup',
Integer.getInteger('appBenchWarmup', 200))
+ }
Review Comment:
Implemented in c17dc649c3. Integer.getInteger is gone; properties parse as
decimal ints and reject unparsable/octal-looking values.
##########
grails-benchmarks/src/report/groovy/org/apache/grails/benchmarks/report/AppIndyBench.groovy:
##########
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.grails.benchmarks.report
+
+import groovy.transform.CompileStatic
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+
+@CompileStatic
+class AppIndyBench {
+
+ static final List<App> APPS = [
+ new App('latency',
':grails-test-examples-latency:integrationTest',
'latencyapp.AppBenchFastPingSpec'),
+ new App('app1', ':grails-test-examples-app1:integrationTest',
'functionaltests.AppBenchInterceptorDemoSpec'),
+ new App('gsp-layout',
':grails-test-examples-gsp-layout:integrationTest',
'org.example.grails.layout.AppBenchDemoRenderTextSpec')
+ ].asImmutable()
+
+ static void main(String[] args) {
+ int exit = run(args, new WrapperGradleRunner())
+ if (exit != 0) {
+ System.exit(exit)
+ }
+ }
+
+ static int run(String[] args, GradleRunner runner) {
+ return run(args, runner, new GitHubComments(), System.getenv())
+ }
+
+ static int run(String[] args, GradleRunner runner, CommentPoster poster,
Map<String, String> environment) {
+ try {
+ Options options = parse(args)
+ Path noindyDir =
recreateDirectory(options.outputDir.resolve('noindy'))
+ Path indyDir = recreateDirectory(options.outputDir.resolve('indy'))
+
+ ['false', 'true'].each { String indy ->
+ Path modeDir = indy == 'true' ? indyDir : noindyDir
+ APPS.each { App app ->
+ Path out = modeDir.resolve(app.name + '.json')
+ runner.run(options.projectDir, gradleArgs(options, app,
indy, out))
+ if (!Files.isRegularFile(out)) {
+ throw new IllegalStateException("Missing result file:
${out}")
+ }
+ }
+ }
+
+ Path report = options.outputDir.resolve('indy-vs-noindy.md')
+ int compareExit = JmhCompare.run(
+ ['--base', noindyDir.toString(), '--head',
indyDir.toString(), '--output', report.toString()] as String[],
+ poster,
+ environment
+ )
+ if (compareExit != 0) {
+ return compareExit
+ }
+ appendStepSummary(report, environment)
+ return 0
+ } catch (Exception error) {
+ error.printStackTrace(System.err)
+ return 2
+ }
+ }
+
+ static List<String> gradleArgs(Options options, App app, String indy, Path
out) {
+ return [
+ '--no-daemon',
+ "--max-workers=${options.maxWorkers}".toString(),
+ app.task,
+ '--tests',
+ app.tests,
+ "-PgrailsIndy=${indy}".toString(),
+ '-PappBench=true',
+ "-PappBenchWarmup=${options.warmup}".toString(),
+ "-PappBenchSamples=${options.samples}".toString(),
+ "-PappBenchForks=${options.forks}".toString(),
+ "-PappBenchOut=${out.toAbsolutePath()}".toString()
+ ]
+ }
+
+ static Options parse(String[] args) {
+ Set<String> values = ['project-dir', 'output-dir', 'warmup',
'samples', 'forks', 'max-workers'] as Set<String>
+ Map<String, String> options = new LinkedHashMap<>()
+ for (int index = 0; index < args.length; index++) {
+ String option = args[index]
+ if (!option.startsWith('--')) {
+ throw new IllegalArgumentException("unknown option: ${option}")
+ }
+ String key = option.substring(2)
+ if (!values.contains(key)) {
+ throw new IllegalArgumentException("unknown option: --${key}")
+ }
+ if (index + 1 >= args.length) {
+ throw new IllegalArgumentException("missing value for
--${key}")
+ }
+ options.put(key, args[++index])
+ }
+ String projectDirValue = options.get('project-dir')
+ if (!projectDirValue) {
+ throw new IllegalArgumentException('--project-dir is required')
+ }
+ Path projectDir = Path.of(projectDirValue).toAbsolutePath().normalize()
+ Path outputDir = options.containsKey('output-dir')
+ ?
Path.of(options.get('output-dir')).toAbsolutePath().normalize()
+ : projectDir.resolve('build').resolve('app-bench')
+ return new Options(
+ projectDir,
+ outputDir,
+ parsePositiveInt(options.getOrDefault('warmup', '200'),
'warmup', true),
+ parsePositiveInt(options.getOrDefault('samples', '1000'),
'samples', false),
+ parsePositiveInt(options.getOrDefault('forks', '2'), 'forks',
false),
+ parsePositiveInt(options.getOrDefault('max-workers', '4'),
'max-workers', false)
+ )
+ }
+
+ private static int parsePositiveInt(String raw, String name, boolean
allowZero) {
+ int value
+ try {
+ value = Integer.parseInt(raw)
+ } catch (NumberFormatException ignored) {
+ throw new IllegalArgumentException("--${name} must be an integer")
+ }
+ if (value < 0 || (!allowZero && value < 1)) {
+ throw new IllegalArgumentException("--${name} must be ${allowZero
? '>= 0' : '>= 1'}")
+ }
+ return value
+ }
+
+ private static void appendStepSummary(Path report, Map<String, String>
environment) {
+ String summary = environment.get('GITHUB_STEP_SUMMARY')
+ if (!summary || !Files.isRegularFile(report)) {
+ return
+ }
+ Files.writeString(
+ Path.of(summary),
+ Files.readString(report, StandardCharsets.UTF_8),
+ StandardCharsets.UTF_8,
+ StandardOpenOption.CREATE,
+ StandardOpenOption.APPEND
+ )
+ }
+
+ static Path recreateDirectory(Path directory) {
+ if (Files.exists(directory)) {
Review Comment:
Implemented in c17dc649c3. Mode trees and the report live under an owned
outputDir/run/ directory that we fully recreate. Caller-owned noindy/indy
siblings at the output root are left alone. Stale indy-vs-noindy.md in run/ is
removed.
##########
grails-benchmarks/src/report/groovy/org/apache/grails/benchmarks/report/AppIndyBench.groovy:
##########
@@ -0,0 +1,244 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.grails.benchmarks.report
+
+import groovy.transform.CompileStatic
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+
+@CompileStatic
+class AppIndyBench {
+
+ static final List<App> APPS = [
+ new App('latency',
':grails-test-examples-latency:integrationTest',
'latencyapp.AppBenchFastPingSpec'),
+ new App('app1', ':grails-test-examples-app1:integrationTest',
'functionaltests.AppBenchInterceptorDemoSpec'),
+ new App('gsp-layout',
':grails-test-examples-gsp-layout:integrationTest',
'org.example.grails.layout.AppBenchDemoRenderTextSpec')
+ ].asImmutable()
+
+ static void main(String[] args) {
+ int exit = run(args, new WrapperGradleRunner())
+ if (exit != 0) {
+ System.exit(exit)
+ }
+ }
+
+ static int run(String[] args, GradleRunner runner) {
+ return run(args, runner, new GitHubComments(), System.getenv())
+ }
+
+ static int run(String[] args, GradleRunner runner, CommentPoster poster,
Map<String, String> environment) {
+ try {
+ Options options = parse(args)
+ Path noindyDir =
recreateDirectory(options.outputDir.resolve('noindy'))
+ Path indyDir = recreateDirectory(options.outputDir.resolve('indy'))
+
+ ['false', 'true'].each { String indy ->
+ Path modeDir = indy == 'true' ? indyDir : noindyDir
+ APPS.each { App app ->
+ Path out = modeDir.resolve(app.name + '.json')
+ runner.run(options.projectDir, gradleArgs(options, app,
indy, out))
+ if (!Files.isRegularFile(out)) {
Review Comment:
Implemented in c17dc649c3. Left as six nested builds for this pass so each
integrationTest keeps its own AppBenchOut and worker cap; combining three apps
in one Gradle invocation is a follow-up because it needs per-task output
routing to avoid sample contamination.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
