This is an automated email from the ASF dual-hosted git repository. aldettinger pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-quarkus.git
The following commit(s) were added to refs/heads/main by this push: new af65c9b436 Merge the performance regression detection prototype #3905 af65c9b436 is described below commit af65c9b436e2dad5088fcc658972811c5d0714e7 Author: aldettinger <aldettin...@gmail.com> AuthorDate: Thu Aug 4 11:55:53 2022 +0200 Merge the performance regression detection prototype #3905 --- pom.xml | 3 +- tooling/perf-regression/README.md | 14 + .../.mvn/wrapper/MavenWrapperDownloader.java | 118 ++++++++ .../.mvn/wrapper/maven-wrapper.properties | 18 ++ .../cq-perf-regression-scenario.hf.yaml | 39 +++ .../cq-perf-regression-sample-base/mvnw | 310 +++++++++++++++++++++ .../cq-perf-regression-sample-base/mvnw.cmd | 180 ++++++++++++ .../cq-perf-regression-sample-base/pom.xml | 242 ++++++++++++++++ .../PerfRegressionSampleRouteBuilder.java | 28 ++ .../src/main/resources/application.properties | 17 ++ .../src/main/resources/request.adm | Bin 0 -> 1597 bytes tooling/perf-regression/pom.xml | 93 +++++++ .../processes-schema-app.diagrams.net | 1 + .../processes-schema-app.diagrams.net.drawio.png | Bin 0 -> 142189 bytes .../performance/regression/FileEditionHelper.java | 78 ++++++ .../performance/regression/MvnwCmdHelper.java | 76 +++++ .../regression/PerfRegressionCommand.java | 113 ++++++++ .../regression/PerformanceRegressionReport.java | 94 +++++++ .../PerformanceRegressionReportTest.java | 44 +++ .../perf-regression-expecteds/nominal.txt | 6 + tooling/pom.xml | 1 + 21 files changed, 1474 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b27b1ce15c..140ac00b2d 100644 --- a/pom.xml +++ b/pom.xml @@ -514,7 +514,8 @@ <exclude>doap.rdf</exclude> <exclude>ide-config/**</exclude> <exclude>id_file</exclude> - <exclude>mvnw*</exclude> + <exclude>**/mvnw*</exclude> + <exclude>**/*.net</exclude> <exclude>node/**</exclude> <exclude>**/resources/routes.1</exclude> <exclude>**/resources/routes.2</exclude> diff --git a/tooling/perf-regression/README.md b/tooling/perf-regression/README.md new file mode 100644 index 0000000000..0a018296b3 --- /dev/null +++ b/tooling/perf-regression/README.md @@ -0,0 +1,14 @@ +# Camel Quarkus Performance regression detection tool + +This Quarkus based command line tool takes a list of Camel Quarkus versions as argument. + +For each Camel Quarkus versions, it: + + Assembles a sample base Camel Quarkus project against the specified Camel Quarkus version + + Setup a performance test in the maven integration-test phase + + Runs the performance test with the help of the [hyperfoil-maven-plugin](https://hyperfoil.io/) + + Collects the mean throughput of the Camel Quarkus route + +At the end of the day, a report is presented to the console, including a status about possible regressions. + +Please find more details about the process in below picture: +![Performance regression detection tool process](processes-schema-app.diagrams.net.drawio.png) diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/.mvn/wrapper/MavenWrapperDownloader.java b/tooling/perf-regression/cq-perf-regression-sample-base/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100644 index 0000000000..d6d3d4969d --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + private static final String WRAPPER_VERSION = "0.5.6"; + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" + + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if(mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if(mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if(!outputFile.getParentFile().exists()) { + if(!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { + String username = System.getenv("MVNW_USERNAME"); + char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); + Authenticator.setDefault(new Authenticator() { + @Override + protected PasswordAuthentication getPasswordAuthentication() { + return new PasswordAuthentication(username, password); + } + }); + } + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/.mvn/wrapper/maven-wrapper.properties b/tooling/perf-regression/cq-perf-regression-sample-base/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000000..60c9e92aa6 --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,18 @@ +## --------------------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## --------------------------------------------------------------------------- +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.1/apache-maven-3.8.1-bin.zip +wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/cq-perf-regression-scenario.hf.yaml b/tooling/perf-regression/cq-perf-regression-sample-base/cq-perf-regression-scenario.hf.yaml new file mode 100644 index 0000000000..82a38dc9e2 --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/cq-perf-regression-scenario.hf.yaml @@ -0,0 +1,39 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# A camel-quarkus performance scenario to detect possible regressions +name: cq-perf-regression-scenario +http: + host: http://localhost:8080 +phases: +- operator: + # Running a fixed number of users during a fixed period of time + always: + users: 100 + # With 5 minutes duration we end up with roughly 8.4% variation across successive runs + # With 10 minutes duration we end up with roughly 3.5% variation across successive runs + # With 60 minutes duration we end up with roughly 1.8% variation across successive runs + # The GUID below will be replaced by a duration during the instantiation process + duration: 372f6453-7527-43b1-850b-3824fc3d1187 + scenario: + - hello: + - randomUUID: + toVar: my-random-uuid + - httpRequest: + POST: /hello + body: + pattern: '{"id":"${my-random-uuid}"}' diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/mvnw b/tooling/perf-regression/cq-perf-regression-sample-base/mvnw new file mode 100755 index 0000000000..a16b5431b4 --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/mvnw @@ -0,0 +1,310 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + if [ -n "$MVNW_REPOURL" ]; then + jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + else + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + fi + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + if $cygwin; then + wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` + fi + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + wget "$jarUrl" -O "$wrapperJarPath" + else + wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" + fi + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then + curl -o "$wrapperJarPath" "$jarUrl" -f + else + curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f + fi + + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + # For Cygwin, switch paths to Windows format before running javac + if $cygwin; then + javaClass=`cygpath --path --windows "$javaClass"` + fi + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/mvnw.cmd b/tooling/perf-regression/cq-perf-regression-sample-base/mvnw.cmd new file mode 100644 index 0000000000..17aca36184 --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/mvnw.cmd @@ -0,0 +1,180 @@ +@REM +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "License"); you may not use this file except in compliance with +@REM the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. +@REM + +@REM ---------------------------------------------------------------------------- +@REM Maven Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + +FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + if "%MVNW_VERBOSE%" == "true" ( + echo Found %WRAPPER_JAR% + ) +) else ( + if not "%MVNW_REPOURL%" == "" ( + SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" + ) + if "%MVNW_VERBOSE%" == "true" ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + ) + + powershell -Command "&{"^ + "$webclient = new-object System.Net.WebClient;"^ + "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ + "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ + "}"^ + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ + "}" + if "%MVNW_VERBOSE%" == "true" ( + echo Finished downloading %WRAPPER_JAR% + ) +) +@REM End of extension + +@REM Provide a "standardized" way to retrieve the CLI args that will +@REM work with both Windows and non-Windows executions. +set MAVEN_CMD_LINE_ARGS=%* + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/pom.xml b/tooling/perf-regression/cq-perf-regression-sample-base/pom.xml new file mode 100644 index 0000000000..7c6b9eacc5 --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/pom.xml @@ -0,0 +1,242 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" + xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus</artifactId> + <!-- The parent version will be set programmatically --> + <version/> + </parent> + <artifactId>cq-perf-regression-sample</artifactId> + <properties> + <compiler-plugin.version>3.8.1</compiler-plugin.version> + <maven.compiler.parameters>true</maven.compiler.parameters> + <maven.compiler.source>11</maven.compiler.source> + <maven.compiler.target>11</maven.compiler.target> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + <surefire-plugin.version>3.0.0-M5</surefire-plugin.version> + </properties> + <!-- The repositories will be set programmatically, if needed --> + <pluginRepositories/> + <repositories/> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-bom</artifactId> + <version>${quarkus.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-bom</artifactId> + <version>${project.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-platform-http</artifactId> + </dependency> + <dependency> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-atlasmap</artifactId> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-maven-plugin</artifactId> + <version>${quarkus.version}</version> + <extensions>true</extensions> + <executions> + <execution> + <goals> + <goal>build</goal> + <goal>generate-code</goal> + <goal>generate-code-tests</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <parameters>${maven.compiler.parameters}</parameters> + </configuration> + </plugin> + <plugin> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <systemPropertyVariables> + <java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager> + <maven.home>${maven.home}</maven.home> + </systemPropertyVariables> + </configuration> + </plugin> + <!-- Display active profile in pre-integration-test phase --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-help-plugin</artifactId> + <executions> + <execution> + <id>show-profiles</id> + <phase>pre-integration-test</phase> + <goals> + <goal>active-profiles</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <!-- Start the quarkus app under test async, will be killed when maven stops--> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <phase>pre-integration-test</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <async>true</async> + </configuration> + </plugin> + <plugin> + <!-- Wait the quarkus app under test to be listening on port 8080 --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>wait-for-quarkus-app-to-listen-on-port</id> + <phase>integration-test</phase> + <configuration> + <target> + <waitfor maxwait="10" + maxwaitunit="second"> + <socket server="localhost" + port="8080" /> + </waitfor> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>io.hyperfoil</groupId> + <artifactId>hyperfoil-maven-plugin</artifactId> + <version>0.19</version> + <!-- Bound to integration-test phase by default --> + <configuration> + <yaml>cq-perf-regression-scenario.hf.yaml</yaml> + <percentiles>true</percentiles> + </configuration> + <executions> + <execution> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + <profiles> + <profile> + <id>native</id> + <activation> + <property> + <name>native</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <artifactId>maven-failsafe-plugin</artifactId> + <version>${surefire-plugin.version}</version> + <executions> + <execution> + <goals> + <goal>integration-test</goal> + <goal>verify</goal> + </goals> + <configuration> + <systemPropertyVariables> + <native.image.path>${project.build.directory}/${project.build.finalName}-runner</native.image.path> + <java.util.logging.manager>org.jboss.logmanager.LogManager</java.util.logging.manager> + <maven.home>${maven.home}</maven.home> + </systemPropertyVariables> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <configuration> + <executable>target/cq-perf-regression-sample-${project.version}-runner</executable> + </configuration> + </plugin> + </plugins> + </build> + <properties> + <quarkus.package.type>native</quarkus.package.type> + </properties> + </profile> + <profile> + <id>jvm</id> + <activation> + <property> + <name>!native</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <configuration> + <executable>java</executable> + <arguments> + <argument>-jar</argument> + <argument>target/quarkus-app/quarkus-run.jar</argument> + </arguments> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/src/main/java/org/apache/camel/quarkus/performance/regression/PerfRegressionSampleRouteBuilder.java b/tooling/perf-regression/cq-perf-regression-sample-base/src/main/java/org/apache/camel/quarkus/performance/regression/PerfRegressionSampleRouteBuilder.java new file mode 100644 index 0000000000..92a1e59795 --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/src/main/java/org/apache/camel/quarkus/performance/regression/PerfRegressionSampleRouteBuilder.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.quarkus.performance.regression; + +import org.apache.camel.builder.RouteBuilder; + +public class PerfRegressionSampleRouteBuilder extends RouteBuilder { + + @Override + public void configure() throws Exception { + from("platform-http:/hello").to("atlasmap:request.adm"); + } + +} diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/src/main/resources/application.properties b/tooling/perf-regression/cq-perf-regression-sample-base/src/main/resources/application.properties new file mode 100644 index 0000000000..c8887b99df --- /dev/null +++ b/tooling/perf-regression/cq-perf-regression-sample-base/src/main/resources/application.properties @@ -0,0 +1,17 @@ +## --------------------------------------------------------------------------- +## Licensed to the Apache Software Foundation (ASF) under one or more +## contributor license agreements. See the NOTICE file distributed with +## this work for additional information regarding copyright ownership. +## The ASF licenses this file to You under the Apache License, Version 2.0 +## (the "License"); you may not use this file except in compliance with +## the License. You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +## --------------------------------------------------------------------------- +quarkus.native.resources.includes = request.adm \ No newline at end of file diff --git a/tooling/perf-regression/cq-perf-regression-sample-base/src/main/resources/request.adm b/tooling/perf-regression/cq-perf-regression-sample-base/src/main/resources/request.adm new file mode 100644 index 0000000000..c9f3bd6ef1 Binary files /dev/null and b/tooling/perf-regression/cq-perf-regression-sample-base/src/main/resources/request.adm differ diff --git a/tooling/perf-regression/pom.xml b/tooling/perf-regression/pom.xml new file mode 100644 index 0000000000..bb536c2e59 --- /dev/null +++ b/tooling/perf-regression/pom.xml @@ -0,0 +1,93 @@ +<?xml version="1.0"?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +--> +<project + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" + xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.camel.quarkus</groupId> + <artifactId>camel-quarkus-tooling</artifactId> + <version>2.12.0-SNAPSHOT</version> + </parent> + <artifactId>perf-regression</artifactId> + <dependencyManagement> + <dependencies> + <dependency> + <groupId>io.quarkus.platform</groupId> + <artifactId>quarkus-bom</artifactId> + <version>${quarkus.version}</version> + <type>pom</type> + <scope>import</scope> + </dependency> + </dependencies> + </dependencyManagement> + <dependencies> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + <version>1.3</version> + </dependency> + <dependency> + <groupId>tech.tablesaw</groupId> + <artifactId>tablesaw-core</artifactId> + <version>0.43.1</version> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-model</artifactId> + </dependency> + <dependency> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-picocli</artifactId> + </dependency> + <dependency> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-junit5</artifactId> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>io.quarkus</groupId> + <artifactId>quarkus-maven-plugin</artifactId> + <version>${quarkus.version}</version> + <extensions>true</extensions> + <executions> + <execution> + <goals> + <goal>build</goal> + <goal>generate-code</goal> + <goal>generate-code-tests</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> diff --git a/tooling/perf-regression/processes-schema-app.diagrams.net b/tooling/perf-regression/processes-schema-app.diagrams.net new file mode 100644 index 0000000000..34f8b80285 --- /dev/null +++ b/tooling/perf-regression/processes-schema-app.diagrams.net @@ -0,0 +1 @@ +<mxfile host="app.diagrams.net" modified="2022-06-17T11:12:04.581Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36" etag="uwBNcUJccGPQZ7_x7ICl" version="20.0.1" type="device"><diagram id="Z4cY-AFyPUgmtRGIdxM-" name="Page-1">7V3dc5s4EP9b7sHT9gEGhHHix7Spr72Zfkw7c3d9lEG21QAiQvjj/vqTkMB8Jjgxie1RJ9PAaiUt2t+uVtJWHTkfwu2fFMarL8RHwQhY/nbk3I4AsMcAjMSP5e8kZXJlS8KSYl8x7Qk/8X9IES1FTbGPkgojIyRgOK4SPRJFyGMVGqSUbKpsCxJUe43hEjUIPz0YNKn/YJ+tJPXatfb0Tw [...] \ No newline at end of file diff --git a/tooling/perf-regression/processes-schema-app.diagrams.net.drawio.png b/tooling/perf-regression/processes-schema-app.diagrams.net.drawio.png new file mode 100644 index 0000000000..4469b658c9 Binary files /dev/null and b/tooling/perf-regression/processes-schema-app.diagrams.net.drawio.png differ diff --git a/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/FileEditionHelper.java b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/FileEditionHelper.java new file mode 100644 index 0000000000..f95d178a3f --- /dev/null +++ b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/FileEditionHelper.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.quarkus.performance.regression; + +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import org.apache.commons.io.FileUtils; +import org.apache.maven.model.Model; +import org.apache.maven.model.Repository; +import org.apache.maven.model.io.xpp3.MavenXpp3Reader; +import org.apache.maven.model.io.xpp3.MavenXpp3Writer; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; + +public class FileEditionHelper { + + // We merely set the duration in the hyperfoil benchmark template + public static void instantiateHyperfoilBenchmark(Path cqVersionUnderTestFolder, String singleScenarioDuration) throws IOException { + File benchmarkFile = cqVersionUnderTestFolder.resolve("cq-perf-regression-scenario.hf.yaml").toFile(); + String benchmarkFileContent = FileUtils.readFileToString(benchmarkFile, StandardCharsets.UTF_8); + benchmarkFileContent = benchmarkFileContent.replaceAll("372f6453-7527-43b1-850b-3824fc3d1187", singleScenarioDuration); + FileUtils.writeStringToFile(benchmarkFile, benchmarkFileContent, StandardCharsets.UTF_8); + } + + // We set the parent version and add staging repositories if needed + public static void instantiatePomFile(Path cqVersionUnderTestFolder, String cqVersion, String cqStagingRepositoryUrl, String camelStagingRepositoryUrl) + throws IOException, XmlPullParserException { + File pomFile = cqVersionUnderTestFolder.resolve("pom.xml").toFile(); + + try (FileReader fileReader = new FileReader(pomFile, StandardCharsets.UTF_8)) { + MavenXpp3Reader pomReader = new MavenXpp3Reader(); + Model pomModel = pomReader.read(fileReader); + + pomModel.getParent().setVersion(cqVersion); + + if (cqStagingRepositoryUrl != null) { + Repository cqStagingRepository = new Repository(); + cqStagingRepository.setId("camel-quarkus-staging"); + cqStagingRepository.setName("Camel Quarkus Staging Repository"); + cqStagingRepository.setUrl(cqStagingRepositoryUrl); + pomModel.getPluginRepositories().add(cqStagingRepository); + pomModel.getRepositories().add(cqStagingRepository); + } + if (camelStagingRepositoryUrl != null) { + Repository camelStagingRepository = new Repository(); + camelStagingRepository.setId("camel-staging"); + camelStagingRepository.setName("Camel Staging Repository"); + camelStagingRepository.setUrl(camelStagingRepositoryUrl); + pomModel.getPluginRepositories().add(camelStagingRepository); + pomModel.getRepositories().add(camelStagingRepository); + } + + try (FileWriter fileWriter = new FileWriter(pomFile, StandardCharsets.UTF_8)) { + MavenXpp3Writer pomWriter = new MavenXpp3Writer(); + pomWriter.write(fileWriter, pomModel); + } + } + } + +} diff --git a/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/MvnwCmdHelper.java b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/MvnwCmdHelper.java new file mode 100644 index 0000000000..21797cb5e4 --- /dev/null +++ b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/MvnwCmdHelper.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.quarkus.performance.regression; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.commons.io.output.TeeOutputStream; + +public class MvnwCmdHelper { + + public static String execute(Path cqVersionUnderTestFolder, String args) { + + ByteArrayOutputStream stdoutAndStderrMemoryStream = null; + FileOutputStream stdoutFileStream = null; + TeeOutputStream teeOutputStream = null; + + try { + File mvnwFile = cqVersionUnderTestFolder.resolve("mvnw").toFile(); + CommandLine cmd = CommandLine.parse(mvnwFile.getAbsolutePath() + " " + args); + + stdoutAndStderrMemoryStream = new ByteArrayOutputStream(); + File logFile = cqVersionUnderTestFolder.resolve("logs.txt").toFile(); + stdoutFileStream = new FileOutputStream(logFile, true); + + stdoutFileStream.write("\n\n**********************************************************************\n".getBytes(StandardCharsets.UTF_8)); + stdoutFileStream.write("**********************************************************************\n".getBytes(StandardCharsets.UTF_8)); + stdoutFileStream.write(("** "+cmd+"\n").getBytes(StandardCharsets.UTF_8)); + stdoutFileStream.write("**********************************************************************\n".getBytes(StandardCharsets.UTF_8)); + stdoutFileStream.write("**********************************************************************\n".getBytes(StandardCharsets.UTF_8)); + + teeOutputStream = new TeeOutputStream(stdoutAndStderrMemoryStream, stdoutFileStream); + DefaultExecutor executor = new DefaultExecutor(); + PumpStreamHandler psh = new PumpStreamHandler(teeOutputStream); + executor.setStreamHandler(psh); + executor.setWorkingDirectory(cqVersionUnderTestFolder.toFile()); + + int exitValue = executor.execute(cmd); + String outAndErr = stdoutAndStderrMemoryStream.toString(StandardCharsets.UTF_8); + if (exitValue != 0) { + throw new RuntimeException("The command '" + cmd + "' has returned exitValue " + exitValue + ", process logs below:\n" + outAndErr); + } + + return outAndErr; + } catch (IOException ex) { + throw new RuntimeException("An issue occurred while attempting to execute 'mvnw " + args + "', more logs may be found in " + cqVersionUnderTestFolder + "/logs.txt if exists", ex); + } finally { + IOUtils.closeQuietly(stdoutAndStderrMemoryStream); + IOUtils.closeQuietly(stdoutFileStream); + IOUtils.closeQuietly(teeOutputStream); + } + } + +} diff --git a/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/PerfRegressionCommand.java b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/PerfRegressionCommand.java new file mode 100644 index 0000000000..ad8f04878b --- /dev/null +++ b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/PerfRegressionCommand.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.quarkus.performance.regression; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.RegExUtils; +import org.codehaus.plexus.util.xml.pull.XmlPullParserException; + +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +@picocli.CommandLine.Command(description = "Run a performance test against a list of Camel Quarkus versions and print a report") +public class PerfRegressionCommand implements Runnable { + + private static Path PERF_SAMPLE_TEMPLATE_FOLDER = Paths.get("cq-perf-regression-sample-base"); + + @Parameters(paramLabel = "<versions>", arity = "1..*", description = "A list of versions, e.g: 2.7.0 2.8.0-SNAPSHOT") + private String[] cqVersions = {}; + + @Option(names = {"-cqs", "--camel-quarkus-staging-repository"}, description = "Camel Quarkus staging repository, e.g: https://repository.apache.org/content/repositories/orgapachecamel-1423") + private String cqStagingRepository; + + @Option(names = {"-cs", "--camel-staging-repository"}, description = "Camel staging repository, e.g: https://repository.apache.org/content/repositories/orgapachecamel-1424") + private String camelStagingRepository; + + @Option(names = {"-d", "--duration"}, defaultValue = "10m", description = "The duration of a single performance test scenario (e.g. 45s, 30m, 1h). Up to 2 scenarios per version could be run.") + private String singleScenarioDuration; + + @Option(names = {"-an", "--also-run-native-mode"}, description = "Tells whether the throughput test should also be run in native mode. By default, run in JVM mode only.") + private boolean alsoRunNativeMode; + + @Override + public void run() { + PerformanceRegressionReport report = new PerformanceRegressionReport(singleScenarioDuration); + + Path cqVersionsUnderTestFolder = Paths.get("target/cq-versions-under-test"); + try { + Files.createDirectories(cqVersionsUnderTestFolder); + FileUtils.cleanDirectory(cqVersionsUnderTestFolder.toFile()); + + for (String cqVersion : cqVersions) { + runPerfRegressionForCqVersion(cqVersionsUnderTestFolder.resolve(cqVersion), cqVersion, report); + } + + System.out.println(report.printAll()); + } catch (IOException|XmlPullParserException e) { + throw new RuntimeException("An issue has been caught while trying to setup performance regression tests.", e); + } + } + + private void runPerfRegressionForCqVersion(Path cqVersionUnderTestFolder, String cqVersion, PerformanceRegressionReport report) throws IOException, XmlPullParserException { + // Copy the template project into a folder dedicated to cqVersion tests + FileUtils.copyDirectory(PERF_SAMPLE_TEMPLATE_FOLDER.toFile(), cqVersionUnderTestFolder.toFile()); + + FileEditionHelper.instantiateHyperfoilBenchmark(cqVersionUnderTestFolder, singleScenarioDuration); + FileEditionHelper.instantiatePomFile(cqVersionUnderTestFolder, cqVersion, cqStagingRepository, camelStagingRepository); + + // Locally sets the right maven version in the maven wrapper + // camel-quarkus >= 2.6.0.CR1 => maven 3.8.4 + // camel-quarkus >= 2.1.0 => maven 3.8.1 + String targetMavenVersion = getTargetMavenVersion(cqVersionUnderTestFolder); + setMvnwMavenVersion(cqVersionUnderTestFolder, targetMavenVersion); + + // Run performance regression test in JVM mode + double jvmThroughput = runPerfRegression(cqVersionUnderTestFolder, "integration-test"); + report.setCategoryMeasureForVersion(cqVersion, "JVM", jvmThroughput); + + // Run performance regression test in native mode + if(alsoRunNativeMode) { + double nativeThroughput = runPerfRegression(cqVersionUnderTestFolder, "integration-test -Dnative -Dquarkus.native.container-build=true"); + report.setCategoryMeasureForVersion(cqVersion, "Native", nativeThroughput); + } + } + + private static String getTargetMavenVersion(Path cqVersionUnderTestFolder) { + String stdoutAndStdErr = MvnwCmdHelper.execute(cqVersionUnderTestFolder, "help:evaluate -Dexpression='target-maven-version' -q -DforceStdout"); + String targetMavenVersion = stdoutAndStdErr.substring(stdoutAndStdErr.lastIndexOf(System.lineSeparator())+System.lineSeparator().length()); + + return "null object or invalid expression".equals(targetMavenVersion) ? "3.8.1" : targetMavenVersion; + } + + private static void setMvnwMavenVersion(Path cqVersionUnderTestFolder, String targetMavenVersion) { + MvnwCmdHelper.execute(cqVersionUnderTestFolder, "wrapper:wrapper -Dmaven="+targetMavenVersion); + } + + private static double runPerfRegression(Path cqVersionUnderTestFolder, String args) { + String stdout = MvnwCmdHelper.execute(cqVersionUnderTestFolder, args); + + // Extract the throughput from a log line like "15:26:23,110 INFO (main) [i.h.m.RunMojo] Requests/sec: 1153.56" + String throughput = RegExUtils.replacePattern(stdout, ".*RunMojo] Requests/sec: ([0-9.,]+).*", "$1"); + return Double.parseDouble(throughput); + } + +} diff --git a/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/PerformanceRegressionReport.java b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/PerformanceRegressionReport.java new file mode 100644 index 0000000000..08588c7b12 --- /dev/null +++ b/tooling/perf-regression/src/main/java/org/apache/camel/quarkus/performance/regression/PerformanceRegressionReport.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.quarkus.performance.regression; + +import java.util.HashMap; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.maven.artifact.versioning.ComparableVersion; + +import tech.tablesaw.api.StringColumn; +import tech.tablesaw.api.Table; + +/** + * Provide a human readable performance regression report ready to be printed to + * the console. For each camel-quarkus version, the report will print: + * + The throughput for each measure in a category (JVM, Native) in a new column + * + The percent increase throughput compared to the previous row in the same column + */ +public class PerformanceRegressionReport { + + private String duration; + private TreeMap<ComparableVersion, Map<String, Double>> measuresPerVersion = new TreeMap<>(); + + public PerformanceRegressionReport(String duration) { + this.duration = duration; + } + + public void setCategoryMeasureForVersion(String cqVersion, String category, double throughput) { + ComparableVersion version = new ComparableVersion(cqVersion); + measuresPerVersion.computeIfAbsent(version, k -> new HashMap<>()).put(category, throughput); + } + + public String printAll() { + Table table = Table.create("Camel Quarkus Throughput Performance Increase Compared to Previous Version"); + + StringColumn cqVersionsColumn = StringColumn.create("Camel Quarkus version"); + StringColumn durationsColumn = StringColumn.create("Duration"); + StringColumn jvmMeasuresColumn = StringColumn.create("JVM req/s [%increase]"); + StringColumn nativeMeasuresColumn = StringColumn.create("Native req/s [%increase]"); + StringColumn statusColumn = StringColumn.create("Status"); + double previousJvmMeasure = Double.POSITIVE_INFINITY; + double previousNativeMeasure = Double.POSITIVE_INFINITY; + + for (Map.Entry<ComparableVersion, Map<String, Double>> measurePerVersion : measuresPerVersion.entrySet()) { + cqVersionsColumn.append(measurePerVersion.getKey().toString()); + durationsColumn.append(duration); + boolean regressionDetected = false; + + double jvmMeasure = measurePerVersion.getValue().get("JVM"); + double percentIncreaseJvm = (previousJvmMeasure == Double.POSITIVE_INFINITY) ? 0.0 : ((jvmMeasure / previousJvmMeasure) - 1.0) * 100.0; + jvmMeasuresColumn.append(String.format("%.2f req/s [%+.2f%%]", jvmMeasure, percentIncreaseJvm)); + previousJvmMeasure = jvmMeasure; + if (percentIncreaseJvm <= -5.00) { + regressionDetected = true; + } + + if (measurePerVersion.getValue().containsKey("Native")) { + double nativeMeasure = measurePerVersion.getValue().get("Native"); + double percentIncreaseNative = (previousNativeMeasure == Double.POSITIVE_INFINITY) ? 0.0 : ((nativeMeasure / previousNativeMeasure) - 1.0) * 100.0; + nativeMeasuresColumn.append(String.format("%.2f req/s [%+.2f%%]", nativeMeasure, percentIncreaseNative)); + previousNativeMeasure = nativeMeasure; + if (percentIncreaseNative <= -5.00) { + regressionDetected = true; + } + } + + statusColumn.append(regressionDetected ? "Potential performance regression" : "OK"); + } + + if (!nativeMeasuresColumn.isEmpty()) { + table.addColumns(cqVersionsColumn, durationsColumn, jvmMeasuresColumn, nativeMeasuresColumn, statusColumn); + } else { + table.addColumns(cqVersionsColumn, durationsColumn, jvmMeasuresColumn, statusColumn); + } + + return table.printAll(); + } + +} diff --git a/tooling/perf-regression/src/test/java/org/apache/camel/quarkus/performance/regression/PerformanceRegressionReportTest.java b/tooling/perf-regression/src/test/java/org/apache/camel/quarkus/performance/regression/PerformanceRegressionReportTest.java new file mode 100644 index 0000000000..6424ea1454 --- /dev/null +++ b/tooling/perf-regression/src/test/java/org/apache/camel/quarkus/performance/regression/PerformanceRegressionReportTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.quarkus.performance.regression; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class PerformanceRegressionReportTest { + + @Test + public void printAllShouldSucceed() throws IOException { + PerformanceRegressionReport sut = new PerformanceRegressionReport("10m"); + sut.setCategoryMeasureForVersion("2.10.0", "JVM", 360.0); + sut.setCategoryMeasureForVersion("2.8.0", "JVM", 380.0); + sut.setCategoryMeasureForVersion("2.9.0", "JVM", 390.0); + + sut.setCategoryMeasureForVersion("2.10.0", "Native", 1000.0); + sut.setCategoryMeasureForVersion("2.8.0", "Native", 1080.0); + sut.setCategoryMeasureForVersion("2.9.0", "Native", 1090.0); + + String expected = IOUtils.resourceToString("/perf-regression-expecteds/nominal.txt", StandardCharsets.UTF_8); + assertEquals(expected, sut.printAll()); + } + +} diff --git a/tooling/perf-regression/src/test/resources/perf-regression-expecteds/nominal.txt b/tooling/perf-regression/src/test/resources/perf-regression-expecteds/nominal.txt new file mode 100644 index 0000000000..ff49459f93 --- /dev/null +++ b/tooling/perf-regression/src/test/resources/perf-regression-expecteds/nominal.txt @@ -0,0 +1,6 @@ + Camel Quarkus Throughput Performance Increase Compared to Previous Version + Camel Quarkus version | Duration | JVM req/s [%increase] | Native req/s [%increase] | Status | +---------------------------------------------------------------------------------------------------------------------------------- + 2.8.0 | 10m | 380.00 req/s [+0.00%] | 1080.00 req/s [+0.00%] | OK | + 2.9.0 | 10m | 390.00 req/s [+2.63%] | 1090.00 req/s [+0.93%] | OK | + 2.10.0 | 10m | 360.00 req/s [-7.69%] | 1000.00 req/s [-8.26%] | Potential performance regression | \ No newline at end of file diff --git a/tooling/pom.xml b/tooling/pom.xml index 309292f9ce..2b43fd8b21 100644 --- a/tooling/pom.xml +++ b/tooling/pom.xml @@ -34,6 +34,7 @@ <modules> <module>maven-plugin</module> + <module>perf-regression</module> <module>test-list</module> </modules>