This is an automated email from the ASF dual-hosted git repository. markusthoemmes pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push: new 866990e Add gatling-based latency test. (#3556) 866990e is described below commit 866990eefc1418e1a8775b05fca34031a6b096b6 Author: Christian Bickel <git...@cbickel.de> AuthorDate: Wed Apr 18 16:05:57 2018 +0200 Add gatling-based latency test. (#3556) --- .travis.yml | 1 + performance/README.md | 30 +++++++ performance/gatling_tests/build.gradle | 6 +- .../src/gatling/resources/data/javaAction.jar | Bin 0 -> 1060 bytes .../data/javaAction.java} | 34 ++++---- .../src/gatling/resources/data/nodeJSAction.js | 5 ++ .../src/gatling/resources/data/pythonAction.py | 9 ++ .../src/gatling/resources/data/swiftAction.swift | 9 ++ .../src/gatling/scala/LatencySimulation.scala | 97 +++++++++++++++++++++ .../extension/whisk/OpenWhiskActionBuilder.scala | 90 ++++++++++++++++++- 10 files changed, 261 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7fc1587..f5b0fd8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -65,5 +65,6 @@ jobs: - TERM=dumb ./performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" 2m - TERM=dumb ./performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" 4 2 2m - OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-ApiV1Simulation + - OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)" EXCLUDED_KINDS="python:default,java:default,swift:default" ./gradlew gatlingRun-LatencySimulation env: - DESCRIPTION="Execute wrk-performance test suite." diff --git a/performance/README.md b/performance/README.md index f089021..e8d2164 100644 --- a/performance/README.md +++ b/performance/README.md @@ -61,3 +61,33 @@ You can run the simulation with (in OPENWHISK_HOME) ``` OPENWHISK_HOST="openwhisk.mydomain.com" CONNECTIONS="10" REQUESTS_PER_SEC="50" ./gradlew gatlingRun-ApiV1Simulation ``` + +##### Latency Simulation + +This simulation creates actions of the following four kinds: `nodejs:default`, `swift:default`, `java:default` and +`python:default`. +Afterwards the action is invoked once. This is the cold-start and will not be part of the thresholds. +Next, the action will be invoked 100 times blocking and one after each other. The last step is, that the action will be deleted. + +Once one language is finished, the next kind will be taken. They are not running in parallel. There are never more than +1 activations in the system, as we only want to meassure latency of warm activations. +As all actions are invoked blocking and only one action is in the system, it doesn't matter how many controllers +and invokers are deployed. If several controllers or invokers are deployed, all controllers send the activation +always to the same invoker. + +The comparison of the thresholds is against the mean response times of the warm activations. + +Available environment variables: + +``` +OPENWHISK_HOST (required) +API_KEY (required, format: UUID:KEY) +MEAN_RESPONSE_TIME (required) +MAX_MEAN_RESPONSE_TIME (default: MEAN_RESPONSE_TIME) +EXCLUDED_KINDS (default: "", format: "python:default,java:default,swift:default") +``` + +You can run the simulation with (in OPENWHISK_HOME) +``` +OPENWHISK_HOST="openwhisk.mydomain.com" MEAN_RESPONSE_TIME="20" API_KEY="UUID:KEY" ./gradlew gatlingRun-LatencySimulation +``` diff --git a/performance/gatling_tests/build.gradle b/performance/gatling_tests/build.gradle index 7e8f1ad..9242fb4 100644 --- a/performance/gatling_tests/build.gradle +++ b/performance/gatling_tests/build.gradle @@ -1,5 +1,5 @@ plugins { - id "com.github.lkishalmi.gatling" version "0.7.1" + id "com.github.lkishalmi.gatling" version "0.7.2" } apply plugin: 'eclipse' @@ -10,8 +10,8 @@ repositories { } dependencies { - compile "org.scala-lang:scala-library:${gradle.scala.version}" - compile "io.gatling.highcharts:gatling-charts-highcharts:2.2.5" + gatling "io.spray:spray-json_2.12:1.3.4" + gatling "commons-io:commons-io:2.6" } tasks.withType(ScalaCompile) { diff --git a/performance/gatling_tests/src/gatling/resources/data/javaAction.jar b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar new file mode 100644 index 0000000..17229eb Binary files /dev/null and b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar differ diff --git a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala b/performance/gatling_tests/src/gatling/resources/data/javaAction.java similarity index 54% copy from performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala copy to performance/gatling_tests/src/gatling/resources/data/javaAction.java index ef6d2ef..56da8db 100644 --- a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala +++ b/performance/gatling_tests/src/gatling/resources/data/javaAction.java @@ -15,24 +15,26 @@ * limitations under the License. */ -package extension.whisk +// Build the jar with the following commands: +// +// javac -cp gson-2.8.2.jar JavaAction.java +// jar cvf javaAction.jar JavaAction.class -import io.gatling.core.Predef._ -import io.gatling.core.action.Action -import io.gatling.core.action.builder.ActionBuilder -import io.gatling.core.session.Expression -import io.gatling.core.structure.ScenarioContext -import io.gatling.http.request.builder.{Http, HttpRequestBuilder} +import com.google.gson.JsonObject; -case class OpenWhiskActionBuilderBase(requestName: Expression[String]) { +public class JavaAction { + public static JsonObject main(JsonObject args) { + String text; - implicit private val http = new Http(requestName) + try { + text = args.getAsJsonPrimitive("text").getAsString(); + } catch(Exception e) { + text = "stranger"; + } - def info() = OpenWhiskActionBuilder(http.get("/api/v1")) -} - -case class OpenWhiskActionBuilder(http: HttpRequestBuilder) extends ActionBuilder { - override def build(ctx: ScenarioContext, next: Action): Action = { - http.build(ctx, next) - } + JsonObject response = new JsonObject(); + System.out.println("Hello " + text + "!"); + response.addProperty("payload", "Hello " + text + "!"); + return response; + } } diff --git a/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js new file mode 100644 index 0000000..8c71017 --- /dev/null +++ b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js @@ -0,0 +1,5 @@ +function main(params) { + var greeting = "Hello" + (params.text || "stranger") + "!"; + console.log(greeting); + return { payload: greeting }; +} diff --git a/performance/gatling_tests/src/gatling/resources/data/pythonAction.py b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py new file mode 100644 index 0000000..c26b060 --- /dev/null +++ b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py @@ -0,0 +1,9 @@ +import sys +def main(dict): + if 'text' in dict: + text = dict['text'] + else: + text = "stranger" + greeting = "Hello " + text + "!" + print(greeting) + return {"payload": greeting} diff --git a/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift new file mode 100644 index 0000000..3f52f66 --- /dev/null +++ b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift @@ -0,0 +1,9 @@ +func main(args: [String:Any]) -> [String:Any] { + if let text = args["text"] as? String { + print("Hello " + text + "!") + return [ "payload" : "Hello " + text + "!" ] + } else { + print("Hello stranger!") + return [ "payload" : "Hello stranger!" ] + } +} diff --git a/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala new file mode 100644 index 0000000..9ec41f0 --- /dev/null +++ b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.nio.charset.StandardCharsets +import java.util.Base64 + +import extension.whisk.Predef._ +import io.gatling.core.Predef._ +import io.gatling.core.session.Expression +import io.gatling.core.util.Resource +import org.apache.commons.io.FileUtils + +class LatencySimulation extends Simulation { + // Specify parameters for the run + val host = sys.env("OPENWHISK_HOST") + + // Specify authentication + val Array(uuid, key) = sys.env("API_KEY").split(":") + + // Specify thresholds + val meanResponseTime: Int = sys.env("MEAN_RESPONSE_TIME").toInt + val maximalMeanResponseTime: Int = sys.env.getOrElse("MAX_MEAN_RESPONSE_TIME", meanResponseTime.toString).toInt + + // Exclude runtimes + val excludedKinds: Seq[String] = sys.env.getOrElse("EXCLUDED_KINDS", "").split(",") + + // Generate the OpenWhiskProtocol + val openWhiskProtocol = openWhisk.apiHost(host) + + /** + * Generate a list of actions to execute. The list is a tuple of (kind, code, actionName, main) + * `kind` is needed to create the action + * `code` is loaded form the files located in `resources/data` + * `actionName` is the name of the action in OpenWhisk + * `main` is only needed for java. This is the name of the class where the main method is located. + */ + val actions: Seq[(String, String, String, String)] = Map( + "nodejs:default" -> (FileUtils + .readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8), "latencyTest_node", ""), + "python:default" -> (FileUtils + .readFileToString(Resource.body("pythonAction.py").get.file, StandardCharsets.UTF_8), "latencyTest_python", ""), + "swift:default" -> (FileUtils + .readFileToString(Resource.body("swiftAction.swift").get.file, StandardCharsets.UTF_8), "latencyTest_swift", ""), + "java:default" -> (Base64.getEncoder.encodeToString( + FileUtils.readFileToByteArray(Resource.body("javaAction.jar").get.file)), "latencyTest_java", "JavaAction")) + .filterNot(e => excludedKinds.contains(e._1)) + .map { + case (kind, (code, name, main)) => + (kind, code, name, main) + } + .toSeq + + // Define scenario + val test = scenario("Invoke one action after each other to test latency") + .foreach(actions, "action") { + val code: Expression[String] = "${action._2}" + exec( + openWhisk("Create ${action._1} action") + .authenticate(uuid, key) + .action("${action._3}") + .create(code, "${action._1}", "${action._4}")) + .exec(openWhisk("Cold ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke()) + .repeat(100) { + exec(openWhisk("Warm ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke()) + } + .exec(openWhisk("Delete ${action._1} action").authenticate(uuid, key).action("${action._3}").delete()) + } + + val testSetup = setUp(test.inject(atOnceUsers(1))) + .protocols(openWhiskProtocol) + + actions + .map { case (kind, _, _, _) => s"Warm $kind invocation" } + .foldLeft(testSetup) { (agg, cur) => + // One failure will make the build yellow + agg + .assertions(details(cur).responseTime.mean.lte(meanResponseTime)) + .assertions(details(cur).responseTime.mean.lt(maximalMeanResponseTime)) + // Mark the build yellow, if there are failed requests. And red if both conditions fail. + .assertions(details(cur).failedRequests.count.is(0)) + .assertions(details(cur).failedRequests.percent.lte(0.1)) + } +} diff --git a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala index ef6d2ef..56dc6ce 100644 --- a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala +++ b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala @@ -23,12 +23,100 @@ import io.gatling.core.action.builder.ActionBuilder import io.gatling.core.session.Expression import io.gatling.core.structure.ScenarioContext import io.gatling.http.request.builder.{Http, HttpRequestBuilder} +import spray.json.DefaultJsonProtocol._ +import spray.json._ case class OpenWhiskActionBuilderBase(requestName: Expression[String]) { implicit private val http = new Http(requestName) - def info() = OpenWhiskActionBuilder(http.get("/api/v1")) + /** Call the `/api/v1`-endpoint of the specified system */ + def info() = { + OpenWhiskActionBuilder(http.get("/api/v1")) + } + + /** + * Specify authentication data. This is needed to perform operations on namespaces or working with entities. + * + * @param uuid The UUID of the namespace + * @param key The key of the namespace + */ + def authenticate(uuid: Expression[String], key: Expression[String]) = { + OpenWhiskActionBuilderWithNamespace(uuid, key) + } +} + +case class OpenWhiskActionBuilderWithNamespace(private val uuid: Expression[String], + private val key: Expression[String], + private val namespace: String = "_")(implicit private val http: Http) { + + /** + * Specify on which namespace you want to perform any action. + * + * @param namespace The namespace you want to use. + */ + def namespace(namespace: String) = { + OpenWhiskActionBuilderWithNamespace(uuid, key, namespace) + } + + /** List all namespaces you have access to, with your current authentication. */ + def list() = { + OpenWhiskActionBuilder(http.get("/api/v1/namespaces").basicAuth(uuid, key)) + } + + /** + * Perform any request against the actions-API. E.g. creating, invoking or deleting actions. + * + * @param actionName Name of the action in the Whisk-system. + */ + def action(actionName: String) = { + OpenWhiskActionBuilderWithAction(uuid, key, namespace, actionName) + } +} + +case class OpenWhiskActionBuilderWithAction(private val uuid: Expression[String], + private val key: Expression[String], + private val namespace: String, + private val action: String)(implicit private val http: Http) { + private val path: Expression[String] = s"/api/v1/namespaces/$namespace/actions/$action" + + /** Fetch the action from OpenWhisk */ + def get() = { + OpenWhiskActionBuilder(http.get(path).basicAuth(uuid, key)) + } + + /** Delete the action from OpenWhisk */ + def delete() = { + OpenWhiskActionBuilder(http.delete(path).basicAuth(uuid, key)) + } + + /** + * Create the action in OpenWhisk. + * + * @param code The code of the action to create. + * @param kind The kind of the action you want to create. Default is `nodejs:default`. + * @param main Main method of your action. This is only needed for java actions. + */ + def create(code: Expression[String], kind: Expression[String] = "nodejs:default", main: Expression[String] = "") = { + + val json: Expression[String] = session => { + code(session).flatMap { c => + kind(session).flatMap { k => + main(session).map { m => + val exec = Map("kind" -> k, "code" -> c) ++ (if (m.size > 0) Map("main" -> m) else Map[String, String]()) + JsObject("exec" -> exec.toJson).compactPrint + } + } + } + } + + OpenWhiskActionBuilder(http.put(path).basicAuth(uuid, key).body(StringBody(json))) + } + + /** Invoke the action. */ + def invoke() = { + OpenWhiskActionBuilder(http.post(path).queryParam("blocking", "true").basicAuth(uuid, key)) + } } case class OpenWhiskActionBuilder(http: HttpRequestBuilder) extends ActionBuilder { -- To stop receiving notification emails like this one, please contact markusthoem...@apache.org.