openwhisk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markusthoem...@apache.org
Subject [incubator-openwhisk] branch master updated: Add gatling-based latency test. (#3556)
Date Wed, 18 Apr 2018 14:06:01 GMT
This is an automated email from the ASF dual-hosted git repository.

markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new 866990e  Add gatling-based latency test. (#3556)
866990e is described below

commit 866990eefc1418e1a8775b05fca34031a6b096b6
Author: Christian Bickel <github@cbickel.de>
AuthorDate: Wed Apr 18 16:05:57 2018 +0200

    Add gatling-based latency test. (#3556)
---
 .travis.yml                                        |   1 +
 performance/README.md                              |  30 +++++++
 performance/gatling_tests/build.gradle             |   6 +-
 .../src/gatling/resources/data/javaAction.jar      | Bin 0 -> 1060 bytes
 .../data/javaAction.java}                          |  34 ++++----
 .../src/gatling/resources/data/nodeJSAction.js     |   5 ++
 .../src/gatling/resources/data/pythonAction.py     |   9 ++
 .../src/gatling/resources/data/swiftAction.swift   |   9 ++
 .../src/gatling/scala/LatencySimulation.scala      |  97 +++++++++++++++++++++
 .../extension/whisk/OpenWhiskActionBuilder.scala   |  90 ++++++++++++++++++-
 10 files changed, 261 insertions(+), 20 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 7fc1587..f5b0fd8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -65,5 +65,6 @@ jobs:
         - TERM=dumb ./performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat
ansible/files/auth.guest)" 2m
         - TERM=dumb ./performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat
ansible/files/auth.guest)" 4 2 2m
         - OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-ApiV1Simulation
+        - OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)"
EXCLUDED_KINDS="python:default,java:default,swift:default" ./gradlew gatlingRun-LatencySimulation
       env:
         - DESCRIPTION="Execute wrk-performance test suite."
diff --git a/performance/README.md b/performance/README.md
index f089021..e8d2164 100644
--- a/performance/README.md
+++ b/performance/README.md
@@ -61,3 +61,33 @@ You can run the simulation with (in OPENWHISK_HOME)
 ```
 OPENWHISK_HOST="openwhisk.mydomain.com" CONNECTIONS="10" REQUESTS_PER_SEC="50" ./gradlew
gatlingRun-ApiV1Simulation
 ```
+
+##### Latency Simulation
+
+This simulation creates actions of the following four kinds: `nodejs:default`, `swift:default`,
`java:default` and
+`python:default`.
+Afterwards the action is invoked once. This is the cold-start and will not be part of the
thresholds.
+Next, the action will be invoked 100 times blocking and one after each other. The last step
is, that the action will be deleted.
+
+Once one language is finished, the next kind will be taken. They are not running in parallel.
There are never more than
+1 activations in the system, as we only want to meassure latency of warm activations.
+As all actions are invoked blocking and only one action is in the system, it doesn't matter
how many controllers
+and invokers are deployed. If several controllers or invokers are deployed, all controllers
send the activation
+always to the same invoker.
+
+The comparison of the thresholds is against the mean response times of the warm activations.
+
+Available environment variables:
+
+```
+OPENWHISK_HOST          (required)
+API_KEY                 (required, format: UUID:KEY)
+MEAN_RESPONSE_TIME      (required)
+MAX_MEAN_RESPONSE_TIME  (default: MEAN_RESPONSE_TIME)
+EXCLUDED_KINDS          (default: "", format: "python:default,java:default,swift:default")
+```
+
+You can run the simulation with (in OPENWHISK_HOME)
+```
+OPENWHISK_HOST="openwhisk.mydomain.com" MEAN_RESPONSE_TIME="20" API_KEY="UUID:KEY" ./gradlew
gatlingRun-LatencySimulation
+```
diff --git a/performance/gatling_tests/build.gradle b/performance/gatling_tests/build.gradle
index 7e8f1ad..9242fb4 100644
--- a/performance/gatling_tests/build.gradle
+++ b/performance/gatling_tests/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-    id "com.github.lkishalmi.gatling" version "0.7.1"
+    id "com.github.lkishalmi.gatling" version "0.7.2"
 }
 
 apply plugin: 'eclipse'
@@ -10,8 +10,8 @@ repositories {
 }
 
 dependencies {
-    compile "org.scala-lang:scala-library:${gradle.scala.version}"
-    compile "io.gatling.highcharts:gatling-charts-highcharts:2.2.5"
+    gatling "io.spray:spray-json_2.12:1.3.4"
+    gatling "commons-io:commons-io:2.6"
 }
 
 tasks.withType(ScalaCompile) {
diff --git a/performance/gatling_tests/src/gatling/resources/data/javaAction.jar b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar
new file mode 100644
index 0000000..17229eb
Binary files /dev/null and b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar
differ
diff --git a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
b/performance/gatling_tests/src/gatling/resources/data/javaAction.java
similarity index 54%
copy from performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
copy to performance/gatling_tests/src/gatling/resources/data/javaAction.java
index ef6d2ef..56da8db 100644
--- a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
+++ b/performance/gatling_tests/src/gatling/resources/data/javaAction.java
@@ -15,24 +15,26 @@
  * limitations under the License.
  */
 
-package extension.whisk
+// Build the jar with the following commands:
+//
+// javac -cp gson-2.8.2.jar JavaAction.java
+// jar cvf javaAction.jar JavaAction.class
 
-import io.gatling.core.Predef._
-import io.gatling.core.action.Action
-import io.gatling.core.action.builder.ActionBuilder
-import io.gatling.core.session.Expression
-import io.gatling.core.structure.ScenarioContext
-import io.gatling.http.request.builder.{Http, HttpRequestBuilder}
+import com.google.gson.JsonObject;
 
-case class OpenWhiskActionBuilderBase(requestName: Expression[String]) {
+public class JavaAction {
+    public static JsonObject main(JsonObject args) {
+        String text;
 
-  implicit private val http = new Http(requestName)
+        try {
+            text = args.getAsJsonPrimitive("text").getAsString();
+        } catch(Exception e) {
+            text = "stranger";
+        }
 
-  def info() = OpenWhiskActionBuilder(http.get("/api/v1"))
-}
-
-case class OpenWhiskActionBuilder(http: HttpRequestBuilder) extends ActionBuilder {
-  override def build(ctx: ScenarioContext, next: Action): Action = {
-    http.build(ctx, next)
-  }
+        JsonObject response = new JsonObject();
+        System.out.println("Hello " + text + "!");
+        response.addProperty("payload", "Hello " + text + "!");
+        return response;
+    }
 }
diff --git a/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js
new file mode 100644
index 0000000..8c71017
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js
@@ -0,0 +1,5 @@
+function main(params) {
+    var greeting = "Hello" + (params.text || "stranger") + "!";
+    console.log(greeting);
+    return { payload: greeting };
+}
diff --git a/performance/gatling_tests/src/gatling/resources/data/pythonAction.py b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py
new file mode 100644
index 0000000..c26b060
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py
@@ -0,0 +1,9 @@
+import sys
+def main(dict):
+    if 'text' in dict:
+        text = dict['text']
+    else:
+        text = "stranger"
+    greeting = "Hello " + text + "!"
+    print(greeting)
+    return {"payload": greeting}
diff --git a/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift
new file mode 100644
index 0000000..3f52f66
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift
@@ -0,0 +1,9 @@
+func main(args: [String:Any]) -> [String:Any] {
+    if let text = args["text"] as? String {
+        print("Hello " + text + "!")
+        return [ "payload" : "Hello " + text + "!" ]
+    } else {
+        print("Hello stranger!")
+        return [ "payload" : "Hello stranger!" ]
+    }
+}
diff --git a/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
new file mode 100644
index 0000000..9ec41f0
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
+import extension.whisk.Predef._
+import io.gatling.core.Predef._
+import io.gatling.core.session.Expression
+import io.gatling.core.util.Resource
+import org.apache.commons.io.FileUtils
+
+class LatencySimulation extends Simulation {
+  // Specify parameters for the run
+  val host = sys.env("OPENWHISK_HOST")
+
+  // Specify authentication
+  val Array(uuid, key) = sys.env("API_KEY").split(":")
+
+  // Specify thresholds
+  val meanResponseTime: Int = sys.env("MEAN_RESPONSE_TIME").toInt
+  val maximalMeanResponseTime: Int = sys.env.getOrElse("MAX_MEAN_RESPONSE_TIME", meanResponseTime.toString).toInt
+
+  // Exclude runtimes
+  val excludedKinds: Seq[String] = sys.env.getOrElse("EXCLUDED_KINDS", "").split(",")
+
+  // Generate the OpenWhiskProtocol
+  val openWhiskProtocol = openWhisk.apiHost(host)
+
+  /**
+   * Generate a list of actions to execute. The list is a tuple of (kind, code, actionName,
main)
+   * `kind` is needed to create the action
+   * `code` is loaded form the files located in `resources/data`
+   * `actionName` is the name of the action in OpenWhisk
+   * `main` is only needed for java. This is the name of the class where the main method
is located.
+   */
+  val actions: Seq[(String, String, String, String)] = Map(
+    "nodejs:default" -> (FileUtils
+      .readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8),
"latencyTest_node", ""),
+    "python:default" -> (FileUtils
+      .readFileToString(Resource.body("pythonAction.py").get.file, StandardCharsets.UTF_8),
"latencyTest_python", ""),
+    "swift:default" -> (FileUtils
+      .readFileToString(Resource.body("swiftAction.swift").get.file, StandardCharsets.UTF_8),
"latencyTest_swift", ""),
+    "java:default" -> (Base64.getEncoder.encodeToString(
+      FileUtils.readFileToByteArray(Resource.body("javaAction.jar").get.file)), "latencyTest_java",
"JavaAction"))
+    .filterNot(e => excludedKinds.contains(e._1))
+    .map {
+      case (kind, (code, name, main)) =>
+        (kind, code, name, main)
+    }
+    .toSeq
+
+  // Define scenario
+  val test = scenario("Invoke one action after each other to test latency")
+    .foreach(actions, "action") {
+      val code: Expression[String] = "${action._2}"
+      exec(
+        openWhisk("Create ${action._1} action")
+          .authenticate(uuid, key)
+          .action("${action._3}")
+          .create(code, "${action._1}", "${action._4}"))
+        .exec(openWhisk("Cold ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke())
+        .repeat(100) {
+          exec(openWhisk("Warm ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke())
+        }
+        .exec(openWhisk("Delete ${action._1} action").authenticate(uuid, key).action("${action._3}").delete())
+    }
+
+  val testSetup = setUp(test.inject(atOnceUsers(1)))
+    .protocols(openWhiskProtocol)
+
+  actions
+    .map { case (kind, _, _, _) => s"Warm $kind invocation" }
+    .foldLeft(testSetup) { (agg, cur) =>
+      // One failure will make the build yellow
+      agg
+        .assertions(details(cur).responseTime.mean.lte(meanResponseTime))
+        .assertions(details(cur).responseTime.mean.lt(maximalMeanResponseTime))
+        // Mark the build yellow, if there are failed requests. And red if both conditions
fail.
+        .assertions(details(cur).failedRequests.count.is(0))
+        .assertions(details(cur).failedRequests.percent.lte(0.1))
+    }
+}
diff --git a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
index ef6d2ef..56dc6ce 100644
--- a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
+++ b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
@@ -23,12 +23,100 @@ import io.gatling.core.action.builder.ActionBuilder
 import io.gatling.core.session.Expression
 import io.gatling.core.structure.ScenarioContext
 import io.gatling.http.request.builder.{Http, HttpRequestBuilder}
+import spray.json.DefaultJsonProtocol._
+import spray.json._
 
 case class OpenWhiskActionBuilderBase(requestName: Expression[String]) {
 
   implicit private val http = new Http(requestName)
 
-  def info() = OpenWhiskActionBuilder(http.get("/api/v1"))
+  /** Call the `/api/v1`-endpoint of the specified system */
+  def info() = {
+    OpenWhiskActionBuilder(http.get("/api/v1"))
+  }
+
+  /**
+   * Specify authentication data. This is needed to perform operations on namespaces or working
with entities.
+   *
+   * @param uuid The UUID of the namespace
+   * @param key The key of the namespace
+   */
+  def authenticate(uuid: Expression[String], key: Expression[String]) = {
+    OpenWhiskActionBuilderWithNamespace(uuid, key)
+  }
+}
+
+case class OpenWhiskActionBuilderWithNamespace(private val uuid: Expression[String],
+                                               private val key: Expression[String],
+                                               private val namespace: String = "_")(implicit
private val http: Http) {
+
+  /**
+   * Specify on which namespace you want to perform any action.
+   *
+   * @param namespace The namespace you want to use.
+   */
+  def namespace(namespace: String) = {
+    OpenWhiskActionBuilderWithNamespace(uuid, key, namespace)
+  }
+
+  /** List all namespaces you have access to, with your current authentication. */
+  def list() = {
+    OpenWhiskActionBuilder(http.get("/api/v1/namespaces").basicAuth(uuid, key))
+  }
+
+  /**
+   * Perform any request against the actions-API. E.g. creating, invoking or deleting actions.
+   *
+   * @param actionName Name of the action in the Whisk-system.
+   */
+  def action(actionName: String) = {
+    OpenWhiskActionBuilderWithAction(uuid, key, namespace, actionName)
+  }
+}
+
+case class OpenWhiskActionBuilderWithAction(private val uuid: Expression[String],
+                                            private val key: Expression[String],
+                                            private val namespace: String,
+                                            private val action: String)(implicit private
val http: Http) {
+  private val path: Expression[String] = s"/api/v1/namespaces/$namespace/actions/$action"
+
+  /** Fetch the action from OpenWhisk */
+  def get() = {
+    OpenWhiskActionBuilder(http.get(path).basicAuth(uuid, key))
+  }
+
+  /** Delete the action from OpenWhisk */
+  def delete() = {
+    OpenWhiskActionBuilder(http.delete(path).basicAuth(uuid, key))
+  }
+
+  /**
+   * Create the action in OpenWhisk.
+   *
+   * @param code The code of the action to create.
+   * @param kind The kind of the action you want to create. Default is `nodejs:default`.
+   * @param main Main method of your action. This is only needed for java actions.
+   */
+  def create(code: Expression[String], kind: Expression[String] = "nodejs:default", main:
Expression[String] = "") = {
+
+    val json: Expression[String] = session => {
+      code(session).flatMap { c =>
+        kind(session).flatMap { k =>
+          main(session).map { m =>
+            val exec = Map("kind" -> k, "code" -> c) ++ (if (m.size > 0) Map("main"
-> m) else Map[String, String]())
+            JsObject("exec" -> exec.toJson).compactPrint
+          }
+        }
+      }
+    }
+
+    OpenWhiskActionBuilder(http.put(path).basicAuth(uuid, key).body(StringBody(json)))
+  }
+
+  /** Invoke the action. */
+  def invoke() = {
+    OpenWhiskActionBuilder(http.post(path).queryParam("blocking", "true").basicAuth(uuid,
key))
+  }
 }
 
 case class OpenWhiskActionBuilder(http: HttpRequestBuilder) extends ActionBuilder {

-- 
To stop receiving notification emails like this one, please contact
markusthoemmes@apache.org.

Mime
View raw message