openwhisk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markusthoem...@apache.org
Subject [incubator-openwhisk] branch master updated: Make error count and percentage configurable for gatling tests. (#3942)
Date Wed, 08 Aug 2018 14:12:07 GMT
This is an automated email from the ASF dual-hosted git repository.

markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new 92a4a0c  Make error count and percentage configurable for gatling tests. (#3942)
92a4a0c is described below

commit 92a4a0c314821681043716cc601b64d543f61e80
Author: Jeremias Werner <jeremias.werner@gmail.com>
AuthorDate: Wed Aug 8 16:12:01 2018 +0200

    Make error count and percentage configurable for gatling tests. (#3942)
---
 tests/performance/README.md                        | 56 +++++++++++++---------
 .../src/gatling/scala/ApiV1Simulation.scala        |  6 ++-
 .../scala/BlockingInvokeOneActionSimulation.scala  |  6 ++-
 .../scala/ColdBlockingInvokeSimulation.scala       |  6 ++-
 .../src/gatling/scala/LatencySimulation.scala      | 37 ++++++++++----
 5 files changed, 75 insertions(+), 36 deletions(-)

diff --git a/tests/performance/README.md b/tests/performance/README.md
index f09d91b..dd780c3 100644
--- a/tests/performance/README.md
+++ b/tests/performance/README.md
@@ -70,11 +70,13 @@ The test is doing as many requests as possible for the given amount of
time (`SE
 Available environment variables:
 
 ```
-OPENWHISK_HOST          (required)
-CONNECTIONS             (required)
-SECONDS                 (default: 10)
-REQUESTS_PER_SEC        (required)
-MIN_REQUESTS_PER_SEC    (default: REQUESTS_PER_SEC)
+OPENWHISK_HOST                (required)
+CONNECTIONS                   (required)
+SECONDS                       (default: 10)
+REQUESTS_PER_SEC              (required)
+MIN_REQUESTS_PER_SEC          (default: REQUESTS_PER_SEC)
+MAX_ERRORS_ALLOWED            (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
 ```
 
 You can run the simulation with (in OPENWHISK_HOME)
@@ -101,14 +103,20 @@ The comparison of the thresholds is against the mean response times
of the warm
 Available environment variables:
 
 ```
-OPENWHISK_HOST          (required)
-API_KEY                 (required, format: UUID:KEY)
-PAUSE_BETWEEN_INVOKES   (default: 0)
-MEAN_RESPONSE_TIME      (required)
-MAX_MEAN_RESPONSE_TIME  (default: MEAN_RESPONSE_TIME)
-EXCLUDED_KINDS          (default: "", format: "python:default,java:default,swift:default")
+OPENWHISK_HOST                (required)
+API_KEY                       (required, format: UUID:KEY)
+PAUSE_BETWEEN_INVOKES         (default: 0)
+MEAN_RESPONSE_TIME            (required)
+MAX_MEAN_RESPONSE_TIME        (default: MEAN_RESPONSE_TIME)
+EXCLUDED_KINDS                (default: "", format: "python:default,java:default,swift:default")
+MAX_ERRORS_ALLOWED            (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
 ```
 
+It is possible to override the `MEAN_RESPONSE_TIME`, `MAX_MEAN_RESPONSE_TIME`, `MAX_ERRORS_ALLOWED`
and `MAX_ERRORS_ALLOWED_PERCENTAGE`
+for each kind by adding the kind as prefix in upper case, like `JAVA_MEAN_RESPONSE_TIME`.
+
+
 You can run the simulation with (in OPENWHISK_HOME)
 ```
 OPENWHISK_HOST="openwhisk.mydomain.com" MEAN_RESPONSE_TIME="20" API_KEY="UUID:KEY" ./gradlew
gatlingRun-LatencySimulation
@@ -130,12 +138,14 @@ The test is doing as many requests as possible for the given amount
of time (`SE
 
 Available environment variables:
 ```
-OPENWHISK_HOST          (required)
-API_KEY                 (required, format: UUID:KEY)
-CONNECTIONS             (required)
-SECONDS                 (default: 10)
-REQUESTS_PER_SEC        (required)
-MIN_REQUESTS_PER_SEC    (default: REQUESTS_PER_SEC)
+OPENWHISK_HOST                (required)
+API_KEY                       (required, format: UUID:KEY)
+CONNECTIONS                   (required)
+SECONDS                       (default: 10)
+REQUESTS_PER_SEC              (required)
+MIN_REQUESTS_PER_SEC          (default: REQUESTS_PER_SEC)
+MAX_ERRORS_ALLOWED            (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
 ```
 
 You can run the simulation with
@@ -160,11 +170,13 @@ The test is doing as many requests as possible for the given amount
of time (`SE
 
 Available environment variables:
 ```
-OPENWHISK_HOST          (required)
-USERS                   (required)
-SECONDS                 (default: 10)
-REQUESTS_PER_SEC        (required)
-MIN_REQUESTS_PER_SEC    (default: REQUESTS_PER_SEC)
+OPENWHISK_HOST                (required)
+USERS                         (required)
+SECONDS                       (default: 10)
+REQUESTS_PER_SEC              (required)
+MIN_REQUESTS_PER_SEC          (default: REQUESTS_PER_SEC)
+MAX_ERRORS_ALLOWED            (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
 ```
 
 You can run the simulation with
diff --git a/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala b/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala
index 91c5302..9fa7604 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala
@@ -30,6 +30,8 @@ class ApiV1Simulation extends Simulation {
   // Specify thresholds
   val requestsPerSec = sys.env("REQUESTS_PER_SEC").toInt
   val minimalRequestsPerSec = sys.env.getOrElse("MIN_REQUESTS_PER_SEC", requestsPerSec.toString).toInt
+  val maxErrorsAllowed: Int = sys.env.getOrElse("MAX_ERRORS_ALLOWED", "0").toInt
+  val maxErrorsAllowedPercentage: Double = sys.env.getOrElse("MAX_ERRORS_ALLOWED_PERCENTAGE",
"0.1").toDouble
 
   // Generate the OpenWhiskProtocol
   val openWhiskProtocol = openWhisk.apiHost(host)
@@ -46,6 +48,6 @@ class ApiV1Simulation extends Simulation {
     .assertions(details("Call api/v1 endpoint").requestsPerSec.gt(minimalRequestsPerSec))
     .assertions(details("Call api/v1 endpoint").requestsPerSec.gt(requestsPerSec))
     // Mark the build yellow, if there are failed requests. And red if both conditions fail.
-    .assertions(details("Call api/v1 endpoint").failedRequests.count.is(0))
-    .assertions(details("Call api/v1 endpoint").failedRequests.percent.lte(0.1))
+    .assertions(details("Call api/v1 endpoint").failedRequests.count.lte(maxErrorsAllowed))
+    .assertions(details("Call api/v1 endpoint").failedRequests.percent.lte(maxErrorsAllowedPercentage))
 }
diff --git a/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala
b/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala
index 79f2751..e49ce52 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala
@@ -39,6 +39,8 @@ class BlockingInvokeOneActionSimulation extends Simulation {
   // Specify thresholds
   val requestsPerSec: Int = sys.env("REQUESTS_PER_SEC").toInt
   val minimalRequestsPerSec: Int = sys.env.getOrElse("MIN_REQUESTS_PER_SEC", requestsPerSec.toString).toInt
+  val maxErrorsAllowed: Int = sys.env.getOrElse("MAX_ERRORS_ALLOWED", "0").toInt
+  val maxErrorsAllowedPercentage: Double = sys.env.getOrElse("MAX_ERRORS_ALLOWED_PERCENTAGE",
"0.1").toDouble
 
   // Generate the OpenWhiskProtocol
   val openWhiskProtocol: OpenWhiskProtocolBuilder = openWhisk.apiHost(host)
@@ -74,6 +76,6 @@ class BlockingInvokeOneActionSimulation extends Simulation {
     .assertions(details("Invoke action").requestsPerSec.gt(minimalRequestsPerSec))
     .assertions(details("Invoke action").requestsPerSec.gt(requestsPerSec))
     // Mark the build yellow, if there are failed requests. And red if both conditions fail.
-    .assertions(details("Invoke action").failedRequests.count.is(0))
-    .assertions(details("Invoke action").failedRequests.percent.lte(0.1))
+    .assertions(details("Invoke action").failedRequests.count.lte(maxErrorsAllowed))
+    .assertions(details("Invoke action").failedRequests.percent.lte(maxErrorsAllowedPercentage))
 }
diff --git a/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
b/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
index 1e06a8b..d7ec74f 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
@@ -38,6 +38,8 @@ class ColdBlockingInvokeSimulation extends Simulation {
   // Specify thresholds
   val requestsPerSec: Int = sys.env("REQUESTS_PER_SEC").toInt
   val minimalRequestsPerSec: Int = sys.env.getOrElse("MIN_REQUESTS_PER_SEC", requestsPerSec.toString).toInt
+  val maxErrorsAllowed: Int = sys.env.getOrElse("MAX_ERRORS_ALLOWED", "0").toInt
+  val maxErrorsAllowedPercentage: Double = sys.env.getOrElse("MAX_ERRORS_ALLOWED_PERCENTAGE",
"0.1").toDouble
 
   // Generate the OpenWhiskProtocol
   val openWhiskProtocol: OpenWhiskProtocolBuilder = openWhisk.apiHost(host)
@@ -85,6 +87,6 @@ class ColdBlockingInvokeSimulation extends Simulation {
     .assertions(details("Invoke action").requestsPerSec.gt(minimalRequestsPerSec))
     .assertions(details("Invoke action").requestsPerSec.gt(requestsPerSec))
     // Mark the build yellow, if there are failed requests. And red if both conditions fail.
-    .assertions(details("Invoke action").failedRequests.count.is(0))
-    .assertions(details("Invoke action").failedRequests.percent.lte(0.1))
+    .assertions(details("Invoke action").failedRequests.count.lte(maxErrorsAllowed))
+    .assertions(details("Invoke action").failedRequests.percent.lte(maxErrorsAllowedPercentage))
 }
diff --git a/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala b/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
index a685681..8830c78 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
@@ -35,9 +35,19 @@ class LatencySimulation extends Simulation {
 
   val pauseBetweenInvokes: Int = sys.env.getOrElse("PAUSE_BETWEEN_INVOKES", "0").toInt
 
+  val MEAN_RESPONSE_TIME = "MEAN_RESPONSE_TIME"
+  val MAX_MEAN_RESPONSE_TIME = "MAX_MEAN_RESPONSE_TIME"
+  val MAX_ERRORS_ALLOWED = "MAX_ERRORS_ALLOWED"
+  val MAX_ERRORS_ALLOWED_PERCENTAGE = "MAX_ERRORS_ALLOWED_PERCENTAGE"
+
   // Specify thresholds
-  val meanResponseTime: Int = sys.env("MEAN_RESPONSE_TIME").toInt
-  val maximalMeanResponseTime: Int = sys.env.getOrElse("MAX_MEAN_RESPONSE_TIME", meanResponseTime.toString).toInt
+  val meanResponseTime: Int = sys.env(MEAN_RESPONSE_TIME).toInt
+  val maximalMeanResponseTime: Int = sys.env.getOrElse(MAX_MEAN_RESPONSE_TIME, meanResponseTime.toString).toInt
+
+  val maxErrorsAllowed: Int = sys.env.getOrElse(MAX_ERRORS_ALLOWED, "0").toInt
+  val maxErrorsAllowedPercentage: Double = sys.env.getOrElse(MAX_ERRORS_ALLOWED_PERCENTAGE,
"0.1").toDouble
+
+  def toKindSpecificKey(kind: String, suffix: String) = kind.split(':').head.toUpperCase
+ "_" + suffix
 
   // Exclude runtimes
   val excludedKinds: Seq[String] = sys.env.getOrElse("EXCLUDED_KINDS", "").split(",")
@@ -91,14 +101,25 @@ class LatencySimulation extends Simulation {
     .protocols(openWhiskProtocol)
 
   actions
-    .map { case (kind, _, _, _) => s"Warm $kind invocation" }
-    .foldLeft(testSetup) { (agg, cur) =>
+    .map { case (kind, _, _, _) => kind }
+    .foldLeft(testSetup) { (agg, kind) =>
+      val cur = s"Warm $kind invocation"
       // One failure will make the build yellow
+      val specificMeanResponseTime: Int =
+        sys.env.getOrElse(toKindSpecificKey(kind, MEAN_RESPONSE_TIME), meanResponseTime.toString).toInt
+      val specificMaxMeanResponseTime =
+        sys.env.getOrElse(toKindSpecificKey(kind, MAX_MEAN_RESPONSE_TIME), maximalMeanResponseTime.toString).toInt
+      val specificMaxErrorsAllowed =
+        sys.env.getOrElse(toKindSpecificKey(kind, MAX_ERRORS_ALLOWED), maxErrorsAllowed.toString).toInt
+      val specificMaxErrorsAllowedPercentage = sys.env
+        .getOrElse(toKindSpecificKey(kind, MAX_ERRORS_ALLOWED_PERCENTAGE), maxErrorsAllowedPercentage.toString)
+        .toDouble
+
       agg
-        .assertions(details(cur).responseTime.mean.lte(meanResponseTime))
-        .assertions(details(cur).responseTime.mean.lt(maximalMeanResponseTime))
+        .assertions(details(cur).responseTime.mean.lte(specificMeanResponseTime))
+        .assertions(details(cur).responseTime.mean.lt(specificMaxMeanResponseTime))
         // Mark the build yellow, if there are failed requests. And red if both conditions
fail.
-        .assertions(details(cur).failedRequests.count.is(0))
-        .assertions(details(cur).failedRequests.percent.lte(0.1))
+        .assertions(details(cur).failedRequests.count.lte(specificMaxErrorsAllowed))
+        .assertions(details(cur).failedRequests.percent.lte(specificMaxErrorsAllowedPercentage))
     }
 }


Mime
View raw message