openwhisk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From markusthoem...@apache.org
Subject [incubator-openwhisk] branch master updated: Allow Elasticsearch client to skip query result hits. (#3849)
Date Thu, 12 Jul 2018 07:55:56 GMT
This is an automated email from the ASF dual-hosted git repository.

markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new bcba2d1  Allow Elasticsearch client to skip query result hits. (#3849)
bcba2d1 is described below

commit bcba2d1d2228e17c5bdb6e9d3cf666fd946b9216
Author: James Dubee <jwdubee@us.ibm.com>
AuthorDate: Thu Jul 12 03:55:53 2018 -0400

    Allow Elasticsearch client to skip query result hits. (#3849)
---
 .../logging/ElasticSearchRestClient.scala          | 11 ++-----
 .../logging/ElasticSearchLogStoreTests.scala       |  3 +-
 .../logging/ElasticSearchRestClientTests.scala     | 36 +++++++++++++++-------
 3 files changed, 30 insertions(+), 20 deletions(-)

diff --git a/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala
b/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala
index 0290e96..9218958 100644
--- a/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala
+++ b/common/scala/src/main/scala/whisk/core/containerpool/logging/ElasticSearchRestClient.scala
@@ -57,7 +57,6 @@ case class EsQueryAggs(aggField: String, agg: EsAgg, field: String)
 case class EsQueryRange(key: String, range: EsRange, value: String)
 case class EsQueryBoolMatch(key: String, value: String)
 case class EsQueryOrder(field: String, kind: EsOrder)
-case class EsQuerySize(size: Integer)
 case class EsQueryAll() extends EsQueryMethod
 case class EsQueryMust(matches: Vector[EsQueryBoolMatch], range: Vector[EsQueryRange] = Vector.empty)
     extends EsQueryMethod
@@ -66,7 +65,8 @@ case class EsQueryTerm(key: String, value: String) extends EsQueryMethod
 case class EsQueryString(queryString: String) extends EsQueryMethod
 case class EsQuery(query: EsQueryMethod,
                    sort: Option[EsQueryOrder] = None,
-                   size: Option[EsQuerySize] = None,
+                   size: Option[Int] = None,
+                   from: Int = 0,
                    aggs: Option[EsQueryAggs] = None)
 
 // Schema of ES query results
@@ -121,11 +121,6 @@ object ElasticSearchJsonProtocol extends DefaultJsonProtocol {
       JsArray(JsObject(query.field -> JsObject("order" -> query.kind.toString.toJson)))
   }
 
-  implicit object EsQuerySizeJsonFormat extends RootJsonFormat[EsQuerySize] {
-    def read(query: JsValue) = ???
-    def write(query: EsQuerySize) = JsNumber(query.size)
-  }
-
   implicit object EsQueryAggsJsonFormat extends RootJsonFormat[EsQueryAggs] {
     def read(query: JsValue) = ???
     def write(query: EsQueryAggs) =
@@ -148,7 +143,7 @@ object ElasticSearchJsonProtocol extends DefaultJsonProtocol {
     }
   }
 
-  implicit val esQueryFormat = jsonFormat4(EsQuery.apply)
+  implicit val esQueryFormat = jsonFormat5(EsQuery.apply)
   implicit val esSearchHitFormat = jsonFormat(EsSearchHit.apply _, "_source")
   implicit val esSearchHitsFormat = jsonFormat2(EsSearchHits.apply)
   implicit val esSearchResultFormat = jsonFormat1(EsSearchResult.apply)
diff --git a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala
b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala
index bd3e0a0..91db374 100644
--- a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala
+++ b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchLogStoreTests.scala
@@ -79,7 +79,8 @@ class ElasticSearchLogStoreTests
     "query" -> JsObject(
       "query_string" -> JsObject("query" -> JsString(
         s"_type: ${defaultConfig.logSchema.userLogs} AND ${defaultConfig.logSchema.activationId}:
$activationId"))),
-    "sort" -> JsArray(JsObject(defaultConfig.logSchema.time -> JsObject("order" ->
JsString("asc"))))).compactPrint
+    "sort" -> JsArray(JsObject(defaultConfig.logSchema.time -> JsObject("order" ->
JsString("asc")))),
+    "from" -> JsNumber(0)).compactPrint
   private val defaultHttpRequest = HttpRequest(
     POST,
     Uri(s"/whisk_user_logs/_search"),
diff --git a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala
b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala
index 040b0fc..2a43b79 100644
--- a/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala
+++ b/tests/src/test/scala/whisk/core/containerpool/logging/ElasticSearchRestClientTests.scala
@@ -87,7 +87,8 @@ class ElasticSearchRestClientTests
               "must" ->
                 JsArray(
                   JsObject("match" -> JsObject("someKey1" -> JsString("someValue1"))),
-                  JsObject("match" -> JsObject("someKey2" -> JsString("someValue2")))))))
+                  JsObject("match" -> JsObject("someKey2" -> JsString("someValue2")))))),
+      "from" -> 0.toJson)
 
     // Test must with ranges
     Seq((EsRangeGte, "gte"), (EsRangeGt, "gt"), (EsRangeLte, "lte"), (EsRangeLt, "lt")).foreach
{
@@ -109,7 +110,8 @@ class ElasticSearchRestClientTests
                   "filter" ->
                     JsArray(
                       JsObject("range" -> JsObject("someKey1" -> JsObject(rangeValue
-> "someValue1".toJson))),
-                      JsObject("range" -> JsObject("someKey2" -> JsObject(rangeValue
-> "someValue2".toJson)))))))
+                      JsObject("range" -> JsObject("someKey2" -> JsObject(rangeValue
-> "someValue2".toJson)))))),
+          "from" -> 0.toJson)
     }
   }
 
@@ -120,7 +122,8 @@ class ElasticSearchRestClientTests
 
         EsQuery(EsQueryAll(), aggs = Some(queryAgg)).toJson shouldBe JsObject(
           "query" -> JsObject("match_all" -> JsObject.empty),
-          "aggs" -> JsObject("someAgg" -> JsObject(aggValue -> JsObject("field"
-> "someField".toJson))))
+          "aggs" -> JsObject("someAgg" -> JsObject(aggValue -> JsObject("field"
-> "someField".toJson))),
+          "from" -> 0.toJson)
     }
   }
 
@@ -128,7 +131,8 @@ class ElasticSearchRestClientTests
     val queryMatch = EsQueryMatch("someField", "someValue")
 
     EsQuery(queryMatch).toJson shouldBe JsObject(
-      "query" -> JsObject("match" -> JsObject("someField" -> JsObject("query" ->
"someValue".toJson))))
+      "query" -> JsObject("match" -> JsObject("someField" -> JsObject("query" ->
"someValue".toJson))),
+      "from" -> 0.toJson)
 
     // Test match with types
     Seq((EsMatchPhrase, "phrase"), (EsMatchPhrasePrefix, "phrase_prefix")).foreach {
@@ -137,21 +141,25 @@ class ElasticSearchRestClientTests
 
         EsQuery(queryMatch).toJson shouldBe JsObject(
           "query" -> JsObject(
-            "match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson,
"type" -> typeValue.toJson))))
+            "match" -> JsObject("someField" -> JsObject("query" -> "someValue".toJson,
"type" -> typeValue.toJson))),
+          "from" -> 0.toJson)
     }
   }
 
   it should "construct a query with term" in {
     val queryTerm = EsQueryTerm("user", "someUser")
 
-    EsQuery(queryTerm).toJson shouldBe JsObject("query" -> JsObject("term" -> JsObject("user"
-> JsString("someUser"))))
+    EsQuery(queryTerm).toJson shouldBe JsObject(
+      "query" -> JsObject("term" -> JsObject("user" -> JsString("someUser"))),
+      "from" -> 0.toJson)
   }
 
   it should "construct a query with query string" in {
     val queryString = EsQueryString("_type: someType")
 
     EsQuery(queryString).toJson shouldBe JsObject(
-      "query" -> JsObject("query_string" -> JsObject("query" -> JsString("_type:
someType"))))
+      "query" -> JsObject("query_string" -> JsObject("query" -> JsString("_type:
someType"))),
+      "from" -> 0.toJson)
   }
 
   it should "construct a query with order" in {
@@ -161,16 +169,22 @@ class ElasticSearchRestClientTests
 
         EsQuery(EsQueryAll(), Some(queryOrder)).toJson shouldBe JsObject(
           "query" -> JsObject("match_all" -> JsObject.empty),
-          "sort" -> JsArray(JsObject("someField" -> JsObject("order" -> orderValue.toJson))))
+          "sort" -> JsArray(JsObject("someField" -> JsObject("order" -> orderValue.toJson))),
+          "from" -> 0.toJson)
     }
   }
 
   it should "construct query with size" in {
-    val querySize = EsQuerySize(1)
+    EsQuery(EsQueryAll(), size = Some(1)).toJson shouldBe JsObject(
+      "query" -> JsObject("match_all" -> JsObject.empty),
+      "size" -> 1.toJson,
+      "from" -> 0.toJson)
+  }
 
-    EsQuery(EsQueryAll(), size = Some(querySize)).toJson shouldBe JsObject(
+  it should "construct query with from" in {
+    EsQuery(EsQueryAll(), from = 1).toJson shouldBe JsObject(
       "query" -> JsObject("match_all" -> JsObject.empty),
-      "size" -> JsNumber(1))
+      "from" -> 1.toJson)
   }
 
   it should "error when search response does not match expected type" in {


Mime
View raw message