openwhisk-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From chet...@apache.org
Subject [incubator-openwhisk] branch master updated: Improve splunk query (#4047)
Date Thu, 04 Oct 2018 07:55:47 GMT
This is an automated email from the ASF dual-hosted git repository.

chetanm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new 27a624e  Improve splunk query (#4047)
27a624e is described below

commit 27a624ecc763bc01eaa997ba9ce5fc55ad748221
Author: tysonnorris <tysonnorris@gmail.com>
AuthorDate: Thu Oct 4 00:54:49 2018 -0700

    Improve splunk query (#4047)
    
    Improves the Splunk query and log format
    
    * Log is now formatted similar to DockerToActivationLogStore
    * Query time offset is made configurable to account for log collection delays
---
 common/scala/src/main/resources/application.conf     | 18 ++++++++++++++++++
 .../core/containerpool/logging/SplunkLogStore.scala  | 18 ++++++++++++++----
 .../containerpool/logging/SplunkLogStoreTests.scala  | 20 ++++++++++++++------
 3 files changed, 46 insertions(+), 10 deletions(-)

diff --git a/common/scala/src/main/resources/application.conf b/common/scala/src/main/resources/application.conf
index b2bcbc6..712b3ef 100644
--- a/common/scala/src/main/resources/application.conf
+++ b/common/scala/src/main/resources/application.conf
@@ -209,6 +209,24 @@ whisk {
         teardown-on-exit = true //set to true to disable the mesos framework on system exit;
set for false for HA deployments
     }
 
+    logstore {
+        #SplunkLogStore configuration
+        #splunk {
+        #    host = "splunkhost"                   #splunk api hostname
+        #    port = 8089                           #splunk api port
+        #    username = "splunkapiusername"        #splunk api username
+        #    password = "splunkapipassword"        #splunk api password
+        #    index = "splunkindex"                 #splunk index name
+        #    log-timestamp-field = "log_timestamp" #splunk field where timestamp is stored
(to reflect log event generated time, not splunk's _time)
+        #    log-stream-field = "log_stream"       #splunk field where stream is stored (stdout/stderr)
+        #    log-message-field = "log_message"     #splunk field where log message is stored
+        #    activation-id-field = "activation_id" #splunk field where activation id is stored
+        #    query-constraints = ""                #additional constraints for splunk queries
+        #    query-timestamp-offset-seconds = ""   #splunk query will be broadened by this
2*<offset value>; e.g. "earliest_time=activation.start - offset" and "latest_time=activation.end
+ offset"
+        #    disableSNI = false                    #if true, disables hostname validation
and cert validation (in case splunk api endpoint is using a self signed cert)
+        #}
+    }
+
     # tracing configuration
     tracing {
         cache-expiry = 30 seconds #how long to keep spans in cache. Set to appropriate value
to trace long running requests
diff --git a/common/scala/src/main/scala/whisk/core/containerpool/logging/SplunkLogStore.scala
b/common/scala/src/main/scala/whisk/core/containerpool/logging/SplunkLogStore.scala
index 1cd93b5..f22ce33 100644
--- a/common/scala/src/main/scala/whisk/core/containerpool/logging/SplunkLogStore.scala
+++ b/common/scala/src/main/scala/whisk/core/containerpool/logging/SplunkLogStore.scala
@@ -60,8 +60,12 @@ case class SplunkLogStoreConfig(host: String,
                                 username: String,
                                 password: String,
                                 index: String,
+                                logTimestampField: String,
+                                logStreamField: String,
                                 logMessageField: String,
                                 activationIdField: String,
+                                queryConstraints: String,
+                                queryTimestampOffsetSeconds: Int,
                                 disableSNI: Boolean)
 case class SplunkResponse(results: Vector[JsObject])
 object SplunkResponseJsonProtocol extends DefaultJsonProtocol {
@@ -106,16 +110,18 @@ class SplunkLogStore(
     //    {"preview":false,"init_offset":0,"messages":[],"fields":[{"name":"log_message"}],"results":[{"log_message":"some
log message"}], "highlighted":{}}
     //note: splunk returns results in reverse-chronological order, therefore we include "|
reverse" to cause results to arrive in chronological order
     val search =
-      s"""search index="${splunkConfig.index}"| spath ${splunkConfig.activationIdField}|
search ${splunkConfig.activationIdField}=${activation.activationId.toString}| table ${splunkConfig.logMessageField}|
reverse"""
+      s"""search index="${splunkConfig.index}"| spath ${splunkConfig.activationIdField}|
search ${splunkConfig.queryConstraints} ${splunkConfig.activationIdField}=${activation.activationId.toString}|
table ${splunkConfig.logTimestampField}, ${splunkConfig.logStreamField}, ${splunkConfig.logMessageField}|
reverse"""
 
     val entity = FormData(
       Map(
         "exec_mode" -> "oneshot",
         "search" -> search,
         "output_mode" -> "json",
-        "earliest_time" -> activation.start.toString, //assume that activation start/end
are UTC zone, and splunk events are the same
+        "earliest_time" -> activation.start
+          .minusSeconds(splunkConfig.queryTimestampOffsetSeconds)
+          .toString, //assume that activation start/end are UTC zone, and splunk events are
the same
         "latest_time" -> activation.end
-          .plusSeconds(5) //add 5s to avoid a timerange of 0 on short-lived activations
+          .plusSeconds(splunkConfig.queryTimestampOffsetSeconds) //add 5s to avoid a timerange
of 0 on short-lived activations
           .toString)).toEntity
 
     logging.debug(this, "sending request")
@@ -130,7 +136,11 @@ class SplunkLogStore(
           .map(r => {
             ActivationLogs(
               r.results
-                .map(_.fields(splunkConfig.logMessageField).convertTo[String]))
+                .map(l =>
+                  //format same as whisk.core.containerpool.logging.LogLine.toFormattedString
+                  f"${l.fields(splunkConfig.logTimestampField).convertTo[String]}%-30s ${l
+                    .fields(splunkConfig.logStreamField)
+                    .convertTo[String]}: ${l.fields(splunkConfig.logMessageField).convertTo[String].trim}"))
           })
       })
   }
diff --git a/tests/src/test/scala/whisk/core/containerpool/logging/SplunkLogStoreTests.scala
b/tests/src/test/scala/whisk/core/containerpool/logging/SplunkLogStoreTests.scala
index dae47e6..c7d2935 100644
--- a/tests/src/test/scala/whisk/core/containerpool/logging/SplunkLogStoreTests.scala
+++ b/tests/src/test/scala/whisk/core/containerpool/logging/SplunkLogStoreTests.scala
@@ -60,15 +60,20 @@ class SplunkLogStoreTests
     "splunk-user",
     "splunk-pass",
     "splunk-index",
+    "log_timestamp",
+    "log_stream",
     "log_message",
     "activation_id",
+    "somefield::somevalue",
+    22,
     disableSNI = false)
 
   behavior of "Splunk LogStore"
 
   val startTime = "2007-12-03T10:15:30Z"
+  val startTimePlusOffset = "2007-12-03T10:15:08Z" //queried end time range is endTime-22
   val endTime = "2007-12-03T10:15:45Z"
-  val endTimePlus5 = "2007-12-03T10:15:50Z" //queried end time range is endTime+5
+  val endTimePlusOffset = "2007-12-03T10:16:07Z" //queried end time range is endTime+22
   val uuid = UUID()
   val user =
     Identity(Subject(), Namespace(EntityName("testSpace"), uuid), BasicAuthenticationAuthKey(uuid,
Secret()), Set.empty)
@@ -110,12 +115,12 @@ class SplunkLogStoreTests
 
               request.uri.path.toString() shouldBe "/services/search/jobs"
               request.headers shouldBe List(Authorization.basic(testConfig.username, testConfig.password))
-              earliestTime shouldBe Some(startTime)
-              latestTime shouldBe Some(endTimePlus5)
+              earliestTime shouldBe Some(startTimePlusOffset)
+              latestTime shouldBe Some(endTimePlusOffset)
               outputMode shouldBe Some("json")
               execMode shouldBe Some("oneshot")
               search shouldBe Some(
-                s"""search index="${testConfig.index}"| spath ${testConfig.activationIdField}|
search ${testConfig.activationIdField}=${activation.activationId.toString}| table ${testConfig.logMessageField}|
reverse""")
+                s"""search index="${testConfig.index}"| spath ${testConfig.activationIdField}|
search ${testConfig.queryConstraints} ${testConfig.activationIdField}=${activation.activationId.toString}|
table ${testConfig.logTimestampField}, ${testConfig.logStreamField}, ${testConfig.logMessageField}|
reverse""")
 
               (
                 Success(
@@ -123,7 +128,7 @@ class SplunkLogStoreTests
                     StatusCodes.OK,
                     entity = HttpEntity(
                       ContentTypes.`application/json`,
-                      """{"preview":false,"init_offset":0,"messages":[],"fields":[{"name":"log_message"}],"results":[{"log_message":"some
log message"},{"log_message":"some other log message"}], "highlighted":{}}"""))),
+                      """{"preview":false,"init_offset":0,"messages":[],"fields":[{"name":"log_message"}],"results":[{"log_timestamp":
"2007-12-03T10:15:30Z", "log_stream":"stdout", "log_message":"some log message"},{"log_timestamp":
"2007-12-03T10:15:31Z", "log_stream":"stderr", "log_message":"some other log message"}], "highlighted":{}}"""))),
                 userContext)
             }
             .recover {
@@ -148,7 +153,10 @@ class SplunkLogStoreTests
     //use the a flow that asserts the request structure and provides a response in the expected
format
     val splunkStore = new SplunkLogStore(system, Some(testFlow), testConfig)
     val result = await(splunkStore.fetchLogs(activation, context))
-    result shouldBe ActivationLogs(Vector("some log message", "some other log message"))
+    result shouldBe ActivationLogs(
+      Vector(
+        "2007-12-03T10:15:30Z           stdout: some log message",
+        "2007-12-03T10:15:31Z           stderr: some other log message"))
   }
 
   it should "fail to connect to bogus host" in {


Mime
View raw message