metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject [18/44] metron git commit: METRON-943 Create traffic connections report in zeppelin (justinleet) closes apache/incubator-metron#573
Date Thu, 01 Jun 2017 21:41:37 GMT
METRON-943 Create traffic connections report in zeppelin (justinleet) closes apache/incubator-metron#573


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b3e7222f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b3e7222f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b3e7222f

Branch: refs/heads/Metron_0.4.0
Commit: b3e7222f1c4fb527ff3b18e8bb47383528d2eb57
Parents: e375936
Author: justinleet <justinjleet@gmail.com>
Authored: Fri May 12 08:49:28 2017 -0400
Committer: leet <leet@apache.org>
Committed: Fri May 12 08:49:28 2017 -0400

----------------------------------------------------------------------
 metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec   | 3 +++
 .../config/zeppelin/metron/metron-connection-volume-report.json   | 1 +
 2 files changed, 4 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/b3e7222f/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index e851c7f..b49e0e6 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -271,6 +271,7 @@ This package installs the Metron Indexing files
 %{metron_home}/config/zeppelin/metron/metron-yaf-telemetry.json
 %{metron_home}/config/zeppelin/metron/metron-connection-report.json
 %{metron_home}/config/zeppelin/metron/metron-ip-report.json
+%{metron_home}/config/zeppelin/metron/metron-connection-volume-report.json
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -400,6 +401,8 @@ This package installs the Metron Management UI %{metron_home}
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 %changelog
+* Tue May 9 2017 Apache Metron <dev@metron.apache.org> - 0.4.0
+- Add Zeppelin Connection Volume Report Dashboard
 * Thu May 4 2017 Ryan Merriman <merrimanr@gmail.com> - 0.4.0
 - Added REST
 * Tue May 2 2017 David Lyle <dlyle65535@gmail.com> - 0.4.0

http://git-wip-us.apache.org/repos/asf/metron/blob/b3e7222f/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
new file mode 100644
index 0000000..22765d4
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-connection-volume-report.json
@@ -0,0 +1 @@
+{"paragraphs":[{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339150477_-816854736","id":"20170509-141230_1340330181","dateCreated":"2017-05-09T14:12:30+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:7251","text":"%spark.sql\n\n#\n#
load the Yaf telemetry that has been archived by Metron\n#\ncreate temporary table yaf\n 
using org.apache.spark.sql.json\n  options (path \"hdfs:///apps/metron/indexing/indexed/yaf\")","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:11+0000","dateStarted":"2017-05-09T14:20:10+0000","result":{"code":"SUCCESS","type":"TEXT","msg":""}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_14943391
 84335_-2067041830","id":"20170509-141304_1479312597","dateCreated":"2017-05-09T14:13:04+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7322","text":"%md\n\n###
Connection Volume (Source) - Yaf\n\nThe volume of connections made from source IPs\n\nThis
IPs retrieved are given by an IPv4 CIDR block.","dateUpdated":"2017-05-09T14:22:02+0000","dateFinished":"2017-05-09T14:22:02+0000","dateStarted":"2017-05-09T14:22:02+0000","result":{"code":"SUCCESS","type":"HTML","msg":"<h3>Connection
Volume (Source) - Yaf</h3>\n<p>The volume of connections made from source IPs</p>\n<p>This
IPs retrieved are given by an IPv4 CIDR block.</p>\n"}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{"CIDR":"192.0.0.0/8"},"forms":{"CIDR":{"name":"CIDR","displayName":"CIDR","type":"input","defaultValue":"","hidden":false}}},"jobName":"paragraph_14943391
 91894_-766575224","id":"20170509-141311_2132481247","dateCreated":"2017-05-09T14:13:11+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7398","text":"%spark\nimport
org.apache.spark.sql.Row\nimport scala.concurrent.duration._\nimport java.util.concurrent.TimeUnit\nimport
org.apache.commons.net.util.SubnetUtils\n\nval cidr = z.input(\"CIDR\").toString\n@transient
val utils = new SubnetUtils(cidr)\nutils.setInclusiveHostCount(true)\n@transient val info
= utils.getInfo\n\nval ipToLong = (ipAddress: String) => {\n    val ipAddressInArray =
ipAddress.split(\"\\\\.\")\n    var result = 0L\n    for (i <- 0 to ipAddressInArray.length-1)
{\n\t    val power = 3 - i\n\t    val ip = Integer.parseInt(ipAddressInArray(i))\n\t    result
+= ip * Math.pow(256.0, power.toDouble).toLong\n    }\n    result\n}\n\nval bcLow = sc.broadcast(ipToLong(info.getLowAddress))\nval
bcHigh = sc.broadcast(ipToLong(info.getHighAddress))\nval bcIpToLong = sc.broadcast(ipToLong
 )\n\nval results = sqlContext.sql(\ns\"\"\"SELECT\n    ip_src_addr,\n    COUNT(*) AS count\nFROM\n
   yaf\nGROUP BY ip_src_addr\nORDER BY ip_src_addr\n\"\"\").flatMap {\n  case Row(ip_src_addr:
String, count: Long) => {\n        val longSrc = bcIpToLong.value(ip_src_addr)\n      
 if(bcLow.value <= longSrc && longSrc <= bcHigh.value) {\n\t\t    List(ip_src_addr
+ \"\\t\" + count)\n        } else {\n            List.empty[String]\n        }\n  }\n  }.collect()\n\nprint(\"%table
ip_src_addr\\tcount\\n\" + results.mkString(\"\\n\"))","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:21+0000","dateStarted":"2017-05-09T14:20:10+0000","result":{"code":"SUCCESS","type":"TABLE","msg":"ip_src_addr\tcount\n192.168.138.158\t111\n192.168.138.2\t1\n192.168.66.1\t89\n192.168.66.121\t69","comment":"","msgTable":[[{"key":"count","value":"192.168.138.158"},{"key":"count","value":"111"}],[{"value":"192.168.138.2"},{"value":"1"}],[{"value":"192.168.66.1"},{"value":"89"}],[{"val
 ue":"192.168.66.121"},{"value":"69"}]],"columnNames":[{"name":"ip_src_addr","index":0,"aggr":"sum"},{"name":"count","index":1,"aggr":"sum"}],"rows":[["192.168.138.158","111"],["192.168.138.2","1"],["192.168.66.1","89"],["192.168.66.121","69"]]}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339578537_-194906756","id":"20170509-141938_486503393","dateCreated":"2017-05-09T14:19:38+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7794","text":"%md\n\n###
Connection Volume (Destination) - Yaf\n\nThe volume of connections made to destination IPs.\n\nThis
IPs retrieved are given by an IPv4 CIDR block.","dateUpdated":"2017-05-09T14:22:06+0000","dateFinished":"2017-05-09T14:22:06+0000","dateStarted":"2017-05-09T14:22:06+0000","result":{"code":"SUCCESS","type":"HTM
 L","msg":"<h3>Connection Volume (Destination) - Yaf</h3>\n<p>The volume
of connections made to destination IPs.</p>\n<p>This IPs retrieved are given by
an IPv4 CIDR block.</p>\n"}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/scala"},"settings":{"params":{"CIDR":"192.0.0.0/8"},"forms":{"CIDR":{"name":"CIDR","displayName":"CIDR","type":"input","defaultValue":"","hidden":false}}},"jobName":"paragraph_1494339202329_1284921236","id":"20170509-141322_1098639923","dateCreated":"2017-05-09T14:13:22+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7474","text":"%spark\nimport
org.apache.spark.sql.Row\nimport scala.concurrent.duration._\nimport java.util.concurrent.TimeUnit\nimport
org.apache.commons.net.util.SubnetUtils\n\nval cidr = z.input(\"CIDR\").toString\n@transient
val utils = new SubnetUtils(cidr)\nutils.setInclusiveHostCo
 unt(true)\n@transient val info = utils.getInfo\n\nval ipToLong = (ipAddress: String) =>
{\n    val ipAddressInArray = ipAddress.split(\"\\\\.\")\n    var result = 0L\n    for (i
<- 0 to ipAddressInArray.length-1) {\n\t    val power = 3 - i\n\t    val ip = Integer.parseInt(ipAddressInArray(i))\n\t
   result += ip * Math.pow(256.0, power.toDouble).toLong\n    }\n    result\n}\n\nval bcLow
= sc.broadcast(ipToLong(info.getLowAddress))\nval bcHigh = sc.broadcast(ipToLong(info.getHighAddress))\nval
bcIpToLong = sc.broadcast(ipToLong)\n\nval results = sqlContext.sql(\ns\"\"\"SELECT\n    ip_dst_addr,\n
   COUNT(*) AS count\nFROM\n    yaf\nGROUP BY ip_dst_addr\nORDER BY ip_dst_addr\n\"\"\").flatMap
{\n  case Row(ip_dst_addr: String, count: Long) => {\n        val longDst = bcIpToLong.value(ip_dst_addr)\n
       if(bcLow.value <= longDst && longDst <= bcHigh.value) {\n\t\t    List(ip_dst_addr
+ \"\\t\" + count)\n        } else {\n            List.empty[String]\n        }\n  }\n  }.collect()\n
 \nprint(\"%table ip_dst_addr\\tcount\\n\" + results.mkString(\"\\n\"))\n","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:31+0000","dateStarted":"2017-05-09T14:20:11+0000","result":{"code":"SUCCESS","type":"TABLE","msg":"ip_dst_addr\tcount\n192.168.138.158\t151\n192.168.138.2\t2\n192.168.66.1\t69\n192.168.66.121\t86","comment":"","msgTable":[[{"key":"count","value":"192.168.138.158"},{"key":"count","value":"151"}],[{"value":"192.168.138.2"},{"value":"2"}],[{"value":"192.168.66.1"},{"value":"69"}],[{"value":"192.168.66.121"},{"value":"86"}]],"columnNames":[{"name":"ip_dst_addr","index":0,"aggr":"sum"},{"name":"count","index":1,"aggr":"sum"}],"rows":[["192.168.138.158","151"],["192.168.138.2","2"],["192.168.66.1","69"],["192.168.66.121","86"]]}},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1494339303735_1
 424887757","id":"20170509-141503_898772342","dateCreated":"2017-05-09T14:15:03+0000","status":"FINISHED","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:7557","dateUpdated":"2017-05-09T14:20:10+0000","dateFinished":"2017-05-09T14:20:31+0000","dateStarted":"2017-05-09T14:20:21+0000","result":{"code":"SUCCESS","type":"TEXT","msg":""}}],"name":"Metron
- Connection Volume Report","id":"2CER9F199","angularObjects":{"2CET9UGAN:shared_process":[],"2CHC4B4TT:shared_process":[],"2CF24S5PD:shared_process":[],"2CF6W9QPU:shared_process":[],"2CJJ4RS82:shared_process":[],"2CGYCAYBG:shared_process":[]},"config":{"looknfeel":"simple"},"info":{}}
\ No newline at end of file


Mime
View raw message