metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject [10/44] metron git commit: METRON-913 Create IP Report in Zeppelin closes apache/incubator-metron#561
Date Thu, 01 Jun 2017 21:41:29 GMT
METRON-913 Create IP Report in Zeppelin closes apache/incubator-metron#561


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8cf109da
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8cf109da
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8cf109da

Branch: refs/heads/Metron_0.4.0
Commit: 8cf109da180ee9f80984b0408afed799fa6c7c43
Parents: 082b218
Author: David Lyle <dlyle65535@gmail.com>
Authored: Tue May 9 14:07:36 2017 -0400
Committer: David Lyle <dlyle65535@gmail.com>
Committed: Tue May 9 14:07:36 2017 -0400

----------------------------------------------------------------------
 .../docker/rpm-docker/SPECS/metron.spec         |   9 +
 .../zeppelin/metron/metron-ip-report.json       | 439 +++++++++++++++++++
 2 files changed, 448 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/8cf109da/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 76129c3..da8f192 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -269,7 +269,11 @@ This package installs the Metron Indexing files
 %{metron_home}/config/zookeeper/indexing/asa.json
 %{metron_home}/config/zookeeper/indexing/error.json
 %{metron_home}/config/zeppelin/metron/metron-yaf-telemetry.json
+<<<<<<< HEAD
 %{metron_home}/config/zeppelin/metron/metron-connection-report.json
+=======
+%{metron_home}/config/zeppelin/metron/metron-ip-report.json
+>>>>>>> 1cace9ff29f31301d74fa6a7b2630d471452e985
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -399,10 +403,15 @@ This package installs the Metron Management UI %{metron_home}
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 %changelog
+<<<<<<< HEAD
 * Thu May 4 2017 Ryan Merriman <merrimanr@gmail.com> - 0.4.0
 - Added REST
 * Fri Apr 28 2017 Apache Metron <dev@metron.apache.org> - 0.4.0
 - Add Zeppelin Connection Report Dashboard
+=======
+* Tue May 2 2017 David Lyle <dlyle65535@gmail.com> - 0.4.0
+- Add Metron IP Report
+>>>>>>> 1cace9ff29f31301d74fa6a7b2630d471452e985
 * Thu Jan 19 2017 Justin Leet <justinjleet@gmail.com> - 0.3.1
 - Replace GeoIP files with new implementation
 * Thu Nov 03 2016 David Lyle <dlyle65535@gmail.com> - 0.2.1

http://git-wip-us.apache.org/repos/asf/metron/blob/8cf109da/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-ip-report.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-ip-report.json
b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-ip-report.json
new file mode 100644
index 0000000..cf226c9
--- /dev/null
+++ b/metron-platform/metron-indexing/src/main/config/zeppelin/metron/metron-ip-report.json
@@ -0,0 +1,439 @@
+{
+  "paragraphs": [
+    {
+      "text": "%md\n\n# Metron IP Report\n\nFor a given Source IP Address, produces the following
reports:\n\n* Most Frequent Connections (defaults to 24 hours)\n* Recent Connections (defaults
to 1 hour)\n* Top DNS Queries (defaults to 24 hours)\n* All Ports Used (defaults to 24 hours)\n*
HTTP User Agents (defaults to 24 hours)\n\nAll reports default to a 50 row limit.\n\n\nTo
execute the reports, select the Source IP Adress in the box below and click on 'Run All Paragraphs'
in the top tool bar.\n\nReports may be executed individually, but be sure to execute the 'Setup'
SQL Context and Tables first.",
+      "dateUpdated": "2017-05-02T15:40:11+0000",
+      "config": {
+        "colWidth": 12,
+        "editorMode": "ace/mode/scala",
+        "editorHide": true,
+        "graph": {
+          "mode": "table",
+          "height": 300,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "enabled": true
+      },
+      "settings": {
+        "params": {},
+        "forms": {}
+      },
+      "jobName": "paragraph_1493739611136_-165301953",
+      "id": "20170502-152938_356593846",
+      "result": {
+        "code": "SUCCESS",
+        "type": "HTML",
+        "msg": "<h1>Metron IP Report</h1>\n<p>For a given Source IP Address,
produces the following reports:</p>\n<ul>\n<li>Most Frequent Connections
(defaults to 24 hours)</li>\n<li>Recent Connections (defaults to 1 hour)</li>\n<li>Top
DNS Queries (defaults to 24 hours)</li>\n<li>All Ports Used (defaults to 24 hours)</li>\n<li>HTTP
User Agents (defaults to 24 hours)</li>\n</ul>\n<p>All reports default to
a 50 row limit.</p>\n<p>To execute the reports, select the Source IP Adress in
the box below and click on 'Run All Paragraphs' in the top tool bar.</p>\n<p>Reports
may be executed individually, but be sure to execute the 'Setup' SQL Context and Tables first.</p>\n"
+      },
+      "dateCreated": "2017-05-02T15:40:11+0000",
+      "status": "READY",
+      "errorMessage": "",
+      "progressUpdateIntervalMs": 500,
+      "$$hashKey": "object:19401"
+    },
+    {
+      "title": "Setup SQL Context and Tables",
+      "text": "%spark\n/**\n* date_sub_hours: Subtract an hours worth of seconds from a timestamp\n*/\ndef
date_sub_hours(t:Long, h:Int) : Long = {\n    t - h*3600\n}\n\n/**\n* source_ip_address: Provides
the same Source IP Address to all reports\n*/\nvar srcIp = z.input(\"Source IP Address\")\ndef
source_ip_address() : String = {\n    srcIp.toString\n}\n\n//Register udfs\nsqlc.udf.register(\"source_ip_address\",
source_ip_address _)\nsqlc.udf.register(\"date_sub_hours\", date_sub_hours _)\n\n//Load Yaf
Data \nval yaf = sqlc.read.json(\"hdfs:///apps/metron/indexing/indexed/yaf\")\nyaf.registerTempTable(\"yaf\")\n\n//Load
Bro Data\nval bro = sqlc.read.json(\"hdfs:///apps/metron/indexing/indexed/bro\")\nbro.registerTempTable(\"bro\")",
+      "dateUpdated": "2017-05-02T15:40:11+0000",
+      "config": {
+        "tableHide": false,
+        "colWidth": 12,
+        "editorMode": "ace/mode/scala",
+        "editorHide": true,
+        "title": true,
+        "graph": {
+          "mode": "table",
+          "height": 300,
+          "optionOpen": false,
+          "keys": [],
+          "values": [],
+          "groups": [],
+          "scatter": {}
+        },
+        "enabled": true
+      },
+      "settings": {
+        "params": {
+          "Source IP Address": ""
+        },
+        "forms": {
+          "Source IP Address": {
+            "name": "Source IP Address",
+            "displayName": "Source IP Address",
+            "type": "input",
+            "defaultValue": "",
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1493739611136_-165301953",
+      "id": "20170502-140420_1010901147",
+      "dateCreated": "2017-05-02T15:40:11+0000",
+      "status": "READY",
+      "errorMessage": "",
+      "progressUpdateIntervalMs": 500,
+      "$$hashKey": "object:19402"
+    },
+    {
+      "title": "Most Frequent Connections - YAF",
+      "text": "%spark.sql\nSELECT\n    ip_dst_addr,\n    COUNT(*) AS count\nFROM yaf\nWHERE\n
   ip_src_addr = source_ip_address() AND\n    timestamp BETWEEN date_sub_hours(unix_timestamp(),${Number
of Hours=24}) * 1000 AND unix_timestamp() * 1000\nGROUP BY ip_dst_addr\nORDER BY count DESC\nLIMIT
${Number of Rows=50}",
+      "dateUpdated": "2017-05-02T15:40:18+0000",
+      "config": {
+        "colWidth": 12,
+        "editorMode": "ace/mode/sql",
+        "title": true,
+        "graph": {
+          "mode": "multiBarChart",
+          "height": 300,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "ip_dst_addr",
+              "index": 0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "ip_dst_addr",
+              "index": 0,
+              "aggr": "sum"
+            },
+            "yAxis": {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          }
+        },
+        "enabled": true,
+        "editorHide": true
+      },
+      "settings": {
+        "params": {
+          "Number of Rows": "50",
+          "Number of Hours": "24",
+          "Source IP Address": "192.168.138.158"
+        },
+        "forms": {
+          "Number of Rows": {
+            "name": "Number of Rows",
+            "defaultValue": "50",
+            "hidden": false
+          },
+          "Number of Hours": {
+            "name": "Number of Hours",
+            "defaultValue": "24",
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1493739611136_-165301953",
+      "id": "20170502-140443_302943667",
+      "dateCreated": "2017-05-02T15:40:11+0000",
+      "status": "READY",
+      "errorMessage": "",
+      "progressUpdateIntervalMs": 500,
+      "$$hashKey": "object:19403"
+    },
+    {
+      "title": "Recent Connections - YAF",
+      "text": "%spark.sql\nSELECT\n    ip_dst_addr,\n    COUNT(*) as count\nFROM yaf\nWHERE\n
   ip_src_addr = source_ip_address() AND\n    timestamp BETWEEN date_sub_hours(unix_timestamp(),${Number
of Hours=1}) * 1000 AND unix_timestamp() * 1000\nGROUP BY ip_dst_addr\nORDER BY count DESC\nLIMIT
${Number of Rows=50}",
+      "dateUpdated": "2017-05-02T15:40:21+0000",
+      "config": {
+        "colWidth": 12,
+        "editorMode": "ace/mode/sql",
+        "title": true,
+        "graph": {
+          "mode": "multiBarChart",
+          "height": 300,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "ip_dst_addr",
+              "index": 0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "ip_dst_addr",
+              "index": 0,
+              "aggr": "sum"
+            },
+            "yAxis": {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          }
+        },
+        "enabled": true,
+        "editorHide": true
+      },
+      "settings": {
+        "params": {
+          "Number of Rows": "50",
+          "Number of Hours": "1",
+          "Source IP Address": "192.168.138.158"
+        },
+        "forms": {
+          "Number of Rows": {
+            "name": "Number of Rows",
+            "defaultValue": "50",
+            "hidden": false
+          },
+          "Number of Hours": {
+            "name": "Number of Hours",
+            "defaultValue": "1",
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1493739611136_-165301953",
+      "id": "20170502-140621_1919836666",
+      "dateCreated": "2017-05-02T15:40:11+0000",
+      "status": "READY",
+      "errorMessage": "",
+      "progressUpdateIntervalMs": 500,
+      "$$hashKey": "object:19404"
+    },
+    {
+      "title": "Top DNS Queries - BRO",
+      "text": "%spark.sql\nSELECT \n    query as domain_name,\n    COUNT(*) as count\nFROM
bro \nWHERE\n    ip_src_addr = source_ip_address()\n    AND\n    timestamp BETWEEN date_sub_hours(unix_timestamp(),${Number
of Hours=24}) * 1000 AND unix_timestamp() * 1000 \n    AND\n    protocol = 'dns' \n    AND
\n    query is not null\nGROUP BY query\nORDER BY count DESC\nLIMIT ${Number of Rows=50}\n
   ",
+      "dateUpdated": "2017-05-02T15:40:23+0000",
+      "config": {
+        "colWidth": 12,
+        "editorMode": "ace/mode/sql",
+        "title": true,
+        "graph": {
+          "mode": "table",
+          "height": 300,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "domain_name",
+              "index": 0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "domain_name",
+              "index": 0,
+              "aggr": "sum"
+            },
+            "yAxis": {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          }
+        },
+        "enabled": true,
+        "editorHide": true
+      },
+      "settings": {
+        "params": {
+          "Number of Rows": "50",
+          "Number of Hours": "24",
+          "Source IP Address": "192.168.138.158"
+        },
+        "forms": {
+          "Number of Rows": {
+            "name": "Number of Rows",
+            "defaultValue": "50",
+            "hidden": false
+          },
+          "Number of Hours": {
+            "name": "Number of Hours",
+            "defaultValue": "24",
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1493739611136_-165301953",
+      "id": "20170502-141019_193020479",
+      "dateCreated": "2017-05-02T15:40:11+0000",
+      "status": "READY",
+      "errorMessage": "",
+      "progressUpdateIntervalMs": 500,
+      "$$hashKey": "object:19405"
+    },
+    {
+      "title": "All Ports Used - YAF",
+      "text": "%spark.sql\nSELECT\n    ip_dst_port,\n    COUNT(*) as count\nFROM yaf \nWHERE\n
   ip_src_addr = source_ip_address()\n    AND\n    timestamp BETWEEN date_sub_hours(unix_timestamp(),${Number
of Hours=24}) * 1000 AND unix_timestamp() * 1000 \nGROUP BY ip_dst_port\nORDER BY count DESC\nLIMIT
${Number of Rows=50}",
+      "dateUpdated": "2017-05-02T15:40:25+0000",
+      "config": {
+        "colWidth": 12,
+        "editorMode": "ace/mode/sql",
+        "title": true,
+        "graph": {
+          "mode": "table",
+          "height": 300,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "ip_dst_port",
+              "index": 0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "ip_dst_port",
+              "index": 0,
+              "aggr": "sum"
+            },
+            "yAxis": {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          }
+        },
+        "enabled": true,
+        "editorHide": true
+      },
+      "settings": {
+        "params": {
+          "Number of Rows": "50",
+          "Number of Hours": "24"
+        },
+        "forms": {
+          "Number of Rows": {
+            "name": "Number of Rows",
+            "defaultValue": "50",
+            "hidden": false
+          },
+          "Number of Hours": {
+            "name": "Number of Hours",
+            "defaultValue": "24",
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1493739611136_-165301953",
+      "id": "20170502-141412_380718762",
+      "dateCreated": "2017-05-02T15:40:11+0000",
+      "status": "READY",
+      "errorMessage": "",
+      "progressUpdateIntervalMs": 500,
+      "$$hashKey": "object:19406"
+    },
+    {
+      "title": "HTTP User Agents - BRO",
+      "text": "%spark.sql\nSELECT\n user_agent,\n COUNT(*) as count\nFROM bro\nWHERE \n 
  ip_src_addr = source_ip_address()\n    AND\n    timestamp BETWEEN date_sub_hours(unix_timestamp(),${Number
of Hours=24}) * 1000 AND unix_timestamp() * 1000 \n    AND\n    protocol = 'http' \nGROUP
BY user_agent\nORDER BY count DESC\nLIMIT ${Number of Rows=50}",
+      "dateUpdated": "2017-05-02T15:40:27+0000",
+      "config": {
+        "colWidth": 12,
+        "editorMode": "ace/mode/sql",
+        "title": true,
+        "graph": {
+          "mode": "table",
+          "height": 300,
+          "optionOpen": false,
+          "keys": [
+            {
+              "name": "user_agent",
+              "index": 0,
+              "aggr": "sum"
+            }
+          ],
+          "values": [
+            {
+              "name": "count",
+              "index": 1,
+              "aggr": "sum"
+            }
+          ],
+          "groups": [],
+          "scatter": {
+            "xAxis": {
+              "name": "user_agent",
+              "index": 0,
+              "aggr": "sum"
+            }
+          }
+        },
+        "enabled": true,
+        "editorHide": true
+      },
+      "settings": {
+        "params": {
+          "Number of Hours": "24",
+          "Number of Rows": "50"
+        },
+        "forms": {
+          "Number of Rows": {
+            "name": "Number of Rows",
+            "defaultValue": "50",
+            "hidden": false
+          },
+          "Number of Hours": {
+            "name": "Number of Hours",
+            "defaultValue": "24",
+            "hidden": false
+          }
+        }
+      },
+      "jobName": "paragraph_1493739611136_-165301953",
+      "id": "20170502-143830_1892618095",
+      "dateCreated": "2017-05-02T15:40:11+0000",
+      "status": "READY",
+      "errorMessage": "",
+      "progressUpdateIntervalMs": 500,
+      "$$hashKey": "object:19407"
+    }
+  ],
+  "name": "Metron IP Report",
+  "id": "2CG7BF9JT",
+  "angularObjects": {
+    "2CEKEU5T4:shared_process": [],
+    "2CGU6FFBF:shared_process": [],
+    "2CF6U7R6S:shared_process": [],
+    "2CG2SK9DG:shared_process": [],
+    "2CFDEBF6X:shared_process": [],
+    "2CFJGFKFD:shared_process": []
+  },
+  "config": {
+    "looknfeel": "default"
+  },
+  "info": {}
+}
\ No newline at end of file


Mime
View raw message