metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jsir...@apache.org
Subject [1/3] incubator-metron git commit: METRON-72 Create unified enrichment topology (merrimanr via jsirota) closes apache/incubator-metron#50
Date Sat, 19 Mar 2016 21:12:39 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master d28083701 -> 68aab6e9a


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/PcapParserIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/PcapParserIntegrationTest.java
b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/PcapParserIntegrationTest.java
new file mode 100644
index 0000000..284e3c0
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/test/java/org/apache/metron/integration/PcapParserIntegrationTest.java
@@ -0,0 +1,218 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.integration;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.HTableInterface;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.Reader;
+import org.apache.metron.hbase.TableProvider;
+import org.apache.metron.integration.util.UnitTestHelper;
+import org.apache.metron.integration.util.integration.ComponentRunner;
+import org.apache.metron.integration.util.integration.Processor;
+import org.apache.metron.integration.util.integration.ReadinessState;
+import org.apache.metron.integration.util.integration.components.FluxTopologyComponent;
+import org.apache.metron.integration.util.integration.components.KafkaWithZKComponent;
+import org.apache.metron.integration.util.mock.MockHTable;
+import org.apache.metron.parsing.parsers.PcapParser;
+import org.apache.metron.pcap.PcapUtils;
+import org.json.simple.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.*;
+
+public class PcapParserIntegrationTest {
+
+  private static String BASE_DIR = "pcap";
+  private static String DATA_DIR = BASE_DIR + "/data_dir";
+  private static String QUERY_DIR = BASE_DIR + "/query";
+  private String topologiesDir = "src/main/resources/Metron_Configs/topologies";
+  private String targetDir = "target";
+
+  public static class Provider implements TableProvider, Serializable {
+    MockHTable.Provider  provider = new MockHTable.Provider();
+    @Override
+    public HTableInterface getTable(Configuration config, String tableName) throws IOException
{
+      return provider.getTable(config, tableName);
+    }
+  }
+
+  private File getOutDir(String targetDir) {
+    File outDir = new File(new File(targetDir), DATA_DIR);
+    if (!outDir.exists()) {
+      outDir.mkdirs();
+    }
+
+    return outDir;
+  }
+
+  private File getQueryDir(String targetDir) {
+    File outDir = new File(new File(targetDir), QUERY_DIR);
+    if (!outDir.exists()) {
+      outDir.mkdirs();
+    }
+    return outDir;
+  }
+  private static void clearOutDir(File outDir) {
+    for(File f : outDir.listFiles()) {
+      f.delete();
+    }
+  }
+
+  private static Map<String, byte[]> readPcaps(Path pcapFile) throws IOException {
+    SequenceFile.Reader reader = new SequenceFile.Reader(new Configuration(),
+            Reader.file(pcapFile)
+            );
+    Map<String, byte[]> ret = new HashMap<>();
+    IntWritable key = new IntWritable();
+    BytesWritable value = new BytesWritable();
+    PcapParser parser = new PcapParser();
+    parser.init();
+    while(reader.next(key, value)) {
+      int keyInt = key.get();
+      byte[] valueBytes = value.copyBytes();
+      JSONObject message = parser.parse(valueBytes).get(0);
+      if (parser.validate(message)) {
+        ret.put(PcapUtils.getSessionKey(message), valueBytes);
+      }
+    }
+    return ret;
+  }
+
+  @Test
+  public void testTopology() throws Exception {
+    if (!new File(topologiesDir).exists()) {
+      topologiesDir = UnitTestHelper.findDir("topologies");
+    }
+    targetDir = UnitTestHelper.findDir("target");
+    final String kafkaTopic = "pcap";
+    final String tableName = "pcap";
+    final String columnFamily = "t";
+    final String columnIdentifier = "value";
+    final File outDir = getOutDir(targetDir);
+    final File queryDir = getQueryDir(targetDir);
+    clearOutDir(outDir);
+    clearOutDir(queryDir);
+
+    File baseDir = new File(new File(targetDir), BASE_DIR);
+    Assert.assertNotNull(topologiesDir);
+    Assert.assertNotNull(targetDir);
+    Path pcapFile = new Path(topologiesDir + "/../../SampleInput/PCAPExampleOutput");
+    final Map<String, byte[]> pcapEntries = readPcaps(pcapFile);
+    Assert.assertTrue(Iterables.size(pcapEntries.keySet()) > 0);
+    final Properties topologyProperties = new Properties() {{
+      setProperty("hbase.provider.impl","" + Provider.class.getName());
+      setProperty("spout.kafka.topic.pcap", kafkaTopic);
+      setProperty("bolt.hbase.table.name",tableName);
+      setProperty("bolt.hbase.table.fields", columnFamily + ":" + columnIdentifier);
+    }};
+    final KafkaWithZKComponent kafkaComponent = new KafkaWithZKComponent().withTopics(new
ArrayList<KafkaWithZKComponent.Topic>() {{
+      add(new KafkaWithZKComponent.Topic(kafkaTopic, 1));
+    }})
+            .withPostStartCallback(new Function<KafkaWithZKComponent, Void>() {
+                                     @Nullable
+                                     @Override
+                                     public Void apply(@Nullable KafkaWithZKComponent kafkaWithZKComponent)
{
+
+                                       topologyProperties.setProperty("kafka.zk", kafkaWithZKComponent.getZookeeperConnect());
+                                       return null;
+                                     }
+                                   }
+            );
+    //.withExistingZookeeper("localhost:2000");
+
+    FluxTopologyComponent fluxComponent = new FluxTopologyComponent.Builder()
+            .withTopologyLocation(new File(topologiesDir + "/pcap/test.yaml"))
+            .withTopologyName("pcap")
+            .withTopologyProperties(topologyProperties)
+            .build();
+
+    final MockHTable pcapTable = (MockHTable)MockHTable.Provider.addToCache(tableName, columnFamily);
+
+    UnitTestHelper.verboseLogging();
+    ComponentRunner runner = new ComponentRunner.Builder()
+            .withComponent("kafka", kafkaComponent)
+            .withComponent("storm", fluxComponent)
+            .withMaxTimeMS(60000)
+            .withMillisecondsBetweenAttempts(6000)
+            .withNumRetries(10)
+            .build();
+    try {
+      runner.start();
+      System.out.println("Components started...");
+      fluxComponent.submitTopology();
+      kafkaComponent.writeMessages(kafkaTopic, pcapEntries.values());
+      System.out.println("Sent pcap data: " + pcapEntries.size());
+      List<byte[]> messages = kafkaComponent.readMessages(kafkaTopic);
+      Assert.assertEquals(pcapEntries.size(), messages.size());
+      System.out.println("Wrote " + pcapEntries.size() + " to kafka");
+      runner.process(new Processor<Void>() {
+        @Override
+        public ReadinessState process(ComponentRunner runner) {
+          int hbaseCount = 0;
+          try {
+            ResultScanner resultScanner = pcapTable.getScanner(columnFamily.getBytes(), columnIdentifier.getBytes());
+            while(resultScanner.next() != null) hbaseCount++;
+          } catch (IOException e) {
+            e.printStackTrace();
+          }
+          if (hbaseCount == pcapEntries.size()) {
+            return ReadinessState.READY;
+          } else {
+            return ReadinessState.NOT_READY;
+          }
+        }
+
+        @Override
+        public Void getResult() {
+          return null;
+        }
+      });
+      ResultScanner resultScanner = pcapTable.getScanner(columnFamily.getBytes(), columnIdentifier.getBytes());
+      Result result;
+      int rowCount = 0;
+      while((result = resultScanner.next()) != null) {
+        String rowKey = new String(result.getRow());
+        byte[] hbaseValue = result.getValue(columnFamily.getBytes(), columnIdentifier.getBytes());
+        byte[] originalValue = pcapEntries.get(rowKey);
+        Assert.assertNotNull("Could not find pcap with key " + rowKey + " in sample data",
originalValue);
+        Assert.assertArrayEquals("Raw values are different for key " + rowKey, originalValue,
hbaseValue);
+        rowCount++;
+      }
+      Assert.assertEquals(pcapEntries.size(), rowCount);
+      System.out.println("Ended");
+    }
+    finally {
+      runner.stop();
+      clearOutDir(outDir);
+      clearOutDir(queryDir);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/.gitignore
----------------------------------------------------------------------
diff --git a/metron-ui/.gitignore b/metron-ui/.gitignore
index c94c2a1..a6dfcf4 100644
--- a/metron-ui/.gitignore
+++ b/metron-ui/.gitignore
@@ -9,9 +9,6 @@ pids
 # Pcap files
 *.pcap
 
-# Config overrides
-config.json
-
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/config.json
----------------------------------------------------------------------
diff --git a/metron-ui/config.json b/metron-ui/config.json
new file mode 100644
index 0000000..dde17b2
--- /dev/null
+++ b/metron-ui/config.json
@@ -0,0 +1,6 @@
+{
+  "auth":false,
+  "secret":"secret",
+  "elasticsearch": { "url": "http://host:port" },
+  "pcap": { "url": "http://host:port/pcapGetter","mock": false }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/metron-ui.js
----------------------------------------------------------------------
diff --git a/metron-ui/lib/metron-ui.js b/metron-ui/lib/metron-ui.js
index c11d7a8..14aeac3 100644
--- a/metron-ui/lib/metron-ui.js
+++ b/metron-ui/lib/metron-ui.js
@@ -23,6 +23,7 @@ var path = require('path');
 var express = require('express');
 
 var connect = require('connect');
+var serveStatic = require('serve-static');
 var flash = require('connect-flash');
 
 var cookieParser = require('cookie-parser');
@@ -37,7 +38,7 @@ var login = require('./modules/login');
 var pcap = require('./modules/pcap');
 
 var app = express();
-var config = require('./config');
+var config = require('../config.json');
 
 
 try {
@@ -51,7 +52,7 @@ app.set('view engine', 'jade');
 app.set('views', path.join(__dirname, 'views/'));
 
 // Cookie middleware
-app.use(connect.logger('dev'));
+//app.use(connect.logger('dev'));
 app.use(flash());
 app.use(cookieParser());
 app.use(cookieSession({
@@ -110,7 +111,7 @@ pcap(app, config);
 login(app, config);
 
 // Serve static assets
-app.use(connect.static(path.join(__dirname, 'public')));
+app.use(serveStatic(path.join(__dirname, 'public')));
 
 
 // Start server

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/modules/es-proxy.js
----------------------------------------------------------------------
diff --git a/metron-ui/lib/modules/es-proxy.js b/metron-ui/lib/modules/es-proxy.js
index b9b5a63..b805fea 100644
--- a/metron-ui/lib/modules/es-proxy.js
+++ b/metron-ui/lib/modules/es-proxy.js
@@ -25,7 +25,7 @@ exports = module.exports = function(config) {
   });
 
   return function(req, res, next) {
-    if (!req.user) {
+    if (config.auth && !req.user) {
       res.send(403, 'Forbidden!');
       return;
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/modules/login.js
----------------------------------------------------------------------
diff --git a/metron-ui/lib/modules/login.js b/metron-ui/lib/modules/login.js
index 7fa2c7b..2ad669a 100644
--- a/metron-ui/lib/modules/login.js
+++ b/metron-ui/lib/modules/login.js
@@ -20,7 +20,7 @@ exports = module.exports = function(app, config) {
   var passport = require('passport');
 
   app.get('/', function (req, res, next) {
-    if (!req.user) {
+    if (config.auth && !req.user ) {
       res.redirect('/login');
       return;
     }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/modules/pcap.js
----------------------------------------------------------------------
diff --git a/metron-ui/lib/modules/pcap.js b/metron-ui/lib/modules/pcap.js
index 5b0a7fc..eaad29e 100644
--- a/metron-ui/lib/modules/pcap.js
+++ b/metron-ui/lib/modules/pcap.js
@@ -62,7 +62,7 @@ exports = module.exports = function(app, config) {
   }
 
   app.get('/pcap/:command', function(req, res) {
-    if (!req.user || !req.user.permissions.pcap) {
+    if (config.auth && (!req.user || !req.user.permissions.pcap)) {
       res.send(403, 'Forbidden!');
       return;
     }
@@ -72,6 +72,16 @@ exports = module.exports = function(app, config) {
     pcapUrl += '?' + querystring.stringify(req.query);
 
     var curl = spawn('curl', ['-s', pcapUrl]);
+
+    if (true) {
+      res.set('Content-Type', 'application/cap');
+      var fileName = req.query.srcIp + "-" + req.query.dstIp + '-' + req.query.srcPort +
'-' + req.query.dstPort + '-' + req.query.protocol + '-' + req.query.includeReverseTraffic;
+      fileName = fileName.replace(/\./g, '_');
+      res.set('Content-Disposition', 'attachment; filename="' + fileName + '.pcap"');
+      curl.stdout.pipe(res);
+      return;
+    }
+
     var tshark = spawn('tshark', ['-i', '-', '-T', 'pdml']);
     var xml = new XmlStream(tshark.stdout);
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/public/app/dashboards/default.json
----------------------------------------------------------------------
diff --git a/metron-ui/lib/public/app/dashboards/default.json b/metron-ui/lib/public/app/dashboards/default.json
index 9cd224e..4d3dd24 100644
--- a/metron-ui/lib/public/app/dashboards/default.json
+++ b/metron-ui/lib/public/app/dashboards/default.json
@@ -1,70 +1,70 @@
 {
-  "title": "New Dashboard",
+  "title": "Metron",
   "services": {
     "query": {
       "list": {
         "0": {
-          "query": "alerts.triggered.priority:1",
-          "alias": "",
-          "color": "#BF1B00",
           "id": 0,
+          "color": "#7EB26D",
+          "alias": "alerts",
           "pin": true,
           "type": "lucene",
-          "enable": true
+          "enable": true,
+          "query": "is_alert=true"
         },
         "1": {
           "id": 1,
-          "color": "#EAB839",
-          "alias": "",
+          "color": "#6ED0E0",
+          "alias": "Yaf",
           "pin": true,
           "type": "lucene",
           "enable": true,
-          "query": "alerts.triggered.priority:2"
+          "query": "_type:yaf_doc"
         },
         "2": {
           "id": 2,
-          "color": "#6ED0E0",
-          "alias": "",
+          "color": "#BA43A9",
+          "alias": "All Events",
           "pin": true,
           "type": "lucene",
           "enable": true,
-          "query": "alerts.triggered.priority:3"
+          "query": "_type:*_doc"
         },
         "3": {
           "id": 3,
-          "color": "#E5AC0E",
-          "alias": "Warning",
-          "pin": false,
+          "color": "#BA43A9",
+          "alias": "All Alerts",
+          "pin": true,
           "type": "lucene",
           "enable": true,
-          "query": ""
+          "query": "_type:*_alert"
         },
         "4": {
           "id": 4,
-          "color": "#E24D42",
-          "alias": "1. Alert",
+          "color": "#1F78C1",
+          "alias": "Bro Events",
           "pin": true,
           "type": "lucene",
           "enable": true,
-          "query": "alerts.triggered.type:alert"
+          "query": "_type:bro_doc"
         },
         "5": {
           "id": 5,
-          "color": "#F2C96D",
-          "alias": "3. Error",
+          "color": "#EF843C",
+          "alias": "Bro Alerts",
           "pin": true,
           "type": "lucene",
           "enable": true,
-          "query": "alerts.triggered.type:error"
+          "query": "_type:bro_alert"
         },
         "6": {
           "id": 6,
-          "color": "#F9934E",
-          "alias": "2. Warning",
+          "color": "#BA43A9",
+          "alias": "Snort Events",
           "pin": true,
           "type": "lucene",
           "enable": true,
-          "query": "alerts.triggered.type:warning"
+          "query": "_type:snort_doc"
         }
       },
       "ids": [
@@ -73,145 +73,192 @@
         2,
         3,
         4,
-        6,
-        5
+        5,
+        6
       ]
     },
     "filter": {
       "list": {
         "0": {
-          "type": "field",
-          "field": "_type",
-          "query": "\"pcap\"",
-          "mandate": "mustNot",
-          "active": true,
-          "alias": "",
-          "id": 0
-        },
-        "1": {
           "type": "time",
-          "field": "message.timestamp",
-          "from": "now-12h",
+          "field": "timestamp",
+          "from": "now-24h",
           "to": "now",
           "mandate": "must",
           "active": true,
           "alias": "",
-          "id": 1
+          "id": 0
         }
       },
       "ids": [
-        0,
-        1
+        0
       ]
     }
   },
   "rows": [
     {
-      "title": "Fixed Overview",
-      "height": "250px",
+      "title": "Histogram",
+      "height": "150px",
       "editable": true,
       "collapse": false,
       "collapsable": true,
       "panels": [
         {
-          "error": false,
-          "span": 3,
+          "span": 6,
           "editable": true,
-          "type": "terms",
+          "type": "histogram",
           "loadingEditor": false,
-          "field": "alerts.triggered.type",
-          "exclude": [],
-          "missing": false,
-          "other": false,
-          "size": 10,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
+          "mode": "count",
+          "time_field": "timestamp",
+          "value_field": null,
+          "x-axis": true,
+          "y-axis": true,
+          "scale": 1,
+          "y_format": "short",
+          "grid": {
+            "max": null,
+            "min": 0
           },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "bar",
-          "counter_pos": "above",
-          "spyable": true,
           "queries": {
-            "mode": "all",
+            "mode": "selected",
             "ids": [
-              0,
-              1,
-              2,
-              3,
               4,
-              5,
-              6
+              5
             ]
           },
           "locked": false,
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Alert Type"
+          "annotate": {
+            "enable": false,
+            "query": "*",
+            "size": 20,
+            "field": "_type",
+            "sort": [
+              "_score",
+              "desc"
+            ]
+          },
+          "auto_int": true,
+          "resolution": 100,
+          "interval": "10m",
+          "intervals": [
+            "auto",
+            "1s",
+            "1m",
+            "5m",
+            "10m",
+            "30m",
+            "1h",
+            "3h",
+            "12h",
+            "1d",
+            "1w",
+            "1y"
+          ],
+          "lines": false,
+          "fill": 0,
+          "linewidth": 3,
+          "points": false,
+          "pointradius": 5,
+          "bars": true,
+          "stack": false,
+          "spyable": true,
+          "zoomlinks": true,
+          "options": true,
+          "legend": true,
+          "show_query": true,
+          "interactive": true,
+          "legend_counts": true,
+          "timezone": "browser",
+          "percentage": false,
+          "zerofill": true,
+          "derivative": false,
+          "tooltip": {
+            "value_type": "individual",
+            "query_as_alias": true
+          },
+          "title": "Bro Data"
         },
         {
-          "error": false,
-          "span": 4,
+          "span": 6,
           "editable": true,
-          "type": "terms",
+          "type": "histogram",
           "loadingEditor": false,
-          "field": "_type",
-          "exclude": [
-            "pcap"
-          ],
-          "missing": false,
-          "other": false,
-          "size": 10,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
+          "mode": "count",
+          "time_field": "timestamp",
+          "value_field": null,
+          "x-axis": true,
+          "y-axis": true,
+          "scale": 1,
+          "y_format": "none",
+          "grid": {
+            "max": null,
+            "min": 0
           },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "pie",
-          "counter_pos": "above",
-          "spyable": true,
           "queries": {
-            "mode": "all",
+            "mode": "selected",
             "ids": [
-              0,
-              1,
-              2,
-              3,
-              4,
-              5,
-              6
+              1
             ]
           },
           "locked": false,
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Alert Source"
-        }
-      ],
-      "notice": false
-    },
-    {
-      "title": "",
-      "height": "150px",
-      "editable": true,
-      "collapse": false,
-      "collapsable": true,
-      "panels": [
+          "annotate": {
+            "enable": false,
+            "query": "*",
+            "size": 20,
+            "field": "_type",
+            "sort": [
+              "_score",
+              "desc"
+            ]
+          },
+          "auto_int": true,
+          "resolution": 100,
+          "interval": "10m",
+          "intervals": [
+            "auto",
+            "1s",
+            "1m",
+            "5m",
+            "10m",
+            "30m",
+            "1h",
+            "3h",
+            "12h",
+            "1d",
+            "1w",
+            "1y"
+          ],
+          "lines": false,
+          "fill": 0,
+          "linewidth": 3,
+          "points": false,
+          "pointradius": 5,
+          "bars": true,
+          "stack": true,
+          "spyable": true,
+          "zoomlinks": true,
+          "options": true,
+          "legend": true,
+          "show_query": true,
+          "interactive": true,
+          "legend_counts": true,
+          "timezone": "browser",
+          "percentage": false,
+          "zerofill": true,
+          "derivative": false,
+          "tooltip": {
+            "value_type": "cumulative",
+            "query_as_alias": true
+          },
+          "title": "Yaf Data"
+        },
         {
-          "span": 8,
+          "span": 12,
           "editable": true,
           "type": "histogram",
           "loadingEditor": false,
           "mode": "count",
-          "time_field": "message.timestamp",
+          "time_field": "timestamp",
           "value_field": null,
           "x-axis": true,
           "y-axis": true,
@@ -224,8 +271,6 @@
           "queries": {
             "mode": "selected",
             "ids": [
-              4,
-              5,
               6
             ]
           },
@@ -242,7 +287,7 @@
           },
           "auto_int": true,
           "resolution": 100,
-          "interval": "5m",
+          "interval": "10m",
           "intervals": [
             "auto",
             "1s",
@@ -279,56 +324,71 @@
             "value_type": "cumulative",
             "query_as_alias": true
           },
-          "title": "Alert History Timeline"
-        },
+          "title": "Snort Data"
+        }
+      ],
+      "notice": false
+    },
+    {
+      "title": "Alerts",
+      "height": "150px",
+      "editable": true,
+      "collapse": false,
+      "collapsable": true,
+      "panels": [
         {
           "error": false,
-          "span": 4,
+          "span": 12,
           "editable": true,
-          "type": "terms",
+          "type": "table",
           "loadingEditor": false,
-          "field": "message.protocol",
-          "exclude": [],
-          "missing": false,
-          "other": false,
           "size": 10,
-          "order": "count",
-          "style": {
-            "font-size": "10pt"
-          },
-          "donut": false,
-          "tilt": false,
-          "labels": true,
-          "arrangement": "horizontal",
-          "chart": "table",
-          "counter_pos": "above",
+          "pages": 100,
+          "offset": 0,
+          "sort": [
+            "timestamp",
+            "desc"
+          ],
+          "overflow": "min-height",
+          "fields": [
+            "_type",
+            "msg",
+            "ip_src_addr",
+            "ip_src_port",
+            "ip_dst_addr",
+            "ip_dst_port"
+          ],
+          "highlight": [],
+          "sortable": true,
+          "header": true,
+          "paging": true,
+          "field_list": false,
+          "all_fields": false,
+          "trimFactor": 400,
+          "localTime": true,
+          "timeField": "timestamp",
           "spyable": true,
           "queries": {
-            "mode": "all",
+            "mode": "selected",
             "ids": [
-              0,
-              1,
-              2,
-              3,
-              4,
-              5,
-              6
+              0
             ]
           },
           "locked": false,
-          "tmode": "terms",
-          "tstat": "total",
-          "valuefield": "",
-          "title": "Protocol"
+          "style": {
+            "font-size": "9pt"
+          },
+          "normTimes": true,
+          "title": "Alerts"
         }
       ],
       "notice": false
     },
     {
-      "title": "",
+      "title": "Events",
       "height": "150px",
       "editable": true,
-      "collapse": true,
+      "collapse": false,
       "collapsable": true,
       "panels": [
         {
@@ -337,36 +397,38 @@
           "editable": true,
           "type": "table",
           "loadingEditor": false,
-          "size": 100,
-          "pages": 5,
+          "size": 10,
+          "pages": 100,
           "offset": 0,
           "sort": [
-            "alerts.triggered.priority",
+            "timestamp",
             "desc"
           ],
           "overflow": "min-height",
           "fields": [
             "_type",
-            "_index",
-            "alerts.triggered.priority"
+            "timestamp",
+            "ip_src_addr",
+            "ip_src_port",
+            "ip_dst_addr",
+            "ip_dst_port",
+            "protocol",
+            "original_string"
           ],
           "highlight": [],
           "sortable": true,
           "header": true,
           "paging": true,
-          "field_list": true,
+          "field_list": false,
           "all_fields": false,
-          "trimFactor": 300,
-          "localTime": false,
-          "timeField": "@timestamp",
+          "trimFactor": 400,
+          "localTime": true,
+          "timeField": "timestamp",
           "spyable": true,
           "queries": {
-            "mode": "all",
+            "mode": "selected",
             "ids": [
-              0,
-              1,
-              2,
-              3
+              4
             ]
           },
           "locked": false,
@@ -374,13 +436,13 @@
             "font-size": "9pt"
           },
           "normTimes": true,
-          "title": "all"
+          "title": "Bro"
         }
       ],
       "notice": false
     },
     {
-      "title": "",
+      "title": "PCAP Data",
       "height": "150px",
       "editable": true,
       "collapse": false,
@@ -392,8 +454,8 @@
           "editable": true,
           "type": "table",
           "loadingEditor": false,
-          "size": 10,
-          "pages": 5000,
+          "size": 25,
+          "pages": 5,
           "offset": 0,
           "sort": [
             "_score",
@@ -401,45 +463,41 @@
           ],
           "overflow": "min-height",
           "fields": [
-            "alerts.triggered.type",
-            "alerts.triggered.priority",
-            "alerts.triggered.title",
-            "alerts.triggered.body",
-            "_type",
-            "message.ip_dst_port",
-            "message.ip_dst_addr",
-            "message.timestamp",
-            "message.protocol",
-            "message.ip_src_addr"
+            "ip_src_addr",
+            "ip_src_port",
+            "ip_dst_addr",
+            "ip_dst_port",
+            "original_string"
           ],
           "highlight": [],
           "sortable": true,
           "header": true,
           "paging": true,
-          "field_list": true,
+          "field_list": false,
           "all_fields": false,
           "trimFactor": 300,
           "localTime": false,
           "timeField": "@timestamp",
           "spyable": true,
           "queries": {
-            "mode": "all",
+            "mode": "selected",
             "ids": [
-              0,
-              1,
-              2,
-              3,
-              4,
-              5,
-              6
+              1
             ]
           },
-          "locked": true,
           "style": {
             "font-size": "9pt"
           },
           "normTimes": true,
-          "title": "Top Alerts"
+          "title": "Yaf"
+        },
+        {
+          "error": false,
+          "span": 12,
+          "editable": true,
+          "type": "pcap",
+          "loadingEditor": false,
+          "title": "PCAP Data"
         }
       ],
       "notice": false
@@ -450,7 +508,7 @@
   "index": {
     "interval": "none",
     "pattern": "[logstash-]YYYY.MM.DD",
-    "default": "_all",
+    "default": "*alert*,*_index*",
     "warm_fields": false
   },
   "style": "dark",
@@ -465,19 +523,22 @@
       "query": "*",
       "pinned": true,
       "history": [
-        "alerts.triggered.type:warning",
-        "alerts.triggered.type:alert",
-        "",
-        "alerts.triggered.priority:3",
-        "alerts.triggered.priority:2",
-        "alerts.triggered.priority:1",
+        "_type:snort_doc",
+        "_type:bro_alert",
+        "_type:bro_doc",
+        "_type:*_alert",
+        "_type:*_doc",
+        "_type:yaf_doc",
+        "is_alert=true",
+        "_type:sourcefire_doc",
+        "_type:sourcefire_alert",
         "*"
       ],
       "remember": 10
     },
     {
       "type": "filtering",
-      "collapse": false,
+      "collapse": true,
       "notice": true,
       "enable": true
     }
@@ -512,9 +573,9 @@
         "2h",
         "1d"
       ],
-      "timefield": "message.timestamp",
+      "timefield": "timestamp",
       "now": true,
-      "filter_id": 1
+      "filter_id": 0
     }
   ],
   "loader": {

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/public/app/panels/pcap/module.html
----------------------------------------------------------------------
diff --git a/metron-ui/lib/public/app/panels/pcap/module.html b/metron-ui/lib/public/app/panels/pcap/module.html
index 893f871..17c57e0 100755
--- a/metron-ui/lib/public/app/panels/pcap/module.html
+++ b/metron-ui/lib/public/app/panels/pcap/module.html
@@ -100,108 +100,8 @@
 
 <div class="table-container">
     <div class="pcap-editor" pcap-settings></div>
-    <div class="pcap-editor">
-      <h6>PCAP Search Results</h6>
-      <table class="table table-bordered table-condensed table-details table-striped"
ng-style="panel.style">
-      <thead>
-        <tr>
-          <td>PCAP ID</td>
-          <td>Source Port</td>
-          <td>Destination Port</td>
-          <td>Source IP</td>
-          <td>Destination IP</td>
-          <td>Protocol</td>
-        <tr>
-      </thead>
-
-      <tbody bindonce ng-repeat="doc in results.hits.hits">
-          <tr ng-click="get_pcap(doc._source.message.pcap_id)" ng-class="">
-            <td>{{ doc._source.message.pcap_id }}</td>
-            <td>{{ doc._source.message.ip_src_port }}</td>
-            <td>{{ doc._source.message.dst_port }}</td>
-            <td>{{ doc._source.message.ip_src_addr }}</td>
-            <td>{{ doc._source.message.ip_dst_addr }}</td>
-            <td>{{ doc._source.message.ip_protocol }}</td>
-          </tr>
-      </tbody>
-      </table>
-    </div>
-
-    <table class="table table-bordered table-condensed table-details table-striped" ng-style="panel.style">
-    <thead>
-      <tr>
-        <td>Packet No.</td>
-        <td>Source</td>
-        <td>Destination</td>
-        <td>Info</td>
-      <tr>
-    </thead>
-    <tbody bindonce ng-repeat="packet in packet_data.pdml.packet">
-        <tr ng-click="$parent.drilldown = packet; setSelected($index);" ng-class="{highlight
: $index === selectedValue}">
-          <td><span>{{$index}}</span></td>
-          <td>{{packet.proto[3].field[10].$.show}}</td>
-          <td>{{packet.proto[3].field[14].$.show}}</td>
-          <td>{{packet.proto[4].field[0].$.showname}}</td>
-        </tr>
-    </tbody>
-    </table>
-
-      <table class="">
-       <thead></thead>
-       <!-- TODO: Fix repeater bug -->
-        <tbody bindonce ng-repeat="event in data| slice:panel.offset:panel.offset+panel.size"
ng-class-odd="'odd'" class="extra-row" ng-show="activePosition == $index">
-          <tr bindonce ng-repeat="prot in event.proto" class="main-row">
-            <td><div ng-click="prot.isVisible = !prot.isVisible">{{protShowname(prot.$)}}</div>
-            <div>
-              <ul ng-show="prot.isVisible">
-                <li ng-repeat="fieldval in prot.field" ng-click="fieldBytes(fieldval)">
-                    <p>{{fieldvalShowname(fieldval)}}</p>
-                </li>
-            </ul>
-            </div>
-            </td>
-          </tr>
-       </tbody>
-      </table>
-
-    <div packet="drilldown"></div>
 </div>
 
-  <script type="text/ng-template" id="packet.html">
-    <div ng-repeat="proto in packet.proto">
-      <span ng-class="{'icon-chevron-down': proto.expanded, 'icon-chevron-right': !proto.expanded}"
ng-click="proto.expanded = select(proto) || !proto.expanded"></span>
-      <span ng-class="{highlight: (selectedData.uid === proto.$.uid) || (selectedData2
=== proto.$.name )}" ng-click="proto.expanded = select(proto) || !proto.expanded">
-        {{ proto.$.showname }}
-      </span>
-
-      <div ng-if="proto.expanded" style="margin-left:50px">
-        <div ng-repeat="f in proto.field">
-          <span ng-class="{highlight: ($parent.selectedData.uid === f.$.uid) || (selectedData2
=== f.$)}" ng-click="select_field(f)">
-            {{ f.$.showname }}
-          </span>
-        </div>
-      </div>
-    </div>
-    <div hex-bytes bytes="packet.hexPacket" selected-bytes="selectedBytes" selected-data="selectedData"></div>
-  </script>
-
-  <script type="text/ng-template" id="hexBytes.html">
-    <br/>
-    <span ng-repeat="b in bytes track by $index">
-      <span class="byte" ng-class="{highlight: selectedBytes.pos <= $index &&
(selectedBytes.pos|num)+(selectedBytes.size|num) > $index, newline:$index%16==0, padright:($index+8)%16==0}"
ng-click="selectByte($index)" style="float:left; padding:4px;">
-        {{ b }}
-      </span>
-    </span>
-    <div style="clear:left">
-    <br/>
-    Position: {{selectedBytes.pos}}<br/>
-    Size: {{selectedBytes.size}}<br/>
-    Selected Field: {{selectedBytes.showname}} ({{selectedBytes.name}})</br>
-    selectedBytes :: {{$parent.selectedBytes}}</br>
-    selectedBytes.expand :: {{$parent.selectedBytes.expanded}}
-    </div>
-  </script>
-
   <!-- Filter PCAP Panel -->
   <script type="text/ng-template" id="pcapSettings.html">
     <div class="editor-row">
@@ -221,9 +121,11 @@
           <div class="editor-option">
               <label class="small">Protocol</label><input type="text" class="input-medium"
ng-model="ip_protocol"></input>
           </div>
-
+          <div class="editor-option">
+            <label class="small">Include Reverse Traffic</label><input type="checkbox"
value="false" class="input-medium" ng-model="include_reverse_traffic"></input>
+          </div>
           <div class="edit-option">
-          <button type="button" class="btn btn-success" ng-click="search()">Search</button>
+            <button type="button" class="btn btn-success" ng-click="getPcap()">Search</button>
           </div>
 
         </form>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/public/app/panels/pcap/module.js
----------------------------------------------------------------------
diff --git a/metron-ui/lib/public/app/panels/pcap/module.js b/metron-ui/lib/public/app/panels/pcap/module.js
index 5991444..47c21f2 100755
--- a/metron-ui/lib/public/app/panels/pcap/module.js
+++ b/metron-ui/lib/public/app/panels/pcap/module.js
@@ -168,6 +168,11 @@ function (angular, app, _, require, kbn) {
         });
     }
 
+    $scope.getPcap = function() {
+      var url = '/pcap/getPcapsByIdentifiers?srcIp=' + $scope.ip_src_addr + '&dstIp='
+ $scope.ip_dst_addr + '&protocol=' + $scope.ip_protocol + '&srcPort=' + $scope.ip_src_port
+ '&dstPort=' + $scope.dst_port + '&includeReverseTraffic=' + $scope.include_reverse_traffic;
+      window.location = url;
+    };
+
     // Query for PCAP IDS
     $scope.search = function() {
       var client = $scope.ejs.Request()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/views/alerts.jade
----------------------------------------------------------------------
diff --git a/metron-ui/lib/views/alerts.jade b/metron-ui/lib/views/alerts.jade
index 0f75033..758e3b5 100644
--- a/metron-ui/lib/views/alerts.jade
+++ b/metron-ui/lib/views/alerts.jade
@@ -1,21 +1,21 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- 
+//
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+//
+
 doctype html
 html
   head

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/views/index.jade
----------------------------------------------------------------------
diff --git a/metron-ui/lib/views/index.jade b/metron-ui/lib/views/index.jade
index 67525e5..4246b58 100644
--- a/metron-ui/lib/views/index.jade
+++ b/metron-ui/lib/views/index.jade
@@ -1,20 +1,20 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+//
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+//
 
 doctype html
 html

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/lib/views/login.jade
----------------------------------------------------------------------
diff --git a/metron-ui/lib/views/login.jade b/metron-ui/lib/views/login.jade
index d76ca9b..d2da35c 100644
--- a/metron-ui/lib/views/login.jade
+++ b/metron-ui/lib/views/login.jade
@@ -1,20 +1,20 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
+//
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+//
 
 doctype html
 html

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/c737aa9d/metron-ui/package.json
----------------------------------------------------------------------
diff --git a/metron-ui/package.json b/metron-ui/package.json
index c904718..d142489 100644
--- a/metron-ui/package.json
+++ b/metron-ui/package.json
@@ -1,62 +1,26 @@
 {
   "name": "metron-ui",
   "version": "0.0.0",
-  "description": "Metron Portal",
-  "main": "index.js",
   "private": true,
   "scripts": {
-    "test": "make test",
-    "seed": "make seed"
+    "start": "node ./bin/www"
   },
-  "repository": {
-    "type": "git",
-    "url": "git://github.com/apache/incubator-metron.git"
-  },
-  "author": "",
-  "license": "Apache-2.0",
-  "bugs": {
-    "url": "https://issues.apache.org/jira/browse/METRON"
-  },
-  "homepage": "http://metron.incubator.apache.org",
   "dependencies": {
-    "async": "^0.9.0",
-    "bcrypt": "^0.7.8",
-    "bluebird": "^1.2.4",
-    "checkit": "^0.2.0-pre",
-    "connect": "^2.16.2",
-    "connect-flash": "^0.1.1",
-    "connect-redis": "^2.0.0",
-    "express": "^4.2.0",
-    "jade": "^1.3.1",
-    "kappa": "^0.14.3",
-    "lodash": "^2.4.1",
-    "lusca": "^1.0.0",
-    "passport": "^0.2.0",
-    "passport-ldapauth": "^0.2.0",
-    "passport-local": "^1.0.0",
-    "redis": "^0.10.2",
-    "ws": "^0.4.31",
-    "xxhashjs": "0.0.5",
-    "http-proxy": "1.3.0",
-    "cookie-parser": "~1.3.2",
-    "body-parser": "~1.6.5",
-    "cookie-session": "~1.0.2",
-    "xml-stream": "~0.4.4"
-  },
-  "devDependencies": {
-    "chai": "^1.9.1",
-    "chance": "^0.5.6",
-    "glob": "^3.2.9",
-    "grunt": "^0.4.4",
-    "grunt-bowercopy": "^1.0.0",
-    "istanbul": "^0.2.7",
-    "karma": "^0.12.14",
-    "karma-coverage": "^0.2.1",
-    "karma-mocha": "^0.1.3",
-    "karma-osx-reporter": "0.0.4",
-    "karma-sinon-chai": "^0.1.5",
-    "mocha": "^1.19.0",
-    "sinon": "^1.9.1",
-    "supertest": "^0.11.0"
+    "body-parser": "~1.13.2",
+    "cookie-parser": "~1.3.5",
+    "debug": "~2.2.0",
+    "express": "~4.13.1",
+    "jade": "~1.11.0",
+    "morgan": "~1.6.1",
+    "serve-favicon": "~2.3.0",
+    "lodash": "~4.6.1",
+    "connect": "3.4.1",
+    "connect-flash": "~0.1.1",
+    "cookie-session": "~2.0.0-alpha.1",
+    "passport": "~0.3.2",
+    "passport-ldapauth": "~0.5.0",
+    "http-proxy": "~1.13.2",
+    "xml-stream": "~0.4.5",
+    "serve-static": "~1.10.2"
   }
-}
+}
\ No newline at end of file



Mime
View raw message