metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject [05/44] metron git commit: METRON-907: Zeppelin Dashboard to execute and download pcap queries closes apache/incubator-metron#559
Date Thu, 01 Jun 2017 21:41:24 GMT
METRON-907: Zeppelin Dashboard to execute and download pcap queries closes apache/incubator-metron#559


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1d27a32d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1d27a32d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1d27a32d

Branch: refs/heads/Metron_0.4.0
Commit: 1d27a32d5fe20ecb57a0cc143c1cb707f79ad20c
Parents: 494643c
Author: cstella <cestella@gmail.com>
Authored: Wed May 3 10:14:41 2017 -0400
Committer: cstella <cestella@gmail.com>
Committed: Wed May 3 10:14:41 2017 -0400

----------------------------------------------------------------------
 .../docker/rpm-docker/SPECS/metron.spec         |  2 +
 .../config/zeppelin/metron/metron-pcap.json     |  1 +
 .../org/apache/metron/pcap/query/CliConfig.java | 19 ++++-
 .../org/apache/metron/pcap/query/CliParser.java |  4 +-
 .../metron/pcap/query/FixedCliConfig.java       |  3 +-
 .../metron/pcap/query/FixedCliParser.java       |  9 +-
 .../org/apache/metron/pcap/query/PcapCli.java   | 23 +++--
 .../metron/pcap/query/QueryCliConfig.java       |  4 +
 .../metron/pcap/query/QueryCliParser.java       |  9 +-
 .../src/main/scripts/pcap_zeppelin_run.sh       | 88 ++++++++++++++++++++
 .../apache/metron/pcap/query/PcapCliTest.java   | 27 +++---
 11 files changed, 158 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index b1ba637..c435c6c 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -288,7 +288,9 @@ This package installs the Metron PCAP files %{metron_home}
 %{metron_home}/bin/pcap_inspector.sh
 %{metron_home}/bin/pcap_query.sh
 %{metron_home}/bin/start_pcap_topology.sh
+%{metron_home}/bin/pcap_zeppelin_run.sh
 %{metron_home}/flux/pcap/remote.yaml
+%{metron_home}/config/zeppelin/metron/metron-pcap.json
 %attr(0644,root,root) %{metron_home}/lib/metron-pcap-backend-%{full_version}.jar
 
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json
b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json
new file mode 100644
index 0000000..8c853a5
--- /dev/null
+++ b/metron-platform/metron-pcap-backend/src/main/config/zeppelin/metron/metron-pcap.json
@@ -0,0 +1 @@
+{"paragraphs":[{"text":"%md\n# Execute Packet Capture Queries\n\nSpecify the following
to filter the packet capture query:\n* *end time* - The ending time of the query in yyyyMMdd
format (e.g. 20170428)\n* *start time* - The starting time of the query in yyyyMMdd format
(e.g. 20170428)\n* *query* - The [Stellar](https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-language)
query (i.e. a Stellar expression that returns `true` or `false`) to specify the packets.\n\nThe
available fields to use in the queries are as follows:\n* `ip_src_addr` - The source IP address
of the packets filtered\n* `ip_src_port` - The source port of the packets filtered\n* `ip_dst_addr`
- The destination IP address of the packets filtered\n* `ip_dst_port` - The destination port
of the packets filtered\n* `packet` - The raw packet (for use with the `BYTEARRAY_MATCHER`
function)\n\nYou can use any [Stellar function](https://github.com/apache/incubator-metron/tree/maste
 r/metron-platform/metron-common#stellar-core-functions)\n\n## Simple Boolean Expressions\n\nFor
example:\n* `ip_dst_port == 8080` would return all packets where the destination port is `8080`\n*
`ip_dst_port in [ 8080, 80 ]` would return all packets where the destination port is either
`8080` or `80`\n\n## Common Network Functions on Metadata\n\nFor example:\n* `IN_SUBNET(ip_dst_addr,
'192.168.0.0/24')` would return all packets whose destination conforms to the CIDR `192.168.0.0/24`\n*
`IN_SUBNET(ip_dst_addr, '192.168.0.0/24') && ip_dst_port == 8080` would return all
packets matching the CIDR and whose destination port is `8080`\n \n## Filtering based on the
Packet Contents\n\nWe use byteseek regular expressions to filter packets.  The syntax for
these is described [here](https://github.com/nishihatapalmer/byteseek/blob/master/sequencesyntax.md).\n*
`BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all packets that contain the
string `/api/v1` in them anywhere.\n* `ip_ds
 t_port==8080 && BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'', packet)` would return all
packets that contain the string `/api/v1` and have a destination port of `8080`\n* `BYTEARRAY_MATCHER('ff(.){5}ff',
packet)` would return all packets containing a binary regex with `0xff` followed by any 5
bytes and then `0xff`\n\n# The Output\nThe output will be a table of links to the various
parts of the packet capture files.  The files will be named in temporal order.","dateUpdated":"2017-05-01T17:36:27+0000","config":{"colWidth":12,"editorMode":"ace/mode/markdown","editorHide":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{"query":"ip_dst_port==8080
&& BYTEARRAY_MATCHER('\\\\\\'/api/v1/\\\\\\'', packet)"},"forms":{}},"jobName":"paragraph_1493658709556_1010155640","id":"20170428-183346_777875025","result":{"code":"SUCCESS","type":"HTML","msg":"<h1>Execute
Packet Capture Queries</h1>\n<p>Specify the fo
 llowing to filter the packet capture query:</p>\n<ul>\n<li><em>end
time</em> - The ending time of the query in yyyyMMdd format (e.g. 20170428)</li>\n<li><em>start
time</em> - The starting time of the query in yyyyMMdd format (e.g. 20170428)</li>\n<li><em>query</em>
- The <a href=\"https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-language\">Stellar</a>
query (i.e. a Stellar expression that returns <code>true</code> or <code>false</code>)
to specify the packets.</li>\n</ul>\n<p>The available fields to use in the
queries are as follows:</p>\n<ul>\n<li><code>ip_src_addr</code>
- The source IP address of the packets filtered</li>\n<li><code>ip_src_port</code>
- The source port of the packets filtered</li>\n<li><code>ip_dst_addr</code>
- The destination IP address of the packets filtered</li>\n<li><code>ip_dst_port</code>
- The destination port of the packets filtered</li>\n<li><code>packet</code>
- The raw packet (for use with the <code>BYTEARR
 AY_MATCHER</code> function)</li>\n</ul>\n<p>You can use any <a
href=\"https://github.com/apache/incubator-metron/tree/master/metron-platform/metron-common#stellar-core-functions\">Stellar
function</a></p>\n<h2>Simple Boolean Expressions</h2>\n<p>For
example:</p>\n<ul>\n<li><code>ip_dst_port == 8080</code> would
return all packets where the destination port is <code>8080</code></li>\n<li><code>ip_dst_port
in [ 8080, 80 ]</code> would return all packets where the destination port is either
<code>8080</code> or <code>80</code></li>\n</ul>\n<h2>Common
Network Functions on Metadata</h2>\n<p>For example:</p>\n<ul>\n<li><code>IN_SUBNET(ip_dst_addr,
'192.168.0.0/24')</code> would return all packets whose destination conforms to the
CIDR <code>192.168.0.0/24</code></li>\n<li><code>IN_SUBNET(ip_dst_addr,
'192.168.0.0/24') &amp;&amp; ip_dst_port == 8080</code> would return all packets
matching the CIDR and whose destination port is <code>8080</code></li>\n</ul>\n<h2>Filtering
based on the Pack
 et Contents</h2>\n<p>We use byteseek regular expressions to filter packets. 
The syntax for these is described <a href=\"https://github.com/nishihatapalmer/byteseek/blob/master/sequencesyntax.md\">here</a>.</p>\n<ul>\n<li><code>BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'',
packet)</code> would return all packets that contain the string <code>/api/v1</code>
in them anywhere.</li>\n<li><code>ip_dst_port==8080 &amp;&amp; BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'',
packet)</code> would return all packets that contain the string <code>/api/v1</code>
and have a destination port of <code>8080</code></li>\n<li><code>BYTEARRAY_MATCHER('ff(.){5}ff',
packet)</code> would return all packets containing a binary regex with <code>0xff</code>
followed by any 5 bytes and then <code>0xff</code></li>\n</ul>\n<h1>The
Output</h1>\n<p>The output will be a table of links to the various parts of the
packet capture files.  The files will be named in temporal order.</p>\n"},"dateCreated":"2017-05-01T17:11:49+0000","stat
 us":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1724","dateFinished":"2017-05-01T17:36:26+0000","dateStarted":"2017-05-01T17:36:26+0000","focus":true},{"text":"%sh\nexport
PCAP_ZEPPELIN_RUN=$(find /usr -name pcap_zeppelin_run.sh)\nexport RECORDS_PER_FILE=10000\nexport
NUMBER_OF_REDUCERS=10\nexport DATE_FORMAT=\"yyyyMMdd\"\nexport PCAP_DATA_PATH=\"/apps/metron/pcap\"\n\n$PCAP_ZEPPELIN_RUN
\"${query}\" \"${start time}\" \"${optional end time}\"","dateUpdated":"2017-05-01T17:35:04+0000","config":{"colWidth":12,"editorMode":"ace/mode/sh","editorHide":false,"graph":{"mode":"table","height":164,"optionOpen":false,"keys":[{"name":"Packets
conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting
at 20170428 ending now","index":0,"aggr":"sum"}],"values":[],"groups":[],"scatter":{"xAxis":{"name":"Packets
conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting
at 20170428 ending now","index":0,"aggr":"sum"}}},"e
 nabled":true},"settings":{"params":{"start time":"20170428","end time":"","query":"BYTEARRAY_MATCHER('\\\\'/api/v1\\\\'',
packet) && ip_dst_port == 8080","optional end time":""},"forms":{"optional end time":{"name":"optional
end time","defaultValue":"","hidden":false},"query":{"name":"query","defaultValue":"","hidden":false},"start
time":{"name":"start time","defaultValue":"","hidden":false}}},"jobName":"paragraph_1493658709562_1009386142","id":"20170428-181957_829993114","result":{"code":"SUCCESS","type":"TABLE","msg":"Packets
conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting
at 20170428 ending now\n%html <a href=\"http://node1:50070/webhdfs/v1/user/zeppelin/queries/pcap-data-20170501173442578/pcap-data-20170501173442578%2B0000.pcap?op=OPEN\">pcap-data-20170501173442578+0000.pcap</a>\n","comment":"","msgTable":[[{"value":"%html
<a href=\"http://node1:50070/webhdfs/v1/user/zeppelin/queries/pcap-data-20170501173442578/pcap-data-20170501173442578
 %2B0000.pcap?op=OPEN\">pcap-data-20170501173442578+0000.pcap</a>"}]],"columnNames":[{"name":"Packets
conforming to BYTEARRAY_MATCHER('\\'/api/v1\\'', packet) && ip_dst_port == 8080 starting
at 20170428 ending now","index":0,"aggr":"sum"}],"rows":[["%html <a href=\"http://node1:50070/webhdfs/v1/user/zeppelin/queries/pcap-data-20170501173442578/pcap-data-20170501173442578%2B0000.pcap?op=OPEN\">pcap-data-20170501173442578+0000.pcap</a>"]]},"dateCreated":"2017-05-01T17:11:49+0000","dateStarted":"2017-05-01T17:35:04+0000","dateFinished":"2017-05-01T17:34:55+0000","status":"RUNNING","progressUpdateIntervalMs":500,"$$hashKey":"object:1725","focus":true},{"dateUpdated":"2017-05-01T17:34:55+0000","config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true,"editorMode":"ace/mode/markdown","editorHide":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1493658709563_1009001393","id":"20170428-184335_
 1604096389","dateCreated":"2017-05-01T17:11:49+0000","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:1726","dateFinished":"2017-05-01T17:34:54+0000","dateStarted":"2017-05-01T17:34:54+0000","result":{"code":"SUCCESS","type":"HTML","msg":"<h1>Troubleshooting</h1>\n<p>If
you are having problems with the above form, the following may help.</p>\n<h2>I
see <code>Terminated by SIGINTERRUPT</code> or something similar in the output!</h2>\n<p>PCAP
filtering happens via a batch process and on busy systems, this can take some time.  You very
well may need to request this of your system administrator.\n<br  />They can do this
by changing the <code>shell.command.timeout.millisecs</code> property for the
<code>sh</code> interpreter to a larger value, likely <code>100000</code>.</p>\n<h2>I
do not see a table of URLs to pcap files in my output, what happened?</h2>\n<p>If
an error happens, the log of the pcap querying utility will be displayed instead of an output.
 Please co
 ntact an administrator with this output to debug further.</p>\n"},"text":"%md\n# Troubleshooting\n\nIf
you are having problems with the above form, the following may help.\n\n## I see `Terminated
by SIGINTERRUPT` or something similar in the output!\n\nPCAP filtering happens via a batch
process and on busy systems, this can take some time.  You very well may need to request this
of your system administrator.\nThey can do this by changing the `shell.command.timeout.millisecs`
property for the `sh` interpreter to a larger value, likely `100000`.\n\n## I do not see a
table of URLs to pcap files in my output, what happened?\n\nIf an error happens, the log of
the pcap querying utility will be displayed instead of an output.  Please contact an administrator
with this output to debug further.","focus":true},{"config":{"colWidth":12,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobN
 ame":"paragraph_1493659778398_1082698332","id":"20170501-172938_213921861","dateCreated":"2017-05-01T17:29:38+0000","status":"READY","progressUpdateIntervalMs":500,"focus":true,"$$hashKey":"object:1972"}],"name":"metron/pcap","id":"2CEEXVR1W","angularObjects":{"2CFNGE6TP:shared_process":[],"2CGUX5TSW:shared_process":[],"2CETVR2AB:shared_process":[],"2CF163WFX:shared_process":[],"2CG4YXKUV:shared_process":[],"2CG6QDFF7:shared_process":[]},"config":{"looknfeel":"default"},"info":{}}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
index 294844f..1d8e3f3 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliConfig.java
@@ -18,12 +18,19 @@
 package org.apache.metron.pcap.query;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.common.system.Clock;
 
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
+import java.util.UUID;
+import java.util.function.Consumer;
+import java.util.function.Function;
 
 public class CliConfig {
+  public interface PrefixStrategy extends Function<Clock, String>{}
+
   private boolean showHelp;
+  private String prefix;
   private String basePath;
   private String baseOutputPath;
   private long startTime;
@@ -32,13 +39,23 @@ public class CliConfig {
   private int numRecordsPerFile;
   private DateFormat dateFormat;
 
-  public CliConfig() {
+
+  public CliConfig(PrefixStrategy prefixStrategy) {
     showHelp = false;
     basePath = "";
     baseOutputPath = "";
     startTime = -1L;
     endTime = -1L;
     numReducers = 0;
+    prefix = prefixStrategy.apply(new Clock());
+  }
+
+  public String getPrefix() {
+    return prefix;
+  }
+
+  public void setPrefix(String prefix) {
+    this.prefix = prefix;
   }
 
   public int getNumReducers() {

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 83e9fcf..d5976ae 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -29,8 +29,10 @@ public class CliParser {
   public static final int NUM_REDUCERS_DEFAULT = 10;
   public static final int NUM_RECORDS_PER_FILE_DEFAULT = 10000;
   private CommandLineParser parser;
+  protected CliConfig.PrefixStrategy prefixStrategy;
 
-  public CliParser() {
+  public CliParser(CliConfig.PrefixStrategy prefixStrategy) {
+    this.prefixStrategy = prefixStrategy;
     parser = new PosixParser();
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
index df653e1..03caed7 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliConfig.java
@@ -27,7 +27,8 @@ public class FixedCliConfig extends CliConfig {
 
   private Map<String, String> fixedFields;
 
-  public FixedCliConfig() {
+  public FixedCliConfig(PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
     this.fixedFields = new LinkedHashMap<>();
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
index fda8692..4e1bfcf 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/FixedCliParser.java
@@ -26,7 +26,8 @@ import org.apache.metron.pcap.PcapHelper;
 public class FixedCliParser extends CliParser {
   private Options fixedOptions;
 
-  public FixedCliParser() {
+  public FixedCliParser(CliConfig.PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
     fixedOptions = buildFixedOptions();
   }
 
@@ -38,6 +39,7 @@ public class FixedCliParser extends CliParser {
     options.addOption(newOption("dp", "ip_dst_port", true, "Destination port"));
     options.addOption(newOption("p", "protocol", true, "IP Protocol"));
     options.addOption(newOption("pf", "packet_filter", true, "Packet Filter regex"));
+    options.addOption(newOption("pre", "prefix", true, "Result file prefix to use"));
     options.addOption(newOption("ir", "include_reverse", false, "Indicates if filter should
check swapped src/dest addresses and IPs"));
     return options;
   }
@@ -51,7 +53,7 @@ public class FixedCliParser extends CliParser {
    */
   public FixedCliConfig parse(String[] args) throws ParseException, java.text.ParseException
{
     CommandLine commandLine = getParser().parse(fixedOptions, args);
-    FixedCliConfig config = new FixedCliConfig();
+    FixedCliConfig config = new FixedCliConfig(prefixStrategy);
     super.parse(commandLine, config);
     config.putFixedField(Constants.Fields.SRC_ADDR.getName(), commandLine.getOptionValue("ip_src_addr"));
     config.putFixedField(Constants.Fields.DST_ADDR.getName(), commandLine.getOptionValue("ip_dst_addr"));
@@ -60,6 +62,9 @@ public class FixedCliParser extends CliParser {
     config.putFixedField(Constants.Fields.PROTOCOL.getName(), commandLine.getOptionValue("protocol"));
     config.putFixedField(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), Boolean.toString(commandLine.hasOption("include_reverse")));
     config.putFixedField(PcapHelper.PacketFields.PACKET_FILTER.getName(), commandLine.getOptionValue("packet_filter"));
+    if(commandLine.hasOption("prefix")) {
+      config.setPrefix(commandLine.getOptionValue("prefix"));
+    }
     return config;
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index d2e6807..b02e2e2 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -36,22 +36,28 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.List;
+import java.util.UUID;
 
 public class PcapCli {
   private static final Logger LOGGER = LoggerFactory.getLogger(PcapCli.class);
+  public static final CliConfig.PrefixStrategy PREFIX_STRATEGY = clock -> {
+    String timestamp = new Clock().currentTimeFormatted("yyyyMMddHHmm");
+    String uuid = UUID.randomUUID().toString().replaceAll("-", "");
+    return String.format("%s-%s", timestamp, uuid);
+  };
   private final PcapJob jobRunner;
   private final ResultsWriter resultsWriter;
-  private final Clock clock;
+  private final CliConfig.PrefixStrategy prefixStrategy;
 
   public static void main(String[] args) {
-    int status = new PcapCli(new PcapJob(), new ResultsWriter(), new Clock()).run(args);
+    int status = new PcapCli(new PcapJob(), new ResultsWriter(), PREFIX_STRATEGY).run(args);
     System.exit(status);
   }
 
-  public PcapCli(PcapJob jobRunner, ResultsWriter resultsWriter, Clock clock) {
+  public PcapCli(PcapJob jobRunner, ResultsWriter resultsWriter, CliConfig.PrefixStrategy
prefixStrategy) {
     this.jobRunner = jobRunner;
     this.resultsWriter = resultsWriter;
-    this.clock = clock;
+    this.prefixStrategy = prefixStrategy;
   }
 
   public int run(String[] args) {
@@ -72,7 +78,7 @@ public class PcapCli {
     }
     CliConfig commonConfig = null;
     if ("fixed".equals(jobType)) {
-      FixedCliParser fixedParser = new FixedCliParser();
+      FixedCliParser fixedParser = new FixedCliParser(prefixStrategy);
       FixedCliConfig config = null;
       try {
         config = fixedParser.parse(otherArgs);
@@ -110,7 +116,7 @@ public class PcapCli {
         return -1;
       }
     } else if ("query".equals(jobType)) {
-      QueryCliParser queryParser = new QueryCliParser();
+      QueryCliParser queryParser = new QueryCliParser(prefixStrategy);
       QueryCliConfig config = null;
       try {
         config = queryParser.parse(otherArgs);
@@ -151,11 +157,12 @@ public class PcapCli {
       return -1;
     }
     try {
+
       Iterable<List<byte[]>> partitions = Iterables.partition(results, commonConfig.getNumRecordsPerFile());
+      int part = 1;
       if (partitions.iterator().hasNext()) {
         for (List<byte[]> data : partitions) {
-          String timestamp = clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ");
-          String outFileName = String.format("pcap-data-%s.pcap", timestamp);
+          String outFileName = String.format("pcap-data-%s+%04d.pcap", commonConfig.getPrefix(),
part++);
           if(data.size() > 0) {
             resultsWriter.write(data, outFileName);
           }

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
index 3d06e1d..67f045f 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliConfig.java
@@ -20,6 +20,10 @@ package org.apache.metron.pcap.query;
 public class QueryCliConfig extends CliConfig {
   private String query;
 
+  public QueryCliConfig(PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
+  }
+
   public String getQuery() {
     return query;
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
index 72b5a95..d6e5cd1 100644
--- a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
+++ b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/QueryCliParser.java
@@ -24,13 +24,15 @@ import org.apache.commons.cli.ParseException;
 public class QueryCliParser extends CliParser {
   private Options queryOptions;
 
-  public QueryCliParser() {
+  public QueryCliParser(CliConfig.PrefixStrategy prefixStrategy) {
+    super(prefixStrategy);
     queryOptions = setupOptions();
   }
 
   private Options setupOptions() {
     Options options = buildOptions();
     options.addOption(newOption("q", "query", true, "Query string to use as a filter"));
+    options.addOption(newOption("pre", "prefix", true, "Result file prefix to use"));
     return options;
   }
 
@@ -43,11 +45,14 @@ public class QueryCliParser extends CliParser {
    */
   public QueryCliConfig parse(String[] args) throws ParseException, java.text.ParseException
{
     CommandLine commandLine = getParser().parse(queryOptions, args);
-    QueryCliConfig config = new QueryCliConfig();
+    QueryCliConfig config = new QueryCliConfig(prefixStrategy);
     super.parse(commandLine, config);
     if (commandLine.hasOption("query")) {
       config.setQuery(commandLine.getOptionValue("query"));
     }
+    if(commandLine.hasOption("prefix")) {
+      config.setPrefix(commandLine.getOptionValue("prefix"));
+    }
     return config;
   }
 

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
new file mode 100755
index 0000000..3561c9d
--- /dev/null
+++ b/metron-platform/metron-pcap-backend/src/main/scripts/pcap_zeppelin_run.sh
@@ -0,0 +1,88 @@
+#!/bin/bash 
+# 
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# 
+
+METRON_VERSION=0.4.0
+METRON_HOME=${METRON_HOME:-"/usr/metron/$METRON_VERSION"}
+DATE_FORMAT=${DATE_FORMAT:-"yyyyMMdd"}
+USER=$(whoami)
+USER_HOMEDIR=${USER_HOMEDIR:-`hdfs getconf -confKey dfs.user.home.dir.prefix`/$USER}
+QUERY_HOME=${QUERY_HOME:-"$USER_HOMEDIR/queries"}
+WEBHDFS_HOSTNAME=${WEBHDFS_HOSTNAME:-`hdfs getconf -confKey dfs.namenode.http-address`}
+QUERY=${QUERY:-$1}
+START_TIME=${START_TIME:-$2}
+END_TIME=${END_TIME:-$3}
+RECORDS_PER_FILE=${RECORDS_PER_FILE:-10000}
+NUMBER_OF_REDUCERS=${NUMBER_OF_REDUCERS:-10}
+PCAP_DATA_PATH=${PCAP_DATA_PATH:-/apps/metron/pcap}
+if [ -z "$QUERY" ]; then
+  echo "You must specify a query."
+  exit 1
+fi
+if [ -z "$START_TIME" ]; then
+  echo "You must specify a start time."
+  exit 2
+fi
+TIMESTAMP_EPOCH=$(date +%s%N | cut -b1-13)
+if [ -f /proc/sys/kernel/random/uuid ];then
+  UUID=$(cat /proc/sys/kernel/random/uuid | sed 's/-//g')
+  PREFIX="$TIMESTAMP_EPOCH-$UUID"
+else
+  QUERY_HASH=$(echo $QUERY | md5sum | awk '{print $1}')
+  PREFIX="$TIMESTAMP_EPOCH-$QUERY_HASH"
+fi
+if [ -z "$END_TIME" ]; then
+  CMD=$($METRON_HOME/bin/pcap_query.sh query --prefix "$PREFIX" --query "$QUERY" -st "$START_TIME"
-df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp "$PCAP_DATA_PATH"
\"2>&1)
+  SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending now"
+else
+  CMD=$($METRON_HOME/bin/pcap_query.sh query --prefix "$PREFIX" --query "$QUERY" -st "$START_TIME"
-et "$END_TIME" -df "$DATE_FORMAT" -rpf "$RECORDS_PER_FILE" -nr "$NUMBER_OF_REDUCERS" -bp
"$PCAP_DATA_PATH" 2>&1)
+  SUMMARY="Packets conforming to $QUERY starting at $START_TIME ending at $END_TIME"
+fi
+
+FAILED=$(echo $CMD | grep "Unable to complete query due to errors")
+if [ -z "$FAILED" ];then
+  PATTERN="pcap-data-$PREFIX"
+  hadoop fs -mkdir -p $QUERY_HOME/$PATTERN && hadoop fs -put $PATTERN* $QUERY_HOME/$PATTERN

+  FAILED=$?
+  if [ $FAILED -eq 0 ];then
+    echo "%html"
+    echo "<h4>$SUMMARY</h4>"
+    echo "<ul>"
+    for i in $(ls $PATTERN*.pcap | sort -n);do
+      FILENAME=$(echo $i | sed 's/+/%2B/g')
+      SIZE=$(du -h $i | awk '{print $1}')
+      echo "<li><a href=\"http://$WEBHDFS_HOSTNAME/webhdfs/v1$QUERY_HOME/$PATTERN/$FILENAME?op=OPEN\">$i</a>
($SIZE)</li>"
+      rm $i
+    done
+    echo "</ul>"
+    echo "<small>NOTE: There are $RECORDS_PER_FILE records per file</small>"
+  else
+    echo "Unable to create $QUERY_HOME/$PATTERN"
+    exit 3
+  fi
+else
+  echo "%html <pre>FAILED JOB:"
+  echo "QUERY: $QUERY"
+  echo "START_TIME: $START_TIME"
+  echo "DATE_FORMAT: $DATE_FORMAT"
+  echo "METRON_HOME: $METRON_HOME"
+  echo "DATE_FORMAT: $DATE_FORMAT"
+  echo "Output:"
+  echo "$CMD"
+  echo "</pre>"
+fi

http://git-wip-us.apache.org/repos/asf/metron/blob/1d27a32d/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 4f441f1..7202819 100644
--- a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++ b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -92,11 +92,10 @@ public class PcapCliTest {
     }};
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(),
eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   @Test
@@ -133,11 +132,10 @@ public class PcapCliTest {
     }};
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(),
eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   @Test
@@ -177,11 +175,10 @@ public class PcapCliTest {
     long startAsNanos = asNanos("2016-06-13-18:35.00", "yyyy-MM-dd-HH:mm.ss");
     long endAsNanos = asNanos("2016-06-15-18:35.00", "yyyy-MM-dd-HH:mm.ss");
     when(jobRunner.query(eq(base_path), eq(base_output_path), eq(startAsNanos), eq(endAsNanos),
anyInt(), eq(query), isA(Configuration.class), isA(FileSystem.class), isA(FixedPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   private long asNanos(String inDate, String format) throws ParseException {
@@ -211,11 +208,10 @@ public class PcapCliTest {
     String query = "some query string";
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(),
eq(query), isA(Configuration.class), isA(FileSystem.class), isA(QueryPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   @Test
@@ -240,11 +236,10 @@ public class PcapCliTest {
     String query = "some query string";
 
     when(jobRunner.query(eq(base_path), eq(base_output_path), anyLong(), anyLong(), anyInt(),
eq(query), isA(Configuration.class), isA(FileSystem.class), isA(QueryPcapFilter.Configurator.class))).thenReturn(iterable);
-    when(clock.currentTimeFormatted("yyyyMMddHHmmssSSSZ")).thenReturn("20160615183527162+0000");
 
-    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+    PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
     assertThat("Expect no errors on run", cli.run(args), equalTo(0));
-    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-20160615183527162+0000.pcap");
+    Mockito.verify(resultsWriter).write(pcaps, "pcap-data-random_prefix+0001.pcap");
   }
 
   // INVALID OPTION CHECKS
@@ -281,7 +276,7 @@ public class PcapCliTest {
       PrintStream errOutStream = new PrintStream(new BufferedOutputStream(ebos));
       System.setErr(errOutStream);
 
-      PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock);
+      PcapCli cli = new PcapCli(jobRunner, resultsWriter, clock -> "random_prefix");
       assertThat("Expect errors on run", cli.run(args), equalTo(-1));
       assertThat("Expect missing required option error: " + ebos.toString(), ebos.toString().contains(optMsg),
equalTo(true));
       assertThat("Expect usage to be printed: " + bos.toString(), bos.toString().contains("usage:
" + type + " filter options"), equalTo(true));


Mime
View raw message