metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmerri...@apache.org
Subject metron git commit: METRON-1555 Update REST to run YARN and MR jobs (merrimanr) closes apache/metron#1019
Date Tue, 26 Jun 2018 15:18:35 GMT
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1554-pcap-query-panel 77570464b -> 165c0e716


METRON-1555 Update REST to run YARN and MR jobs (merrimanr) closes apache/metron#1019


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/165c0e71
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/165c0e71
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/165c0e71

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: 165c0e716342d743460123a9de6303178ed32830
Parents: 7757046
Author: merrimanr <merrimanr@gmail.com>
Authored: Tue Jun 26 10:18:01 2018 -0500
Committer: rmerriman <merrimanr@gmail.com>
Committed: Tue Jun 26 10:18:01 2018 -0500

----------------------------------------------------------------------
 dependencies_with_url.csv                       |   1 +
 .../package/scripts/params/params_linux.py      |  10 ++
 .../package/scripts/params/status_params.py     |   6 +
 .../CURRENT/package/scripts/rest_commands.py    |  37 ++++
 .../CURRENT/package/scripts/rest_master.py      |   4 +
 .../apache/metron/rest/model/PcapResponse.java  |  38 +++++
 .../rest/model/pcap/FixedPcapRequest.java       | 115 +++++++++++++
 .../metron/rest/model/pcap/PcapRequest.java     |  91 ++++++++++
 metron-interface/metron-rest/README.md          |   8 +
 metron-interface/metron-rest/pom.xml            |  33 ++++
 .../apache/metron/rest/MetronRestConstants.java |   3 +
 .../apache/metron/rest/config/PcapConfig.java   |  35 ++++
 .../metron/rest/controller/PcapController.java  |  53 ++++++
 .../apache/metron/rest/service/PcapService.java |  27 +++
 .../rest/service/impl/PcapServiceImpl.java      | 120 +++++++++++++
 .../src/main/resources/application.yml          |   6 +-
 .../metron-rest/src/main/scripts/metron-rest.sh |   2 +-
 .../apache/metron/rest/config/TestConfig.java   |   7 +
 .../PcapControllerIntegrationTest.java          | 129 ++++++++++++++
 .../apache/metron/rest/mock/MockPcapJob.java    |  97 +++++++++++
 .../rest/service/impl/PcapServiceImplTest.java  | 167 +++++++++++++++++++
 21 files changed, 987 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index df3bcd2..9fe3b2a 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -206,6 +206,7 @@ io.confluent:kafka-avro-serializer:jar:1.0:compile,ASLv2,https://github.com/conf
 io.confluent:kafka-schema-registry-client:jar:1.0:compile,ASLv2,https://github.com/confluentinc/schema-registry/
 io.netty:netty-all:jar:4.0.23.Final:compile,ASLv2,
 io.netty:netty-all:jar:4.0.23.Final:provided,ASLv2,
+io.netty:netty-all:jar:4.1.23.Final:compile,ASLv2,
 io.netty:netty:jar:3.10.5.Final:compile,Apache License, Version 2.0,http://netty.io/
 io.netty:netty:jar:3.6.2.Final:compile,Apache License, Version 2.0,http://netty.io/
 io.netty:netty:jar:3.7.0.Final:compile,Apache License, Version 2.0,http://netty.io/

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index f44d05f..a2d8185 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -39,6 +39,8 @@ import status_params
 config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
+hdp_version = default("/commandParams/version", None)
+
 hostname = config['hostname']
 metron_home = status_params.metron_home
 
@@ -357,3 +359,11 @@ metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-in
 bolt_hdfs_rotation_policy = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy']
 bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_units']
 bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
+
+# Pcap
+pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap")
+pcap_configured_flag_file = status_params.pcap_configured_flag_file
+
+# MapReduce
+metron_user_hdfs_dir = '/user/' + metron_user
+metron_user_hdfs_dir_configured_flag_file = status_params.metron_user_hdfs_dir_configured_flag_file

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index ed2edfb..cb26a87 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -114,3 +114,9 @@ metron_user = config['configurations']['metron-env']['metron_user']
 
 metron_principal_name = config['configurations']['metron-env']['metron_principal_name']
 metron_keytab_path = config['configurations']['metron-env']['metron_service_keytab']
+
+# Pcap
+pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured'
+
+# MapReduce
+metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured'
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index e97af05..73bafbb 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -37,6 +37,8 @@ class RestCommands:
     __kafka_acl_configured = False
     __hbase_configured = False
     __hbase_acl_configured = False
+    __pcap_configured = False
+    __metron_user_hdfs_dir_configured = False
 
     def __init__(self, params):
         if params is None:
@@ -46,6 +48,8 @@ class RestCommands:
         self.__kafka_acl_configured = os.path.isfile(self.__params.rest_kafka_acl_configured_flag_file)
         self.__hbase_configured = os.path.isfile(self.__params.rest_hbase_configured_flag_file)
         self.__hbase_acl_configured = os.path.isfile(self.__params.rest_hbase_acl_configured_flag_file)
+        self.__pcap_configured = os.path.isfile(self.__params.pcap_configured_flag_file)
+        self.__metron_user_hdfs_dir_configured = os.path.isfile(self.__params.metron_user_hdfs_dir_configured_flag_file)
         Directory(params.metron_rest_pid_dir,
                   mode=0755,
                   owner=params.metron_user,
@@ -74,6 +78,12 @@ class RestCommands:
     def is_hbase_acl_configured(self):
         return self.__hbase_acl_configured
 
+    def is_pcap_configured(self):
+        return self.__pcap_configured
+
+    def is_metron_user_hdfs_dir_configured(self):
+        return self.__metron_user_hdfs_dir_configured
+
     def set_kafka_configured(self):
         metron_service.set_configured(self.__params.metron_user, self.__params.rest_kafka_configured_flag_file, "Setting Kafka configured to True for rest")
 
@@ -86,6 +96,12 @@ class RestCommands:
     def set_hbase_acl_configured(self):
         metron_service.set_configured(self.__params.metron_user, self.__params.rest_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for rest")
 
+    def set_pcap_configured(self):
+        metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file, "Setting Pcap configured to True")
+
+    def set_metron_user_hdfs_dir_configured(self):
+        metron_service.set_configured(self.__params.metron_user, self.__params.metron_user_hdfs_dir_configured_flag_file, "Setting Metron user HDFS directory configured to True")
+
     def init_kafka_topics(self):
         Logger.info('Creating Kafka topics for rest')
         metron_service.init_kafka_topics(self.__params, self.__get_topics())
@@ -100,6 +116,26 @@ class RestCommands:
         groups = ['metron-rest']
         metron_service.init_kafka_acl_groups(self.__params, groups)
 
+    def init_pcap(self):
+        Logger.info("Creating HDFS location for Pcap")
+        self.__params.HdfsResource(self.__params.pcap_hdfs_dir,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.metron_group,
+                                   mode=0755,
+                                   )
+
+    def create_metron_user_hdfs_dir(self):
+        Logger.info("Creating HDFS location for Metron user")
+        self.__params.HdfsResource(self.__params.metron_user_hdfs_dir,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.metron_group,
+                                   mode=0755,
+                                   )
+
     def start_rest_application(self):
         """
         Start the REST application
@@ -125,6 +161,7 @@ class RestCommands:
           "export METRON_INDEX_CP={metron_indexing_classpath};"
           "export METRON_LOG_DIR={metron_log_dir};"
           "export METRON_PID_FILE={pid_file};"
+          "export HDP_VERSION={hdp_version};"
           "{metron_home}/bin/metron-rest.sh;"
           "unset METRON_JDBC_PASSWORD;"
         ))

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
index 2f2d3f9..c842214 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
@@ -51,6 +51,10 @@ class RestMaster(Script):
             commands.init_kafka_topics()
         if not commands.is_hbase_configured():
             commands.create_hbase_tables()
+        if not commands.is_pcap_configured():
+            commands.init_pcap()
+        if not commands.is_metron_user_hdfs_dir_configured():
+            commands.create_metron_user_hdfs_dir()
         if params.security_enabled and not commands.is_hbase_acl_configured():
             commands.set_hbase_acls()
         if params.security_enabled and not commands.is_kafka_acl_configured():

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java
new file mode 100644
index 0000000..1d33675
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/PcapResponse.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class PcapResponse {
+
+  private List<byte[]> pcaps = new ArrayList<>();
+
+  public List<byte[]> getPcaps(){
+    if(pcaps == null) {
+      return new ArrayList<>();
+    } else {
+      return pcaps;
+    }
+  }
+
+  public void setPcaps(List<byte[]> pcaps) {
+    this.pcaps = pcaps;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
new file mode 100644
index 0000000..758340b
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model.pcap;
+
+public class FixedPcapRequest extends PcapRequest {
+
+  private String ipSrcAddr;
+  private String ipDstAddr;
+  private Integer ipSrcPort;
+  private Integer ipDstPort;
+  private String protocol;
+  private String packetFilter;
+  private Boolean includeReverse = false;
+
+  public String getIpSrcAddr() {
+    return ipSrcAddr;
+  }
+
+  public void setIpSrcAddr(String ipSrcAddr) {
+    this.ipSrcAddr = ipSrcAddr;
+  }
+
+  public String getIpDstAddr() {
+    return ipDstAddr;
+  }
+
+  public void setIpDstAddr(String ipDstAddr) {
+    this.ipDstAddr = ipDstAddr;
+  }
+
+  public Integer getIpSrcPort() {
+    return ipSrcPort;
+  }
+
+  public void setIpSrcPort(Integer ipSrcPort) {
+    this.ipSrcPort = ipSrcPort;
+  }
+
+  public Integer getIpDstPort() {
+    return ipDstPort;
+  }
+
+  public void setIpDstPort(Integer ipDstPort) {
+    this.ipDstPort = ipDstPort;
+  }
+
+  public String getProtocol() {
+    return protocol;
+  }
+
+  public void setProtocol(String protocol) {
+    this.protocol = protocol;
+  }
+
+  public String getPacketFilter() {
+    return packetFilter;
+  }
+
+  public void setPacketFilter(String packetFilter) {
+    this.packetFilter = packetFilter;
+  }
+
+  public Boolean getIncludeReverse() {
+    return includeReverse;
+  }
+
+  public void setIncludeReverse(Boolean includeReverse) {
+    this.includeReverse = includeReverse;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    FixedPcapRequest fixedPcapRequest = (FixedPcapRequest) o;
+
+    return (super.equals(o)) &&
+            (getIpSrcAddr() != null ? getIpSrcAddr().equals(fixedPcapRequest.getIpSrcAddr()) : fixedPcapRequest.getIpSrcAddr() != null) &&
+            (getIpDstAddr() != null ? getIpDstAddr().equals(fixedPcapRequest.getIpDstAddr()) : fixedPcapRequest.getIpDstAddr() != null) &&
+            (getIpSrcPort() != null ? getIpSrcPort().equals(fixedPcapRequest.getIpSrcPort()) : fixedPcapRequest.getIpSrcPort() != null) &&
+            (getIpDstPort() != null ? getIpDstPort().equals(fixedPcapRequest.getIpDstPort()) : fixedPcapRequest.getIpDstPort() != null) &&
+            (getProtocol() != null ? getProtocol().equals(fixedPcapRequest.getProtocol()) : fixedPcapRequest.getProtocol() != null) &&
+            (getPacketFilter() != null ? getPacketFilter().equals(fixedPcapRequest.getPacketFilter()) : fixedPcapRequest.getPacketFilter() != null) &&
+            (getIncludeReverse() != null ? getIncludeReverse().equals(fixedPcapRequest.getIncludeReverse()) : fixedPcapRequest.getIncludeReverse() != null);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = super.hashCode();
+    result = 31 * result + (getIpSrcAddr() != null ? getIpSrcAddr().hashCode() : 0);
+    result = 31 * result + (getIpDstAddr() != null ? getIpDstAddr().hashCode() : 0);
+    result = 31 * result + (getIpSrcPort() != null ? getIpSrcPort().hashCode() : 0);
+    result = 31 * result + (getIpDstPort() != null ? getIpDstPort().hashCode() : 0);
+    result = 31 * result + (getProtocol() != null ? getProtocol().hashCode() : 0);
+    result = 31 * result + (getPacketFilter() != null ? getPacketFilter().hashCode() : 0);
+    result = 31 * result + (getIncludeReverse() != null ? getIncludeReverse().hashCode() : 0);
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
new file mode 100644
index 0000000..6062f94
--- /dev/null
+++ b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.model.pcap;
+
+public class PcapRequest {
+
+  private String baseOutputPath;
+  private String basePath;
+  private Long startTime = 0L;
+  private Long endTime = System.currentTimeMillis();
+  private Integer numReducers = 1;
+
+  public String getBaseOutputPath() {
+    return baseOutputPath;
+  }
+
+  public void setBaseOutputPath(String baseOutputPath) {
+    this.baseOutputPath = baseOutputPath;
+  }
+
+  public String getBasePath() {
+    return basePath;
+  }
+
+  public void setBasePath(String basePath) {
+    this.basePath = basePath;
+  }
+
+  public Long getStartTime() {
+    return startTime;
+  }
+
+  public void setStartTime(Long startTime) {
+    this.startTime = startTime;
+  }
+
+  public Long getEndTime() {
+    return endTime;
+  }
+
+  public void setEndTime(Long endTime) {
+    this.endTime = endTime;
+  }
+
+  public Integer getNumReducers() {
+    return numReducers;
+  }
+
+  public void setNumReducers(Integer numReducers) {
+    this.numReducers = numReducers;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (o == null || getClass() != o.getClass()) return false;
+
+    PcapRequest pcapRequest = (PcapRequest) o;
+
+    return (getBaseOutputPath() != null ? getBaseOutputPath().equals(pcapRequest.getBaseOutputPath()) : pcapRequest.getBaseOutputPath() != null) &&
+            (getBasePath() != null ? getBasePath().equals(pcapRequest.getBasePath()) : pcapRequest.getBasePath() == null) &&
+            (getStartTime() != null ? getStartTime().equals(pcapRequest.getStartTime()) : pcapRequest.getStartTime() == null) &&
+            (getEndTime() != null ? getEndTime().equals(pcapRequest.getEndTime()) : pcapRequest.getEndTime() == null) &&
+            (getNumReducers() != null ? getNumReducers().equals(pcapRequest.getNumReducers()) : pcapRequest.getNumReducers() == null);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = getBaseOutputPath() != null ? getBaseOutputPath().hashCode() : 0;
+    result = 31 * result + (getBasePath() != null ? getBasePath().hashCode() : 0);
+    result = 31 * result + (getStartTime() != null ? getStartTime().hashCode() : 0);
+    result = 31 * result + (getEndTime() != null ? getEndTime().hashCode() : 0);
+    result = 31 * result + (getNumReducers() != null ? getNumReducers().hashCode() : 0);
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 2a6a0e0..016c6f9 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -241,6 +241,7 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
 | [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)|
 | [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
 | [ `GET /api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
+| [ `GET /api/v1/pcap/fixed`](#get-apiv1pcapfixed)|
 | [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
 | [ `POST /api/v1/search/search`](#get-apiv1searchsearch)|
 | [ `POST /api/v1/search/group`](#get-apiv1searchgroup)|
@@ -485,6 +486,13 @@ Request and Response objects are JSON formatted.  The JSON schemas are available
   * Returns:
     * 200 - Returns 'true' if the status changed and 'false' if it did not.
 
+### `POST /api/v1/pcap/fixed`
+  * Description: Executes a Fixed Pcap Query.
+  * Input:
+    * fixedPcapRequest - A Fixed Pcap Request which includes fixed filter fields like ip source address and protocol.
+  * Returns:
+    * 200 - Returns a PcapResponse containing an array of pcaps.
+
 ### `POST /api/v1/search/search`
   * Description: Searches the indexing store. GUIDs must be quoted to ensure correct results.
   * Input:

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/pom.xml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml
index 9c9c372..6e9553e 100644
--- a/metron-interface/metron-rest/pom.xml
+++ b/metron-interface/metron-rest/pom.xml
@@ -170,6 +170,28 @@
                 </exclusion>
             </exclusions>
         </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-yarn-client</artifactId>
+        <version>${global_hadoop_version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+        <version>${global_hadoop_version}</version>
+        <exclusions>
+          <exclusion>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+          </exclusion>
+        </exclusions>
+      </dependency>
         <dependency>
             <groupId>org.apache.metron</groupId>
             <artifactId>metron-hbase-client</artifactId>
@@ -372,6 +394,17 @@
             <artifactId>org.eclipse.persistence.jpa</artifactId>
             <version>${eclipse.link.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.metron</groupId>
+            <artifactId>metron-pcap</artifactId>
+            <version>${project.parent.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index c4873f9..0989d12 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -74,4 +74,7 @@ public class MetronRestConstants {
   public static final String USER_SETTINGS_HBASE_CF_SPRING_PROPERTY = "user.settings.cf";
 
   public static final String LOGGING_SYSTEM_PROPERTY = "org.springframework.boot.logging.LoggingSystem";
+
+  public static final String PCAP_INPUT_PATH_SPRING_PROPERTY = "pcap.input.path";
+  public static final String PCAP_OUTPUT_PATH_SPRING_PROPERTY = "pcap.output.path";
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java
new file mode 100644
index 0000000..8da5f96
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/PcapConfig.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.config;
+
+import org.apache.metron.pcap.mr.PcapJob;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+
+@Configuration
+@Profile("!" + TEST_PROFILE)
+public class PcapConfig {
+
+  @Bean
+  public PcapJob pcapJob() {
+    return new PcapJob();
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
new file mode 100644
index 0000000..11ff1cd
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import io.swagger.annotations.ApiOperation;
+import io.swagger.annotations.ApiParam;
+import io.swagger.annotations.ApiResponse;
+import io.swagger.annotations.ApiResponses;
+import org.apache.metron.rest.model.PcapResponse;
+import org.apache.metron.rest.model.pcap.FixedPcapRequest;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.service.PcapService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestMethod;
+import org.springframework.web.bind.annotation.RestController;
+
+import java.io.IOException;
+
+@RestController
+@RequestMapping("/api/v1/pcap")
+public class PcapController {
+
+  @Autowired
+  private PcapService pcapQueryService;
+
+  @ApiOperation(value = "Executes a Fixed Pcap Query.")
+  @ApiResponses(value = { @ApiResponse(message = "Returns a PcapResponse containing an array of pcaps.", code = 200)})
+  @RequestMapping(value = "/fixed", method = RequestMethod.POST)
+  ResponseEntity<PcapResponse> fixed(@ApiParam(name="fixedPcapRequest", value="A Fixed Pcap Request"
+          + " which includes fixed filter fields like ip source address and protocol.", required=true)@RequestBody FixedPcapRequest fixedPcapRequest) throws RestException {
+    PcapResponse pcapsResponse = pcapQueryService.fixed(fixedPcapRequest);
+    return new ResponseEntity<>(pcapsResponse, HttpStatus.OK);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
new file mode 100644
index 0000000..ce8372c
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service;
+
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.PcapResponse;
+import org.apache.metron.rest.model.pcap.FixedPcapRequest;
+
+public interface PcapService {
+
+  PcapResponse fixed(FixedPcapRequest fixedPcapRequest) throws RestException;
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
new file mode 100644
index 0000000..4dae1e5
--- /dev/null
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/PcapServiceImpl.java
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.hadoop.SequenceFileIterable;
+import org.apache.metron.common.utils.timestamp.TimestampConverters;
+import org.apache.metron.pcap.PcapHelper;
+import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
+import org.apache.metron.pcap.mr.PcapJob;
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.PcapResponse;
+import org.apache.metron.rest.model.pcap.FixedPcapRequest;
+import org.apache.metron.rest.service.PcapService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.env.Environment;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Service
+public class PcapServiceImpl implements PcapService {
+
+  private Environment environment;
+  private Configuration configuration;
+  private PcapJob pcapJob;
+
+  @Autowired
+  public PcapServiceImpl(Environment environment, Configuration configuration, PcapJob pcapJob) {
+    this.environment = environment;
+    this.configuration = configuration;
+    this.pcapJob = pcapJob;
+  }
+
+  @Override
+  public PcapResponse fixed(FixedPcapRequest fixedPcapRequest) throws RestException {
+    if (fixedPcapRequest.getBasePath() == null) {
+      fixedPcapRequest.setBasePath(environment.getProperty(MetronRestConstants.PCAP_INPUT_PATH_SPRING_PROPERTY));
+    }
+    if (fixedPcapRequest.getBaseOutputPath() == null) {
+      fixedPcapRequest.setBaseOutputPath(environment.getProperty(MetronRestConstants.PCAP_OUTPUT_PATH_SPRING_PROPERTY));
+    }
+    PcapResponse response = new PcapResponse();
+    SequenceFileIterable results;
+    try {
+      results = pcapJob.query(
+              new Path(fixedPcapRequest.getBasePath()),
+              new Path(fixedPcapRequest.getBaseOutputPath()),
+              TimestampConverters.MILLISECONDS.toNanoseconds(fixedPcapRequest.getStartTime()),
+              TimestampConverters.MILLISECONDS.toNanoseconds(fixedPcapRequest.getEndTime()),
+              fixedPcapRequest.getNumReducers(),
+              getFixedFields(fixedPcapRequest),
+              configuration,
+              getFileSystem(),
+              new FixedPcapFilter.Configurator()
+      );
+      if (results != null) {
+        List<byte[]> pcaps = new ArrayList<>();
+        results.iterator().forEachRemaining(pcaps::add);
+        response.setPcaps(pcaps);
+      }
+    } catch (IOException | ClassNotFoundException | InterruptedException e) {
+      throw new RestException(e);
+    }
+    return response;
+  }
+
+  protected Map<String, String> getFixedFields(FixedPcapRequest fixedPcapRequest) {
+    Map<String, String> fixedFields = new HashMap<>();
+    if (fixedPcapRequest.getIpSrcAddr() != null) {
+      fixedFields.put(Constants.Fields.SRC_ADDR.getName(), fixedPcapRequest.getIpSrcAddr());
+    }
+    if (fixedPcapRequest.getIpDstAddr() != null) {
+      fixedFields.put(Constants.Fields.DST_ADDR.getName(), fixedPcapRequest.getIpDstAddr());
+    }
+    if (fixedPcapRequest.getIpSrcPort() != null) {
+      fixedFields.put(Constants.Fields.SRC_PORT.getName(), fixedPcapRequest.getIpSrcPort().toString());
+    }
+    if (fixedPcapRequest.getIpDstPort() != null) {
+      fixedFields.put(Constants.Fields.DST_PORT.getName(), fixedPcapRequest.getIpDstPort().toString());
+    }
+    if (fixedPcapRequest.getProtocol() != null) {
+      fixedFields.put(Constants.Fields.PROTOCOL.getName(), fixedPcapRequest.getProtocol());
+    }
+    if (fixedPcapRequest.getIncludeReverse() != null) {
+      fixedFields.put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), fixedPcapRequest.getIncludeReverse().toString());
+    }
+    if (fixedPcapRequest.getPacketFilter() != null) {
+      fixedFields.put(PcapHelper.PacketFields.PACKET_FILTER.getName(), fixedPcapRequest.getPacketFilter());
+    }
+    return fixedFields;
+  }
+
+  protected FileSystem getFileSystem() throws IOException {
+    return FileSystem.get(configuration);
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/resources/application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application.yml b/metron-interface/metron-rest/src/main/resources/application.yml
index 97359b9..df67ab3 100644
--- a/metron-interface/metron-rest/src/main/resources/application.yml
+++ b/metron-interface/metron-rest/src/main/resources/application.yml
@@ -71,4 +71,8 @@ spring.jpa.generate-ddl: true
 user:
   settings:
     table: user_settings
-    cf: cf
\ No newline at end of file
+    cf: cf
+
+pcap:
+  input.path: /apps/metron/pcap
+  output.path: /tmp
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
index 2c7c75b..cf7bf58 100644
--- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
+++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh
@@ -110,7 +110,7 @@ fi
 echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}"
 
 echo "Starting application"
-${JAVA_HOME}/bin/java ${METRON_JVMFLAGS} \
+${JAVA_HOME}/bin/java -Dhdp.version=${HDP_VERSION} ${METRON_JVMFLAGS} \
 -cp ${METRON_REST_CLASSPATH} \
 org.apache.metron.rest.MetronRestApplication \
 ${METRON_SPRING_OPTIONS} >> ${METRON_LOG_DIR}/metron-rest.log 2>&1 & echo $! > ${METRON_PID_FILE};

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
index 008f3fc..a9e70d2 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/config/TestConfig.java
@@ -37,7 +37,9 @@ import org.apache.metron.integration.ComponentRunner;
 import org.apache.metron.integration.UnableToStartException;
 import org.apache.metron.integration.components.KafkaComponent;
 import org.apache.metron.integration.components.ZKServerComponent;
+import org.apache.metron.pcap.mr.PcapJob;
 import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.mock.MockPcapJob;
 import org.apache.metron.rest.mock.MockStormCLIClientWrapper;
 import org.apache.metron.rest.mock.MockStormRestTemplate;
 import org.apache.metron.rest.service.impl.StormCLIWrapper;
@@ -185,4 +187,9 @@ public class TestConfig {
   public UserSettingsClient userSettingsClient() throws RestException, IOException {
     return new UserSettingsClient(new MockHBaseTableProvider().addToCache("user_settings", "cf"), Bytes.toBytes("cf"));
   }
+
+  @Bean
+  public PcapJob mockPcapJob() {
+    return new MockPcapJob();
+  }
 }

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
new file mode 100644
index 0000000..5e4875a
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/PcapControllerIntegrationTest.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.controller;
+
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.utils.JSONUtils;
+import org.apache.metron.pcap.PcapHelper;
+import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
+import org.apache.metron.rest.mock.MockPcapJob;
+import org.apache.metron.rest.model.PcapResponse;
+import org.apache.metron.rest.service.PcapService;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.setup.MockMvcBuilders;
+import org.springframework.web.context.WebApplicationContext;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
+@RunWith(SpringRunner.class)
+@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
+@ActiveProfiles(TEST_PROFILE)
+public class PcapControllerIntegrationTest {
+
+  /**
+   {
+   "basePath": "/apps/metron/pcap",
+   "baseOutputPath": "/tmp",
+   "endTime": 10,
+   "includeReverse": "true",
+   "ipDstAddr": "192.168.1.1",
+   "ipDstPort": "1000",
+   "ipSrcAddr": "192.168.1.2",
+   "ipSrcPort": "2000",
+   "numReducers": 2,
+   "packetFilter": "filter",
+   "protocol": "TCP",
+   "startTime": 1
+   }
+   */
+  @Multiline
+  public static String fixedJson;
+
+  @Autowired
+  private PcapService pcapService;
+
+  @Autowired
+  private WebApplicationContext wac;
+
+  private MockMvc mockMvc;
+
+  private String pcapUrl = "/api/v1/pcap";
+  private String user = "user";
+  private String password = "password";
+
+  @Before
+  public void setup() throws Exception {
+    this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
+  }
+
+  @Test
+  public void testSecurity() throws Exception {
+    this.mockMvc.perform(post(pcapUrl + "/fixed").with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
+            .andExpect(status().isUnauthorized());
+  }
+
+  @Test
+  public void testFixed() throws Exception {
+    MockPcapJob mockPcapJob = (MockPcapJob) wac.getBean("mockPcapJob");
+    List<byte[]> results = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes());
+    mockPcapJob.setResults(results);
+
+    PcapResponse expectedReponse = new PcapResponse();
+    expectedReponse.setPcaps(results);
+    this.mockMvc.perform(post(pcapUrl + "/fixed").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(fixedJson))
+            .andExpect(status().isOk())
+            .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+            .andExpect(content().json(JSONUtils.INSTANCE.toJSON(expectedReponse, false)));
+
+    Assert.assertEquals("/apps/metron/pcap", mockPcapJob.getBasePath());
+    Assert.assertEquals("/tmp", mockPcapJob.getBaseOutputPath());
+    Assert.assertEquals(1, mockPcapJob.getStartTime());
+    Assert.assertEquals(10, mockPcapJob.getEndTime());
+    Assert.assertEquals(2, mockPcapJob.getNumReducers());
+    Assert.assertTrue(mockPcapJob.getFilterImpl() instanceof FixedPcapFilter.Configurator);
+    Map<String, String> actualFixedFields = mockPcapJob.getFixedFields();
+    Assert.assertEquals("192.168.1.2", actualFixedFields.get(Constants.Fields.SRC_ADDR.getName()));
+    Assert.assertEquals("2000", actualFixedFields.get(Constants.Fields.SRC_PORT.getName()));
+    Assert.assertEquals("192.168.1.1", actualFixedFields.get(Constants.Fields.DST_ADDR.getName()));
+    Assert.assertEquals("1000", actualFixedFields.get(Constants.Fields.DST_PORT.getName()));
+    Assert.assertEquals("true", actualFixedFields.get(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName()));
+    Assert.assertEquals("TCP", actualFixedFields.get(Constants.Fields.PROTOCOL.getName()));
+    Assert.assertEquals("filter", actualFixedFields.get(PcapHelper.PacketFields.PACKET_FILTER.getName()));
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
new file mode 100644
index 0000000..3aa9ce3
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockPcapJob.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.mock;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.hadoop.SequenceFileIterable;
+import org.apache.metron.pcap.filter.PcapFilterConfigurator;
+import org.apache.metron.pcap.mr.PcapJob;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class MockPcapJob extends PcapJob {
+
+  private String basePath;
+  private String baseOutputPath;
+  private long beginNS;
+  private long endNS;
+  private int numReducers;
+  private Map<String, String> fixedFields;
+  private PcapFilterConfigurator filterImpl;
+  private SequenceFileIterable sequenceFileIterable;
+
+  public MockPcapJob() {
+    sequenceFileIterable = mock(SequenceFileIterable.class);
+  }
+
+  @SuppressWarnings(value = "unchecked")
+  @Override
+  public <T> SequenceFileIterable query(Path basePath, Path baseOutputPath, long beginNS, long endNS, int numReducers, T fields, Configuration conf, FileSystem fs, PcapFilterConfigurator<T> filterImpl) throws IOException, ClassNotFoundException, InterruptedException {
+    this.basePath = basePath.toString();
+    this.baseOutputPath = baseOutputPath.toString();
+    this.beginNS = beginNS;
+    this.endNS = endNS;
+    this.numReducers = numReducers;
+    if (fields instanceof Map) {
+      this.fixedFields = (Map<String, String>) fields;
+    }
+    this.filterImpl = filterImpl;
+    return sequenceFileIterable;
+  }
+
+  public void setResults(List<byte[]> pcaps) {
+    when(sequenceFileIterable.iterator()).thenReturn(pcaps.iterator());
+  }
+
+  public String getBasePath() {
+    return basePath;
+  }
+
+  public String getBaseOutputPath() {
+    return baseOutputPath;
+  }
+
+  public long getStartTime() {
+    return beginNS / 1000000;
+  }
+
+  public long getEndTime() {
+    return endNS / 1000000;
+  }
+
+  public int getNumReducers() {
+    return numReducers;
+  }
+
+  public Map<String, String> getFixedFields() {
+    return fixedFields;
+  }
+
+  public PcapFilterConfigurator getFilterImpl() {
+    return filterImpl;
+  }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/165c0e71/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
new file mode 100644
index 0000000..b6f2ea8
--- /dev/null
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/PcapServiceImplTest.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.rest.service.impl;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.metron.common.Constants;
+import org.apache.metron.common.hadoop.SequenceFileIterable;
+import org.apache.metron.pcap.PcapHelper;
+import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
+import org.apache.metron.pcap.mr.PcapJob;
+import org.apache.metron.rest.MetronRestConstants;
+import org.apache.metron.rest.RestException;
+import org.apache.metron.rest.model.PcapResponse;
+import org.apache.metron.rest.model.pcap.FixedPcapRequest;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.springframework.core.env.Environment;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+@SuppressWarnings("ALL")
+public class PcapServiceImplTest {
+  @Rule
+  public final ExpectedException exception = ExpectedException.none();
+
+  Environment environment;
+  Configuration configuration;
+  PcapJob pcapJob;
+
+  @Before
+  public void setUp() throws Exception {
+    environment = mock(Environment.class);
+    pcapJob = mock(PcapJob.class);
+    configuration = mock(Configuration.class);
+
+    when(environment.getProperty(MetronRestConstants.PCAP_INPUT_PATH_SPRING_PROPERTY)).thenReturn("/input/path");
+    when(environment.getProperty(MetronRestConstants.PCAP_OUTPUT_PATH_SPRING_PROPERTY)).thenReturn("/output/path");
+  }
+
+
+  @Test
+  public void fixedShouldProperlyCallPcapJobQuery() throws Exception {
+    FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
+    fixedPcapRequest.setBaseOutputPath("baseOutputPath");
+    fixedPcapRequest.setBasePath("basePath");
+    fixedPcapRequest.setStartTime(1L);
+    fixedPcapRequest.setEndTime(2L);
+    fixedPcapRequest.setNumReducers(2);
+    fixedPcapRequest.setIpSrcAddr("ip_src_addr");
+    fixedPcapRequest.setIpDstAddr("ip_dst_addr");
+    fixedPcapRequest.setIpSrcPort(1000);
+    fixedPcapRequest.setIpDstPort(2000);
+    fixedPcapRequest.setProtocol("tcp");
+    fixedPcapRequest.setPacketFilter("filter");
+    fixedPcapRequest.setIncludeReverse(true);
+
+    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, pcapJob));
+    FileSystem fileSystem = mock(FileSystem.class);
+    doReturn(fileSystem).when(pcapService).getFileSystem();
+    Map<String, String> expectedFields = new HashMap<String, String>() {{
+      put(Constants.Fields.SRC_ADDR.getName(), "ip_src_addr");
+      put(Constants.Fields.DST_ADDR.getName(), "ip_dst_addr");
+      put(Constants.Fields.SRC_PORT.getName(), "1000");
+      put(Constants.Fields.DST_PORT.getName(), "2000");
+      put(Constants.Fields.PROTOCOL.getName(), "tcp");
+      put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true");
+      put(PcapHelper.PacketFields.PACKET_FILTER.getName(), "filter");
+    }};
+    List<byte[]> expectedPcaps = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes());
+    SequenceFileIterable results = mock(SequenceFileIterable.class);
+    when(results.iterator()).thenReturn(expectedPcaps.iterator());
+    when(pcapJob.query(eq(new Path("basePath")),
+            eq(new Path("baseOutputPath")),
+            eq(1000000L),
+            eq(2000000L),
+            eq(2),
+            eq(expectedFields),
+            eq(configuration),
+            any(FileSystem.class),
+            any(FixedPcapFilter.Configurator.class))).thenReturn(results);
+
+    PcapResponse pcapsResponse = pcapService.fixed(fixedPcapRequest);
+    Assert.assertEquals(expectedPcaps, pcapsResponse.getPcaps());
+  }
+
+  @Test
+  public void fixedShouldProperlyCallPcapJobQueryWithDefaults() throws Exception {
+    FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
+
+    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, pcapJob));
+    FileSystem fileSystem = mock(FileSystem.class);
+    doReturn(fileSystem).when(pcapService).getFileSystem();
+    Map<String, String> expectedFields = new HashMap<String, String>() {{
+      put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "false");
+    }};
+    List<byte[]> expectedPcaps = Arrays.asList("pcap1".getBytes(), "pcap2".getBytes());
+    SequenceFileIterable results = mock(SequenceFileIterable.class);
+    when(results.iterator()).thenReturn(expectedPcaps.iterator());
+    when(pcapJob.query(eq(new Path("/input/path")),
+            eq(new Path("/output/path")),
+            eq(0L),
+            eq(fixedPcapRequest.getEndTime() * 1000000),
+            eq(1),
+            eq(expectedFields),
+            eq(configuration),
+            any(FileSystem.class),
+            any(FixedPcapFilter.Configurator.class))).thenReturn(results);
+
+    PcapResponse pcapsResponse = pcapService.fixed(fixedPcapRequest);
+    Assert.assertEquals(expectedPcaps, pcapsResponse.getPcaps());
+  }
+
+  @Test
+  public void fixedShouldThrowRestException() throws Exception {
+    exception.expect(RestException.class);
+    exception.expectMessage("some exception");
+
+    FixedPcapRequest fixedPcapRequest = new FixedPcapRequest();
+
+    PcapServiceImpl pcapService = spy(new PcapServiceImpl(environment, configuration, pcapJob));
+    FileSystem fileSystem = mock(FileSystem.class);
+    doReturn(fileSystem).when(pcapService).getFileSystem();
+
+    when(pcapJob.query(any(),
+            any(),
+            eq(0L),
+            eq(fixedPcapRequest.getEndTime() * 1000000),
+            eq(1),
+            any(),
+            any(),
+            any(FileSystem.class),
+            any(FixedPcapFilter.Configurator.class))).thenThrow(new IOException("some exception"));
+
+    pcapService.fixed(fixedPcapRequest);
+  }
+}


Mime
View raw message