metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject [01/50] [abbrv] metron git commit: METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124
Date Fri, 31 Aug 2018 19:33:20 GMT
Repository: metron
Updated Branches:
  refs/remotes/apache/feature/METRON-1699-create-batch-profiler 113ac0d3a -> f5eaef3c2


METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e78bc530
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e78bc530
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e78bc530

Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: e78bc530260aa622e8263cda2b94be99194dffb5
Parents: 8763fba
Author: merrimanr <merrimanr@gmail.com>
Authored: Mon Jul 23 09:08:12 2018 -0500
Committer: rmerriman <merrimanr@gmail.com>
Committed: Mon Jul 23 09:08:12 2018 -0500

----------------------------------------------------------------------
 .../CURRENT/configuration/metron-rest-env.xml   | 25 ++++++++++++++++++++
 .../package/scripts/params/params_linux.py      |  5 +++-
 .../CURRENT/package/scripts/rest_commands.py    | 18 ++++++++++++--
 .../METRON/CURRENT/package/templates/metron.j2  |  4 ++++
 .../METRON/CURRENT/themes/metron_theme.json     | 10 ++++++++
 .../src/main/config/rest_application.yml        |  6 ++++-
 .../src/main/config/pcap.properties             |  2 +-
 7 files changed, 65 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 066b527..20f9767 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,4 +168,29 @@
         <description>The field name where the threat triage score can be found in the
search indices. This setting primarily affects the Alerts UI.</description>
         <value>threat:triage:score</value>
     </property>
+    <property>
+        <name>pcap_base_path</name>
+        <display-name>Pcap Base Path</display-name>
+        <description>The HDFS path where pcap files are stored.  These files are the
input to pcap queries.</description>
+        <value>/apps/metron/pcap/input</value>
+    </property>
+    <property>
+        <name>pcap_base_interim_result_path</name>
+        <display-name>Pcap Base Interim Result Path</display-name>
+        <description>The HDFS path where interim results of a pcap query are stored,
before they are separated into pages.</description>
+        <value>/apps/metron/pcap/interim</value>
+    </property>
+    <property>
+        <name>pcap_final_output_path</name>
+        <display-name>Pcap Final Output Path</display-name>
+        <description>The HDFS path where the final results of a pcap query are stored.</description>
+        <value>/apps/metron/pcap/output</value>
+    </property>
+    <property>
+        <name>pcap_page_size</name>
+        <display-name>Pcap Page Size</display-name>
+        <description>The number of pcaps written to a page/file as a result of a pcap
query.</description>
+        <value>10</value>
+    </property>
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index b8d6123..73d3469 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -386,7 +386,10 @@ bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env'
 bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
 
 # Pcap
-pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap")
+pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
+pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
+pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
 pcap_configured_flag_file = status_params.pcap_configured_flag_file
 
 # MapReduce

http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index 2fd3f0d..463dca1 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -117,8 +117,22 @@ class RestCommands:
         metron_service.init_kafka_acl_groups(self.__params, groups)
 
     def init_pcap(self):
-        Logger.info("Creating HDFS location for Pcap")
-        self.__params.HdfsResource(self.__params.pcap_hdfs_dir,
+        Logger.info("Creating HDFS locations for Pcap")
+        self.__params.HdfsResource(self.__params.pcap_base_path,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.metron_group,
+                                   mode=0755,
+                                   )
+        self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.metron_group,
+                                   mode=0755,
+                                   )
+        self.__params.HdfsResource(self.__params.pcap_final_output_path,
                                    type="directory",
                                    action="create_on_execute",
                                    owner=self.__params.metron_user,

http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
index 049b6ee..278d6f8 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2
@@ -40,3 +40,7 @@ SECURITY_ENABLED={{security_enabled|lower}}
 KAFKA_SECURITY_PROTOCOL="{{kafka_security_protocol}}"
 PARSER_TOPOLOGY_OPTIONS="/home/{{metron_user}}/.storm/storm.config"
 METRON_ESCALATION_TOPIC="{{metron_escalation_topic}}"
+PCAP_BASE_PATH="{{pcap_base_path}}"
+PCAP_BASE_INTERIM_RESULT_PATH="{{pcap_base_interim_result_path}}"
+PCAP_FINAL_OUTPUT_PATH="{{pcap_final_output_path}}"
+PCAP_PAGE_SIZE="{{pcap_page_size}}"

http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 17b4460..9f5b04e 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -820,6 +820,10 @@
           "subsection-name": "subsection-rest"
         },
         {
+          "config": "metron-rest-env/pcap_page_size",
+          "subsection-name": "subsection-rest"
+        },
+        {
           "config": "metron-management-ui-env/metron_management_ui_port",
           "subsection-name": "subsection-management-ui"
         },
@@ -1421,6 +1425,12 @@
         }
       },
       {
+        "config": "metron-rest-env/pcap_page_size",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
         "config": "metron-management-ui-env/metron_management_ui_port",
         "widget": {
           "type": "text-field"

http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 3999393..7486112 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -55,4 +55,8 @@ kerberos:
   keytab: ${METRON_SERVICE_KEYTAB}
 
 pcap:
-  pdml.script.path: ${METRON_HOME}/bin/pcap_to_pdml.sh
\ No newline at end of file
+  pdml.script.path: ${METRON_HOME}/bin/pcap_to_pdml.sh
+  base.path: ${PCAP_BASE_PATH}
+  base.interim.result.path: ${PCAP_BASE_INTERIM_RESULT_PATH}
+  final.output.path: ${PCAP_FINAL_OUTPUT_PATH}
+  page.size: ${PCAP_PAGE_SIZE}

http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
index 7160178..848b588 100644
--- a/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
+++ b/metron-platform/metron-pcap-backend/src/main/config/pcap.properties
@@ -27,6 +27,6 @@ kafka.pcap.start=UNCOMMITTED_EARLIEST
 kafka.pcap.numPackets=1000
 kafka.pcap.maxTimeMS=300000
 kafka.pcap.ts_scheme=FROM_KEY
-kafka.pcap.out=/apps/metron/pcap
+kafka.pcap.out=/apps/metron/pcap/input
 kafka.pcap.ts_granularity=MICROSECONDS
 kafka.spout.parallelism=1


Mime
View raw message