metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject metron git commit: METRON-1709 Add controls to start / stop the PCAP topology from Ambari. (MohanDV via nickwallen) closes apache/metron#1201
Date Thu, 20 Sep 2018 15:10:23 GMT
Repository: metron
Updated Branches:
  refs/heads/master 3372aa360 -> 15194c3b4


METRON-1709 Add controls to start / stop the PCAP topology from Ambari. (MohanDV via nickwallen)
closes apache/metron#1201


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/15194c3b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/15194c3b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/15194c3b

Branch: refs/heads/master
Commit: 15194c3b4e7d9d9d161b0ddfa556395b34cfef7c
Parents: 3372aa3
Author: MohanDV <mohan.dv@gmail.com>
Authored: Thu Sep 20 11:10:07 2018 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Thu Sep 20 11:10:07 2018 -0400

----------------------------------------------------------------------
 .../roles/ambari_config/vars/single_node_vm.yml |   3 +-
 .../roles/ambari_config/vars/small_cluster.yml  |   3 +-
 .../METRON/CURRENT/role_command_order.json      |   9 +-
 .../common-services/METRON/CURRENT/metainfo.xml |  39 ++++
 .../package/scripts/params/params_linux.py      |   4 +
 .../package/scripts/params/status_params.py     |   1 +
 .../CURRENT/package/scripts/pcap_commands.py    | 201 +++++++++++++++++++
 .../CURRENT/package/scripts/pcap_master.py      | 105 ++++++++++
 .../CURRENT/package/scripts/rest_commands.py    |  43 ----
 .../CURRENT/package/scripts/rest_master.py      |   7 -
 .../METRON/CURRENT/service_advisor.py           |   5 +
 11 files changed, 365 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index a3c643b..962726c 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -32,6 +32,7 @@ es_master: [ES_MASTER]
 kibana_master: [KIBANA_MASTER]
 metron_indexing: [METRON_INDEXING]
 metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
 metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
 metron_parsers : [METRON_PARSERS]
 metron_rest: [METRON_REST]
@@ -40,7 +41,7 @@ metron_alerts_ui: [METRON_ALERTS_UI]
 
 metron_components: >
   {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master)
| union(hadoop_slave) | union(zookeeper_slave) |
-  union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) |
union(metron_indexing) | union(metron_profiler) |
+  union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) |
union(metron_indexing) | union(metron_profiler) | union(metron_pcap) |
   union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui)
| union(metron_alerts_ui) | union(es_master)  }}
 
 cluster_name: "metron_cluster"

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
index 218e267..818b5f3 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
@@ -32,6 +32,7 @@ es_slave: [ES_SLAVE]
 kibana_master: [KIBANA_MASTER]
 metron_indexing: [METRON_INDEXING]
 metron_profiler: [METRON_PROFILER]
+metron_pcap: [METRON_PCAP]
 metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
 metron_parsers : [METRON_PARSERS]
 metron_rest: [METRON_REST]
@@ -45,7 +46,7 @@ master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_ma
 master_2_host:
   - "{{groups.ambari_slave[1]}}"
 metron_components: >
-    {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers)
| union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave)
| union(storm_slave) |
+    {{ metron_indexing | union(metron_profiler) | union(metron_pcap) |union(metron_enrichment_master)
| union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui)
| union(hadoop_slave) | union(storm_slave) |
     union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
 metron_host:
   - "{{ groups.metron[0] }}"

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
index fcb2206..ee6ca23 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
@@ -6,23 +6,26 @@
         "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
         "METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"],
         "METRON_PROFILER-INSTALL" : ["METRON_ENRICHMENT-INSTALL"],
+        "METRON_PCAP-INSTALL" : ["METRON_INDEXING-INSTALL"],
         "METRON_REST-INSTALL" : ["METRON_PARSERS-INSTALL"],
         "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START",
"STORM_REST_API-START" ,"METRON_ENRICHMENT_MASTER-START"],
         "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START",
"STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
         "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START",
"STORM_REST_API-START", "METRON_PARSERS-START"],
         "METRON_PROFILER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START",
"STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_ENRICHMENT-INSTALL"],
+        "METRON_PCAP-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START",
"STORM_REST_API-START"],
         "METRON_REST-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START",
"STORM_REST_API-START", "METRON_PARSERS-INSTALL","METRON_INDEXING-INSTALL","METRON_ENRICHMENT-INSTALL"],
         "METRON_MANAGEMENT_UI-START" : ["METRON_REST-START"],
         "METRON_ALERTS_UI-START" : ["METRON_REST-START"],
 
-        "STORM_REST_API-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP",
"METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP",
"METRON_ALERTS_UI-STOP"],
-        "STORM_UI_SERVER-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP",
"METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP",
"METRON_ALERTS_UI-STOP"],
+        "STORM_REST_API-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP",
"METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_PCAP-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP",
"METRON_ALERTS_UI-STOP"],
+        "STORM_UI_SERVER-STOP" : ["METRON_PARSERS-STOP", "METRON_ENRICHMENT_MASTER-STOP",
"METRON_INDEXING-STOP", "METRON_PROFILER-STOP", "METRON_PCAP-STOP", "METRON_REST-STOP", "METRON_MANAGEMENT_UI-STOP",
"METRON_ALERTS_UI-STOP"],
 
-        "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "METRON_ENRICHMENT_MASTER-START",
"METRON_INDEXING-START", "METRON_PROFILER-START", "METRON_REST-START", "METRON_MANAGEMENT_UI-START",
"METRON_ALERTS_UI-START"],
+        "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "METRON_ENRICHMENT_MASTER-START",
"METRON_INDEXING-START", "METRON_PROFILER-START", "METRON_PCAP-START", "METRON_REST-START",
"METRON_MANAGEMENT_UI-START", "METRON_ALERTS_UI-START"],
         "METRON_PARSERS_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK",
"ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK",
"STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
         "METRON_ENRICHMENT_MASTER_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START",
"NAMENODE_SERVICE_CHECK-SERVICE_CHECK", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK",
"STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK", "HBASE_MASTER_SERVICE_CHECK-SERVICE_CHECK",
"HBASE_REGIONSERVER_SERVICE_CHECK-SERVICE_CHECK"],
         "METRON_INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_INDEXING-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK",
"ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK",
"STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
         "METRON_PROFILER_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PROFILER-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK",
"ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK",
"STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK", "HBASE_MASTER_SERVICE_CHECK-SERVICE_CHECK",
"HBASE_REGIONSERVER_SERVICE_CHECK-SERVICE_CHECK"],
+        "METRON_PCAP_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PCAP-START", "NAMENODE_SERVICE_CHECK-SERVICE_CHECK",
"ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK", "KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK",
"STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
         "METRON_REST_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_REST-START", "ZOOKEEPER_SERVER_SERVICE_CHECK-SERVICE_CHECK",
"KAFKA_BROKER_SERVICE_CHECK-SERVICE_CHECK", "STORM_REST_API_SERVICE_CHECK-SERVICE_CHECK"],
         "METRON_MANAGEMENT_UI_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_MANAGEMENT_UI-START",
"METRON_REST_SERVICE_CHECK-SERVICE_CHECK"],
         "METRON_ALERTS_UI_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ALERTS_UI-START", "METRON_REST_SERVICE_CHECK-SERVICE_CHECK"]

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 644ba97..5965506 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -159,6 +159,45 @@
         </component>
 
         <component>
+          <name>METRON_PCAP</name>
+          <displayName>Metron PCAP</displayName>
+          <category>MASTER</category>
+          <cardinality>1</cardinality>
+          <versionAdvertised>false</versionAdvertised>
+          <dependencies>
+            <dependency>
+              <name>HDFS/HDFS_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
+              <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+              <scope>cluster</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
+              <name>KAFKA/KAFKA_BROKER</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
+          <commandScript>
+            <script>scripts/pcap_master.py</script>
+            <scriptType>PYTHON</scriptType>
+            <timeout>600</timeout>
+          </commandScript>
+          <configuration-dependencies>
+            <config-type>metron-rest-env</config-type>
+          </configuration-dependencies>
+        </component>
+
+        <component>
           <name>METRON_INDEXING</name>
           <displayName>Metron Indexing</displayName>
           <category>MASTER</category>

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 0525c7f..dbad44d 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -388,6 +388,8 @@ bolt_hdfs_rotation_policy_units = config['configurations']['metron-indexing-env'
 bolt_hdfs_rotation_policy_count = config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
 
 # Pcap
+metron_pcap_topology = 'pcap'
+pcap_input_topic = 'pcap'
 pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
 pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
 pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path']
@@ -396,6 +398,8 @@ pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue']
 pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size']
 pcap_configured_flag_file = status_params.pcap_configured_flag_file
 pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file
+pcap_acl_configured_flag_file = status_params.pcap_acl_configured_flag_file
+
 
 # MapReduce
 metron_user_hdfs_dir = '/user/' + metron_user

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 99f5ec0..2c711cf 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -121,6 +121,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt
 # Pcap
 pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured'
 pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured'
+pcap_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_acl_configured'
 
 # MapReduce
 metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured'
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
new file mode 100644
index 0000000..2190916
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_commands.py
@@ -0,0 +1,201 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from datetime import datetime
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+import metron_service
+import metron_security
+
+
+# Wrap major operations and functionality in this class
+class PcapCommands:
+    __params = None
+    __configured = False
+    __acl_configured = False
+    __pcap_topology = None
+    __pcap_perm_configured = False
+    __pcap_topic = None
+
+    def __init__(self, params):
+        if params is None:
+            raise ValueError("params argument is required for initialization")
+        self.__params = params
+        self.__configured = os.path.isfile(self.__params.pcap_configured_flag_file)
+        self.__acl_configured = os.path.isfile(self.__params.pcap_acl_configured_flag_file)
+        self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file)
+        self.__pcap_topology = params.metron_pcap_topology
+        self.__pcap_topic = params.pcap_input_topic
+
+    def __get_topics(self):
+        return [self.__pcap_topic]
+
+    def __get_kafka_acl_groups(self):
+        return ['pcap']
+
+    def is_configured(self):
+        return self.__configured
+
+    def is_acl_configured(self):
+        return self.__acl_configured
+
+    def is_hdfs_perm_configured(self):
+        return self.__pcap_perm_configured
+
+    def set_configured(self):
+        metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file,
"Setting PCAP configured to True")
+
+    def set_acl_configured(self):
+        metron_service.set_configured(self.__params.metron_user, self.__params.pcap_acl_configured_flag_file,
"Setting PCAP ACL configured to true")
+
+    def set_hdfs_perm_configured(self):
+        metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file,
"Setting PCAP HDFS perm configured to True")
+
+    def init_pcap(self):
+        self.init_kafka_topics()
+        self.init_hdfs_dir()
+        Logger.info("Done initializing PCAP configuration")
+
+    def init_hdfs_dir(self):
+        Logger.info("Creating HDFS locations for PCAP")
+        # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group.
+        # Kerberized Metron runs under it's own user.
+        ownership = 0755 if self.__params.security_enabled else 0775
+        self.__params.HdfsResource(self.__params.pcap_base_path,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.hadoop_group,
+                                   mode=ownership,
+                                   )
+        self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.hadoop_group,
+                                   mode=ownership,
+                                   )
+        self.__params.HdfsResource(self.__params.pcap_final_output_path,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.hadoop_group,
+                                   mode=ownership,
+                                   )
+
+    def init_kafka_topics(self):
+        Logger.info('Creating Kafka topic for PCAP')
+        metron_service.init_kafka_topics(self.__params, self.__get_topics())
+
+    def init_kafka_acls(self):
+        Logger.info('Creating Kafka ACLs for PCAP')
+        metron_service.init_kafka_acls(self.__params, self.__get_topics())
+        metron_service.init_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
+
+    def is_topology_active(self, env):
+        env.set_params(self.__params)
+        active = True
+        topologies = metron_service.get_running_topologies(self.__params)
+        is_running = False
+        if self.__pcap_topology in topologies:
+            is_running = topologies[self.__pcap_topology] in ['ACTIVE', 'REBALANCING']
+        active &= is_running
+        return active
+
+    def start_pcap_topology(self, env):
+        Logger.info('Starting Metron PCAP topology')
+        start_cmd_template = """{0}/bin/start_pcap_topology.sh"""
+        if not self.is_topology_active(env):
+            if self.__params.security_enabled:
+                metron_security.kinit(self.__params.kinit_path_local,
+                                      self.__params.metron_keytab_path,
+                                      self.__params.metron_principal_name,
+                                      execute_user=self.__params.metron_user)
+            start_cmd = start_cmd_template.format(self.__params.metron_home)
+            Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
+        else :
+            Logger.info('PCAP topology already started')
+
+        Logger.info('Finished starting pcap topologies')
+
+    def stop_pcap_topology(self, env):
+        Logger.info('Stopping Metron PCAP topology')
+        if self.is_topology_active(env):
+            if self.__params.security_enabled:
+                metron_security.kinit(self.__params.kinit_path_local,
+                                      self.__params.metron_keytab_path,
+                                      self.__params.metron_principal_name,
+                                      execute_user=self.__params.metron_user)
+            stop_cmd = 'storm kill ' + self.__pcap_topology
+            Execute(stop_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
+
+        else :
+            Logger.info('PCAP topology already stopped')
+
+        Logger.info('Finished starting PCAP topologies')
+
+    def restart_pcap_topology(self, env):
+        Logger.info('Restarting the PCAP topology')
+        self.stop_pcap_topology(env)
+
+        # Wait for old topology to be cleaned up by Storm, before starting again.
+        retries = 0
+        topology_active = self.is_topology_active(env)
+        while self.is_topology_active(env) and retries < 3:
+            Logger.info('Existing topology still active. Will wait and retry')
+            time.sleep(10)
+            retries += 1
+
+        if not topology_active:
+            Logger.info('Waiting for storm kill to complete')
+            time.sleep(30)
+            self.start_pcap_topology(env)
+            Logger.info('Done restarting the PCAP topologies')
+        else:
+            Logger.warning('Retries exhausted. Existing topology not cleaned up.  Aborting
topology start.')
+
+    def service_check(self, env):
+        """
+        Performs a service check for the PCAP.
+        :param env: Environment
+        """
+        Logger.info('Checking Kafka topic for PCAP')
+        metron_service.check_kafka_topics(self.__params, self.__get_topics())
+
+        Logger.info("Checking for PCAP sequence files directory in HDFS for PCAP")
+        metron_service.check_hdfs_dir_exists(self.__params, self.__params.hdfs_pcap_sequencefiles_dir)
+
+        if self.__params.security_enabled:
+            Logger.info('Checking Kafka ACLs for PCAP')
+            metron_service.check_kafka_acls(self.__params, self.__get_topics())
+            metron_service.check_kafka_acl_groups(self.__params, self.__get_kafka_acl_groups())
+
+        Logger.info("Checking for PCAP topologies")
+        if not self.is_topology_active(env):
+            raise Fail("PCAP topologies not running")
+
+        Logger.info("PCAP service check completed successfully")

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
new file mode 100644
index 0000000..a3bc1b4
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/pcap_master.py
@@ -0,0 +1,105 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import File
+from resource_management.core.source import Template
+from resource_management.libraries.functions.format import format
+from resource_management.core.source import StaticFile
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.script import Script
+
+from metron_security import storm_security_setup
+import metron_service
+import metron_security
+from pcap_commands import PcapCommands
+
+
+class Pcap(Script):
+    __configured = False
+
+    def install(self, env):
+        from params import params
+        env.set_params(params)
+        self.install_packages(env)
+
+    def configure(self, env, upgrade_type=None, config_dir=None):
+        from params import params
+        env.set_params(params)
+
+        if not metron_service.is_zk_configured(params):
+            metron_service.init_zk_config(params)
+            metron_service.set_zk_configured(params)
+        metron_service.refresh_configs(params)
+
+        commands = PcapCommands(params)
+
+        if not commands.is_configured():
+            commands.init_kafka_topics()
+            commands.init_hdfs_dir()
+        if params.security_enabled and not commands.is_acl_configured():
+            commands.init_kafka_acls()
+            commands.set_acl_configured()
+        Logger.info("Calling security setup")
+        storm_security_setup(params)
+        if not commands.is_configured():
+            commands.set_configured()
+
+    def start(self, env, upgrade_type=None):
+        from params import params
+        env.set_params(params)
+        self.configure(env)
+        commands = PcapCommands(params)
+        if params.security_enabled:
+            metron_security.kinit(params.kinit_path_local,
+                                  params.metron_keytab_path,
+                                  params.metron_principal_name,
+                                  execute_user=params.metron_user)
+
+        if params.security_enabled and not commands.is_hdfs_perm_configured():
+            commands.init_hdfs_dir()
+            commands.set_hdfs_perm_configured()
+        if params.security_enabled and not commands.is_acl_configured():
+            commands.init_kafka_acls()
+            commands.set_acl_configured()
+
+        commands.start_pcap_topology(env)
+
+    def stop(self, env, upgrade_type=None):
+        from params import params
+        env.set_params(params)
+        commands = PcapCommands(params)
+        commands.stop_pcap_topology(env)
+
+    def status(self, env):
+        from params import status_params
+        env.set_params(status_params)
+        commands = PcapCommands(status_params)
+        if not commands.is_topology_active(env):
+            raise ComponentIsNotRunning()
+
+    def restart(self, env):
+        from params import params
+        env.set_params(params)
+        self.configure(env)
+        commands = PcapCommands(params)
+        commands.restart_pcap_topology(env)
+
+if __name__ == "__main__":
+    Pcap().execute()

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
index d44f478..c410b94 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
@@ -37,8 +37,6 @@ class RestCommands:
     __kafka_acl_configured = False
     __hbase_configured = False
     __hbase_acl_configured = False
-    __pcap_configured = False
-    __pcap_perm_configured = False
     __metron_user_hdfs_dir_configured = False
 
     def __init__(self, params):
@@ -49,8 +47,6 @@ class RestCommands:
         self.__kafka_acl_configured = os.path.isfile(self.__params.rest_kafka_acl_configured_flag_file)
         self.__hbase_configured = os.path.isfile(self.__params.rest_hbase_configured_flag_file)
         self.__hbase_acl_configured = os.path.isfile(self.__params.rest_hbase_acl_configured_flag_file)
-        self.__pcap_configured = os.path.isfile(self.__params.pcap_configured_flag_file)
-        self.__pcap_perm_configured = os.path.isfile(self.__params.pcap_perm_configured_flag_file)
         self.__metron_user_hdfs_dir_configured = os.path.isfile(self.__params.metron_user_hdfs_dir_configured_flag_file)
         Directory(params.metron_rest_pid_dir,
                   mode=0755,
@@ -80,12 +76,6 @@ class RestCommands:
     def is_hbase_acl_configured(self):
         return self.__hbase_acl_configured
 
-    def is_pcap_configured(self):
-        return self.__pcap_configured
-
-    def is_pcap_perm_configured(self):
-        return self.__pcap_perm_configured
-
     def is_metron_user_hdfs_dir_configured(self):
         return self.__metron_user_hdfs_dir_configured
 
@@ -101,12 +91,6 @@ class RestCommands:
     def set_hbase_acl_configured(self):
         metron_service.set_configured(self.__params.metron_user, self.__params.rest_hbase_acl_configured_flag_file,
"Setting HBase ACL configured to True for rest")
 
-    def set_pcap_configured(self):
-        metron_service.set_configured(self.__params.metron_user, self.__params.pcap_configured_flag_file,
"Setting Pcap configured to True")
-
-    def set_pcap_perm_configured(self):
-        metron_service.set_configured(self.__params.metron_user, self.__params.pcap_perm_configured_flag_file,
"Setting Pcap perm configured to True")
-
     def set_metron_user_hdfs_dir_configured(self):
         metron_service.set_configured(self.__params.metron_user, self.__params.metron_user_hdfs_dir_configured_flag_file,
"Setting Metron user HDFS directory configured to True")
 
@@ -124,33 +108,6 @@ class RestCommands:
         groups = ['metron-rest']
         metron_service.init_kafka_acl_groups(self.__params, groups)
 
-    def init_pcap(self):
-        Logger.info("Creating HDFS locations for Pcap")
-        # Non Kerberized Metron runs under 'storm', requiring write under the 'hadoop' group.
-        # Kerberized Metron runs under it's own user.
-        ownership = 0755 if self.__params.security_enabled else 0775
-        self.__params.HdfsResource(self.__params.pcap_base_path,
-                                   type="directory",
-                                   action="create_on_execute",
-                                   owner=self.__params.metron_user,
-                                   group=self.__params.hadoop_group,
-                                   mode=ownership,
-                                   )
-        self.__params.HdfsResource(self.__params.pcap_base_interim_result_path,
-                                   type="directory",
-                                   action="create_on_execute",
-                                   owner=self.__params.metron_user,
-                                   group=self.__params.hadoop_group,
-                                   mode=ownership,
-                                   )
-        self.__params.HdfsResource(self.__params.pcap_final_output_path,
-                                   type="directory",
-                                   action="create_on_execute",
-                                   owner=self.__params.metron_user,
-                                   group=self.__params.hadoop_group,
-                                   mode=ownership,
-                                   )
-
     def create_metron_user_hdfs_dir(self):
         Logger.info("Creating HDFS location for Metron user")
         self.__params.HdfsResource(self.__params.metron_user_hdfs_dir,

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
index 791ca77..43224ad 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_master.py
@@ -51,8 +51,6 @@ class RestMaster(Script):
             commands.init_kafka_topics()
         if not commands.is_hbase_configured():
             commands.create_hbase_tables()
-        if not commands.is_pcap_configured():
-            commands.init_pcap()
         if not commands.is_metron_user_hdfs_dir_configured():
             commands.create_metron_user_hdfs_dir()
         if params.security_enabled and not commands.is_hbase_acl_configured():
@@ -60,11 +58,6 @@ class RestMaster(Script):
         if params.security_enabled and not commands.is_kafka_acl_configured():
             commands.init_kafka_acls()
             commands.set_kafka_acl_configured()
-        if params.security_enabled and not commands.is_pcap_perm_configured():
-            # If we Kerberize the cluster, we need to call this again, to remove write perms
from hadoop group
-            # If we start off Kerberized, it just does the same thing twice.
-            commands.init_pcap()
-            commands.set_pcap_perm_configured()
 
     def start(self, env, upgrade_type=None):
         from params import params

http://git-wip-us.apache.org/repos/asf/metron/blob/15194c3b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index 3bf5c7c..b008fc0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -44,6 +44,7 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
         metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
         metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
         metronProfilerHost = self.getHosts(componentsList, "METRON_PROFILER")[0]
+        metronPcapHost = self.getHosts(componentsList, "METRON_PCAP")[0]
         metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
         metronRESTHost = self.getHosts(componentsList, "METRON_REST")[0]
         metronManagementUIHost = self.getHosts(componentsList, "METRON_MANAGEMENT_UI")[0]
@@ -83,6 +84,10 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
             message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost)
             items.append({ "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'METRON_INDEXING', "host": metronIndexingHost })
 
+        if metronParsersHost != metronPcapHost:
+            message = "Metron PCAP must be co-located with Metron Parsers on {0}".format(metronParsersHost)
+            items.append({ "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'METRON_PCAP', "host": metronPcapHost })
+
         if metronParsersHost != metronProfilerHost:
             message = "Metron Profiler must be co-located with Metron Parsers on {0}".format(metronParsersHost)
             items.append({ "type": 'host-component', "level": 'ERROR', "message": message,
"component-name": 'METRON_PROFILER', "host": metronProfilerHost })


Mime
View raw message