metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject incubator-metron git commit: METRON-383: Create Ambari Service Definition for Metron Parsers (mmiklavc via cestella) closes apache/incubator-metron#218
Date Tue, 06 Sep 2016 19:17:22 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master fc2dc3d6b -> 47bc9b6de


METRON-383: Create Ambari Service Definition for Metron Parsers (mmiklavc via cestella) closes
apache/incubator-metron#218


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/47bc9b6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/47bc9b6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/47bc9b6d

Branch: refs/heads/master
Commit: 47bc9b6defa3707b07ec1fa0123bbb72843a4b33
Parents: fc2dc3d
Author: mmiklavc <michael.miklavcic@gmail.com>
Authored: Tue Sep 6 12:17:11 2016 -0700
Committer: cstella <cestella@gmail.com>
Committed: Tue Sep 6 12:17:11 2016 -0700

----------------------------------------------------------------------
 metron-deployment/packaging/ambari/.gitignore   |   3 +
 .../0.2.0BETA/configuration/metron-parsers.xml  |  57 +++++
 .../PARSERS/0.2.0BETA/metainfo.xml              | 108 +++++++++
 .../0.2.0BETA/package/scripts/commands.py       | 235 +++++++++++++++++++
 .../package/scripts/params/__init__.py          |  18 ++
 .../0.2.0BETA/package/scripts/params/params.py  |  32 +++
 .../package/scripts/params/params_linux.py      | 114 +++++++++
 .../package/scripts/params/params_windows.py    |  20 ++
 .../package/scripts/params/status_params.py     |  22 ++
 .../0.2.0BETA/package/scripts/parser_master.py  |  77 ++++++
 .../0.2.0BETA/package/scripts/service_check.py  |  40 ++++
 .../package/templates/metron-global.json        |   1 +
 .../0.2.0BETA/quicklinks/quicklinks.json        |  28 +++
 .../PARSERS/0.2.0BETA/service_advisor.py        |  68 ++++++
 .../HDP/2.3/services/PARSERS/metainfo.xml       |  27 +++
 .../services/PARSERS/role_command_order.json    |  17 ++
 16 files changed, 867 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore
new file mode 100644
index 0000000..db8ded0
--- /dev/null
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -0,0 +1,3 @@
+archive.zip
+*.hash
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
new file mode 100755
index 0000000..71966ea
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/configuration/metron-parsers.xml
@@ -0,0 +1,57 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration supports_final="true">
+    <property>
+        <name>metron_home</name>
+        <value>/usr/metron/0.2.0BETA</value>
+        <description>Metron home directory</description>
+        <display-name>Metron home</display-name>
+    </property>
+    <property>
+        <name>parsers</name>
+        <value>bro,snort,yaf</value>
+        <description>Metron parsers to deploy</description>
+        <display-name>Metron parsers</display-name>
+    </property>
+    <property>
+        <name>metron_apps_hdfs_dir</name>
+        <value>/apps/metron</value>
+        <description>Metron apps HDFS dir</description>
+        <display-name>Metron apps HDFS dir</display-name>
+    </property>
+    <property>
+        <name>metron_zookeeper_config_dir</name>
+        <value>config/zookeeper</value>
+        <description>Metron Zookeeper config dir. Relative path to Metron home.</description>
+        <display-name>Metron Zookeeper config dir</display-name>
+    </property>
+    <property>
+        <name>metron_user</name>
+        <value>metron</value>
+        <property-type>USER</property-type>
+        <description>The user for Metron</description>
+        <display-name>Metron User</display-name>
+    </property>
+    <property>
+        <name>metron_group</name>
+        <value>metron</value>
+        <description>The group for Metron</description>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
new file mode 100755
index 0000000..f1e5ab7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/metainfo.xml
@@ -0,0 +1,108 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+    <schemaVersion>2.0</schemaVersion>
+    <services>
+        <service>
+            <name>PARSERS</name>
+            <displayName>Metron Parsers</displayName>
+            <comment>Parsers</comment>
+            <version>0.2.0BETA</version>
+
+            <components>
+                <component>
+                    <name>PARSER_MASTER</name>
+                    <displayName>Metron Parser Topologies</displayName>
+                    <category>MASTER</category>
+                    <cardinality>1</cardinality>
+                    <versionAdvertised>true</versionAdvertised>
+                    <dependencies>
+                        <dependency>
+                            <name>HDFS/HDFS_CLIENT</name>
+                            <scope>host</scope>
+                            <auto-deploy>
+                                <enabled>true</enabled>
+                            </auto-deploy>
+                        </dependency>
+                        <dependency>
+                            <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+                            <scope>cluster</scope>
+                            <auto-deploy>
+                                <enabled>true</enabled>
+                            </auto-deploy>
+                        </dependency>
+                        <dependency>
+                            <name>KAFKA/KAFKA_BROKER</name>
+                            <scope>cluster</scope>
+                            <auto-deploy>
+                                <enabled>true</enabled>
+                            </auto-deploy>
+                        </dependency>
+                    </dependencies>
+                    <commandScript>
+                        <script>scripts/parser_master.py</script>
+                        <scriptType>PYTHON</scriptType>
+                        <timeout>600</timeout>
+                    </commandScript>
+                    <configuration-dependencies>
+                        <config-type>metron-parsers</config-type>
+                    </configuration-dependencies>
+                </component>
+            </components>
+
+            <osSpecifics>
+                <osSpecific>
+                    <osFamily>any</osFamily>
+                    <packages>
+                        <package>
+                            <name>metron-common</name>
+                        </package>
+                        <package>
+                            <name>metron-parsers</name>
+                        </package>
+                    </packages>
+                </osSpecific>
+            </osSpecifics>
+
+            <commandScript>
+                <script>scripts/service_check.py</script>
+                <scriptType>PYTHON</scriptType>
+                <timeout>300</timeout>
+            </commandScript>
+
+            <requiredServices>
+                <service>HDFS</service>
+                <service>KAFKA</service>
+                <service>STORM</service>
+                <service>ZOOKEEPER</service>
+            </requiredServices>
+
+            <configuration-dependencies>
+                <config-type>metron-parsers</config-type>
+            </configuration-dependencies>
+
+            <quickLinksConfigurations>
+                <quickLinksConfiguration>
+                    <fileName>quicklinks.json</fileName>
+                    <default>true</default>
+                </quickLinksConfiguration>
+            </quickLinksConfigurations>
+
+        </service>
+    </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
new file mode 100755
index 0000000..6b33e55
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/commands.py
@@ -0,0 +1,235 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+import re
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import Template
+from resource_management.libraries.functions import format
+
+
+# Wrap major operations and functionality in this class
+class Commands:
+    __params = None
+    __parser_list = None
+    __configured = False
+
+    def __init__(self, params):
+        if params is None:
+            raise ValueError("params argument is required for initialization")
+        self.__params = params
+        self.__parser_list = self.__get_parsers(params)
+        self.__configured = os.path.isfile(self.__params.configured_flag_file)
+
+    # get list of parsers
+    def __get_parsers(self, params):
+        return params.parsers.replace(' ', '').split(',')
+
+    def is_configured(self):
+        return self.__configured
+
+    def set_configured(self):
+        File(self.__params.configured_flag_file,
+             content="",
+             owner=self.__params.metron_user,
+             mode=0775)
+
+    # Possible storm topology status states
+    # http://storm.apache.org/releases/0.10.0/javadocs/backtype/storm/generated/TopologyStatus.html
+    class StormStatus:
+        ACTIVE = "ACTIVE"
+        INACTIVE = "INACTIVE"
+        KILLED = "KILLED"
+        REBALANCING = "REBALANCING"
+
+    def init_parsers(self):
+        Logger.info(
+            "Copying grok patterns from local directory '{}' to HDFS '{}'".format(self.__params.local_grok_patterns_dir,
+                                                                                  self.__params.metron_apps_dir))
+        self.__params.HdfsResource(self.__params.metron_apps_dir,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   mode=0775,
+                                   source=self.__params.local_grok_patterns_dir)
+
+        Logger.info("Creating global.json file")
+        File(self.__params.metron_zookeeper_config_path + '/global.json',
+             content=Template("metron-global.json"),
+             owner=self.__params.metron_user,
+             mode=0775)
+        Logger.info("Done initializing parser configuration")
+
+    def get_parser_list(self):
+        return self.__parser_list
+
+    def setup_repo(self):
+        def local_repo():
+            Logger.info("Setting up local repo")
+            Execute("yum -y install createrepo")
+            Execute("createrepo /localrepo")
+            Execute("chmod -R o-w+r /localrepo")
+            Execute("echo \"[METRON-0.2.0BETA]\n"
+                    "name=Metron 0.2.0BETA packages\n"
+                    "baseurl=file:///localrepo\n"
+                    "gpgcheck=0\n"
+                    "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+        def remote_repo():
+            print('Using remote repo')
+
+        yum_repo_types = {
+            'local': local_repo,
+            'remote': remote_repo
+        }
+        repo_type = self.__params.yum_repo_type
+        if repo_type in yum_repo_types:
+            yum_repo_types[repo_type]()
+        else:
+            raise ValueError("Unsupported repo type '{}'".format(repo_type))
+
+    def init_kafka_topics(self):
+        Logger.info('Creating Kafka topics')
+        command_template = """{}/kafka-topics.sh \
+                                --zookeeper {} \
+                                --create \
+                                --topic {} \
+                                --partitions {} \
+                                --replication-factor {} \
+                                --config retention.bytes={}"""
+        num_partitions = 1
+        replication_factor = 1
+        retention_gigabytes = 10
+        retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+        Logger.info("Creating main topics for parsers")
+        for parser_name in self.get_parser_list():
+            Logger.info("Creating topic'{}'".format(parser_name))
+            Execute(command_template.format(self.__params.kafka_bin_dir,
+                                            self.__params.zookeeper_quorum,
+                                            parser_name,
+                                            num_partitions,
+                                            replication_factor,
+                                            retention_bytes))
+        Logger.info("Creating topics for error handling")
+        Execute(command_template.format(self.__params.kafka_bin_dir,
+                                        self.__params.zookeeper_quorum,
+                                        "parser_invalid",
+                                        num_partitions,
+                                        replication_factor,
+                                        retention_bytes))
+        Execute(command_template.format(self.__params.kafka_bin_dir,
+                                        self.__params.zookeeper_quorum,
+                                        "parser_error",
+                                        num_partitions, replication_factor,
+                                        retention_bytes))
+        Logger.info("Done creating Kafka topics")
+
+    def init_parser_config(self):
+        Logger.info('Loading parser config into ZooKeeper')
+        Execute(format(
+            "{metron_home}/bin/zk_load_configs.sh --mode PUSH -i {metron_zookeeper_config_path}
-z {zookeeper_quorum}"),
+            path=format("{java_home}/bin")
+        )
+
+    def start_parser_topologies(self):
+        Logger.info("Starting Metron parser topologies: {}".format(self.get_parser_list()))
+        start_cmd_template = """{}/bin/start_parser_topology.sh \
+                                    -k {} \
+                                    -z {} \
+                                    -s {}"""
+        for parser in self.get_parser_list():
+            Logger.info('Starting ' + parser)
+            Execute(start_cmd_template.format(self.__params.metron_home, self.__params.kafka_brokers,
+                                              self.__params.zookeeper_quorum, parser))
+
+        Logger.info('Finished starting parser topologies')
+
+    def stop_parser_topologies(self):
+        Logger.info('Stopping parsers')
+        for parser in self.get_parser_list():
+            Logger.info('Stopping ' + parser)
+            stop_cmd = 'storm kill ' + parser
+            Execute(stop_cmd)
+        Logger.info('Done stopping parser topologies')
+
+    def restart_parser_topologies(self):
+        Logger.info('Restarting the parser topologies')
+        self.stop_parser_topologies()
+        attempt_count = 0
+        while self.topologies_exist():
+            if attempt_count > 2:
+                raise Exception("Unable to kill topologies")
+            attempt_count += 1
+            time.sleep(10)
+        self.start_parser_topologies()
+        Logger.info('Done restarting the parser topologies')
+
+    def topologies_exist(self):
+        cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (stdout, stderr) = cmd_open.communicate()
+        stdout_lines = stdout.splitlines()
+        if stdout_lines:
+            status_lines = self.__get_status_lines(stdout_lines)
+            for parser in self.get_parser_list():
+                for line in status_lines:
+                    items = re.sub('[\s]+', ' ', line).split()
+                    if items and items[0] == parser:
+                        return True
+        return False
+
+    def topologies_running(self):
+        cmd_open = subprocess.Popen(["storm", "list"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        (stdout, stderr) = cmd_open.communicate()
+        stdout_lines = stdout.splitlines()
+        all_running = False
+        if stdout_lines:
+            all_running = True
+            status_lines = self.__get_status_lines(stdout_lines)
+            for parser in self.get_parser_list():
+                parser_found = False
+                is_running = False
+                for line in status_lines:
+                    items = re.sub('[\s]+', ' ', line).split()
+                    if items and items[0] == parser:
+                        status = items[1]
+                        parser_found = True
+                        is_running = self.__is_running(status)
+                all_running &= parser_found and is_running
+        return all_running
+
+    def __get_status_lines(self, lines):
+        status_lines = []
+        do_stat = False
+        skipped = 0
+        for line in lines:
+            if line.startswith("Topology_name"):
+                do_stat = True
+            if do_stat and skipped == 2:
+                status_lines += [line]
+            elif do_stat:
+                skipped += 1
+        return status_lines
+
+    def __is_running(self, status):
+        return status in [self.StormStatus.ACTIVE, self.StormStatus.REBALANCING]

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
new file mode 100755
index 0000000..242460e
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/__init__.py
@@ -0,0 +1,18 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
new file mode 100755
index 0000000..953435d
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+    from params_windows import *
+else:
+    from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
new file mode 100755
index 0000000..de7dfc9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_linux.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import functools
+import os
+
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script import Script
+
+# server configurations
+config = Script.get_config()
+
+hostname = config['hostname']
+#print(config['configurations'])
+metron_home = config['configurations']['metron-parsers']['metron_home']
+parsers = config['configurations']['metron-parsers']['parsers']
+metron_user = config['configurations']['metron-parsers']['metron_user']
+metron_group = config['configurations']['metron-parsers']['metron_group']
+metron_zookeeper_config_dir = config['configurations']['metron-parsers']['metron_zookeeper_config_dir']
+#metron_zookeeper_config_path = metron_home + "/" + metron_zookeeper_config_dir
+metron_zookeeper_config_path = format('{metron_home}/{metron_zookeeper_config_dir}')
+configured_flag_file = metron_zookeeper_config_path + '/../metron_is_configured'
+yum_repo_type = 'local'
+
+# hadoop params
+stack_root = Script.get_stack_root()
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+
+# zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+    if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+        zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+    else:
+        zookeeper_clientPort = '2181'
+    zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+    # last port config
+    zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Kafka
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+    if 'port' in config['configurations']['kafka-broker']:
+        kafka_broker_port = config['configurations']['kafka-broker']['port']
+    else:
+        kafka_broker_port = '6667'
+    kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+    kafka_brokers += ':' + kafka_broker_port
+
+metron_apps_dir = config['configurations']['metron-parsers']['metron_apps_hdfs_dir']
+
+local_grok_patterns_dir = format("{metron_home}/patterns")
+hdfs_grok_patterns_dir = format("{metron_apps_dir}/patterns")
+
+# for create_hdfs_directory
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+hostname = config["hostname"]
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+dfs_type = default("/commandParams/dfs_type", "")
+
+# create partial functions with common arguments for every HdfsResource call
+# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource
in code
+HdfsResource = functools.partial(
+    HdfsResource,
+    user=hdfs_user,
+    hdfs_resource_ignore_file="/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+    security_enabled=security_enabled,
+    keytab=hdfs_user_keytab,
+    kinit_path_local=kinit_path_local,
+    hadoop_bin_dir=hadoop_bin_dir,
+    hadoop_conf_dir=hadoop_conf_dir,
+    principal_name=hdfs_principal_name,
+    hdfs_site=hdfs_site,
+    default_fs=default_fs,
+    immutable_paths=get_not_managed_resources(),
+    dfs_type=dfs_type
+)

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
new file mode 100755
index 0000000..4d11b35
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/params_windows.py
@@ -0,0 +1,20 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+raise NotImplementedError

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
new file mode 100755
index 0000000..ee83414
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/params/status_params.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+parsers = "bro,yaf,snort"
+configured_flag_file = ""

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
new file mode 100755
index 0000000..4a5b910
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/parser_master.py
@@ -0,0 +1,77 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class ParserMaster(Script):
+    def get_component_name(self):
+        # TODO add this at some point - currently will cause problems with hdp-select
+        # return "parser-master"
+        pass
+
+    def install(self, env):
+        from params import params
+        env.set_params(params)
+        commands = Commands(params)
+        commands.setup_repo()
+        Logger.info('Install RPM packages')
+        self.install_packages(env)
+
+    def configure(self, env, upgrade_type=None, config_dir=None):
+        from params import params
+        env.set_params(params)
+
+    def start(self, env, upgrade_type=None):
+        from params import params
+        env.set_params(params)
+        commands = Commands(params)
+        if not commands.is_configured():
+            commands.init_parsers()
+            commands.init_kafka_topics()
+            commands.init_parser_config()
+            commands.set_configured()
+        commands.start_parser_topologies()
+
+    def stop(self, env, upgrade_type=None):
+        from params import params
+        env.set_params(params)
+        commands = Commands(params)
+        commands.stop_parser_topologies()
+
+    def status(self, env):
+        from params import status_params
+        env.set_params(status_params)
+        commands = Commands(status_params)
+        if not commands.topologies_running():
+            raise ComponentIsNotRunning()
+
+    def restart(self, env):
+        from params import params
+        env.set_params(params)
+        commands = Commands(params)
+        commands.restart_parser_topologies()
+
+
+if __name__ == "__main__":
+    ParserMaster().execute()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..6ef7e4a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/scripts/service_check.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class ServiceCheck(Script):
+    def service_check(self, env):
+        from params import params
+        env.set_params(params)
+        commands = Commands(params)
+        all_found = commands.topologies_running()
+        if all_found:
+            exit(0)
+        else:
+            exit(1)
+
+
+if __name__ == "__main__":
+    ServiceCheck().execute()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
new file mode 100755
index 0000000..0967ef4
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/package/templates/metron-global.json
@@ -0,0 +1 @@
+{}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+  "name": "default",
+  "description": "default quick links configuration",
+  "configuration": {
+    "protocol":
+    {
+      "type":"HTTP_ONLY"
+    },
+
+    "links": [
+      {
+        "name": "storm_ui",
+        "label": "Storm UI",
+        "requires_user_name": "false",
+        "component_name": "STORM_UI_SERVER",
+        "url":"%@://%@:%@/",
+        "port":{
+          "http_property": "ui.port",
+          "http_default_port": "8744",
+          "https_property": "ui.port",
+          "https_default_port": "8744",
+          "regex": "^(\\d+)$",
+          "site": "storm-site"
+        }
+      }
+    ]
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
new file mode 100755
index 0000000..84b466a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/PARSERS/0.2.0BETA/service_advisor.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import imp
+import os
+import traceback
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+    with open(PARENT_FILE, 'rb') as fp:
+        service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb',
imp.PY_SOURCE))
+except Exception as e:
+    traceback.print_exc()
+    print("Failed to load parent service_advisor file '{}'".format(PARENT_FILE))
+
+
+class TODO_PARSERS020BETAServiceAdvisor(service_advisor.ServiceAdvisor):
+    # colocate Metron Parser Master with KAFKA_BROKERs
+    def TODO_colocateService(self, hostsComponentsMap, serviceComponents):
+        parsersMasterComponent = [component for component in serviceComponents if
+                                  component["StackServiceComponents"]["component_name"] ==
"PARSER_MASTER"][0]
+        if not self.isComponentHostsPopulated(parsersMasterComponent):
+            for hostName in hostsComponentsMap.keys():
+                hostComponents = hostsComponentsMap[hostName]
+                if ({"name": "KAFKA_BROKER"} in hostComponents) and {"name": "PARSER_MASTER"}
not in hostComponents:
+                    hostsComponentsMap[hostName].append({"name": "PARSER_MASTER"})
+                if ({"name": "KAFKA_BROKER"} not in hostComponents) and {"name": "PARSER_MASTER"}
in hostComponents:
+                    hostsComponentsMap[hostName].remove({"name": "PARSER_MASTER"})
+
+    def TODO_getServiceComponentLayoutValidations(self, services, hosts):
+        componentsListList = [service["components"] for service in services["services"]]
+        componentsList = [item["StackServiceComponents"] for sublist in componentsListList
for item in sublist]
+        hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+        hostsCount = len(hostsList)
+
+        masterHosts = self.getHosts(componentsList, "PARSER_MASTER")
+        expectedMasterHosts = set(self.getHosts(componentsList, "KAFKA_BROKER"))
+
+        items = []
+
+        mismatchHosts = sorted(expectedMasterHosts.symmetric_difference(set(masterHosts)))
+        if len(mismatchHosts) > 0:
+            hostsString = ', '.join(mismatchHosts)
+            message = "Metron Parsers Master must be installed on Kafka Brokers. " \
+                      "The following {0} host(s) do not satisfy the colocation recommendation:
{1}".format(
+                len(mismatchHosts), hostsString)
+            items.append(
+                {"type": 'host-component', "level": 'WARN', "message": message, "component-name":
'PARSER_MASTER'})
+
+        return items

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
new file mode 100755
index 0000000..e0292cf
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+    <schemaVersion>2.0</schemaVersion>
+    <services>
+        <service>
+            <name>PARSERS</name>
+            <version>0.2.0BETA</version>
+            <extends>common-services/PARSERS/0.2.0BETA</extends>
+        </service>
+    </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/47bc9b6d/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
new file mode 100755
index 0000000..c861646
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.3/services/PARSERS/role_command_order.json
@@ -0,0 +1,17 @@
+{
+  "_comment": "Record format:",
+  "_comment": "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2,
...]",
+  "general_deps": {
+    "_comment": "dependencies for Metron Parsers",
+    "PARSER_MASTER-START": [
+      "NAMENODE-START",
+      "ZOOKEEPER_SERVER-START",
+      "KAFKA_BROKER-START",
+      "STORM_REST_API-START"
+    ],
+    "_comment": "<SERVICE_NAME>_SERVICE_CHECK-SERVICE_CHECK",
+    "PARSERS_SERVICE_CHECK-SERVICE_CHECK": [
+      "PARSER_MASTER-START"
+    ]
+  }
+}



Mime
View raw message