metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject incubator-metron git commit: METRON-385 Create Ambari Service Definition for Indexing (justinleet via cestella) closes apache/incubator-metron#222
Date Mon, 12 Sep 2016 14:48:51 GMT
Repository: incubator-metron
Updated Branches:
  refs/heads/master 7eceac2aa -> 2e5538c4e


METRON-385 Create Ambari Service Definition for Indexing (justinleet via cestella) closes
apache/incubator-metron#222


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/2e5538c4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/2e5538c4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/2e5538c4

Branch: refs/heads/master
Commit: 2e5538c4e7abd21aa73ecea4ef88c596e51896e0
Parents: 7eceac2
Author: justinleet <justinjleet@gmail.com>
Authored: Mon Sep 12 10:47:58 2016 -0400
Committer: cstella <cestella@gmail.com>
Committed: Mon Sep 12 10:47:58 2016 -0400

----------------------------------------------------------------------
 metron-deployment/packaging/ambari/.gitignore   |   1 -
 .../0.2.0BETA/configuration/metron-indexing.xml |  48 ++++++
 .../INDEXING/0.2.0BETA/metainfo.xml             | 105 +++++++++++++
 .../0.2.0BETA/package/scripts/commands.py       | 146 +++++++++++++++++++
 .../package/scripts/indexing_master.py          |  71 +++++++++
 .../0.2.0BETA/package/scripts/params.py         |  29 ++++
 .../0.2.0BETA/package/scripts/params_linux.py   |  70 +++++++++
 .../0.2.0BETA/package/scripts/params_windows.py |  17 +++
 .../0.2.0BETA/package/scripts/service_check.py  |  37 +++++
 .../0.2.0BETA/package/scripts/status_params.py  |  19 +++
 .../0.2.0BETA/quicklinks/quicklinks.json        |  28 ++++
 .../HDP/2.4/services/INDEXING/metainfo.xml      |  26 ++++
 .../services/INDEXING/role_command_order.json   |   9 ++
 .../src/main/config/elasticsearch.properties    |   5 +-
 14 files changed, 609 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/.gitignore b/metron-deployment/packaging/ambari/.gitignore
index db8ded0..e708548 100644
--- a/metron-deployment/packaging/ambari/.gitignore
+++ b/metron-deployment/packaging/ambari/.gitignore
@@ -1,3 +1,2 @@
 archive.zip
 *.hash
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
new file mode 100755
index 0000000..3a1c8f7
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/configuration/metron-indexing.xml
@@ -0,0 +1,48 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<configuration supports_final="true">
+    <property>
+        <name>metron_home</name>
+        <value>/usr/metron/0.2.0BETA</value>
+        <description>Metron home directory</description>
+        <display-name>Metron home</display-name>
+    </property>
+    <property>
+        <name>metron_apps_hdfs_dir</name>
+        <value>/apps/metron</value>
+        <description>Metron apps HDFS dir</description>
+        <display-name>Metron apps HDFS dir</display-name>
+    </property>
+    <property>
+        <name>metron_user</name>
+        <value>metron</value>
+        <property-type>USER</property-type>
+        <description>The user for Metron</description>
+        <display-name>Metron User</display-name>
+    </property>
+    <property>
+        <name>metron_group</name>
+        <value>metron</value>
+        <description>The group for Metron</description>
+    </property>
+    <property>
+        <name>metron_indexing_topology</name>
+        <value>indexing</value>
+        <description>The Storm topology name for Indexing</description>
+    </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
new file mode 100755
index 0000000..d2dce5a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/metainfo.xml
@@ -0,0 +1,105 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<metainfo>
+    <schemaVersion>2.0</schemaVersion>
+    <services>
+        <service>
+            <name>INDEXING</name>
+            <displayName>Indexing</displayName>
+            <comment>Indexing</comment>
+            <version>0.2.0BETA</version>
+
+            <components>
+                <component>
+                    <name>INDEXING_MASTER</name>
+                    <displayName>Indexing Master-Eligible Node</displayName>
+                    <category>MASTER</category>
+                    <cardinality>1</cardinality>
+                    <versionAdvertised>true</versionAdvertised>
+                    <dependencies>
+                        <dependency>
+                            <name>HDFS/HDFS_CLIENT</name>
+                            <scope>host</scope>
+                            <auto-deploy>
+                                <enabled>true</enabled>
+                            </auto-deploy>
+                        </dependency>
+                        <dependency>
+                            <name>ZOOKEEPER/ZOOKEEPER_SERVER</name>
+                            <scope>cluster</scope>
+                            <auto-deploy>
+                                <enabled>true</enabled>
+                            </auto-deploy>
+                        </dependency>
+                        <dependency>
+                            <name>KAFKA/KAFKA_BROKER</name>
+                            <scope>cluster</scope>
+                            <auto-deploy>
+                                <enabled>true</enabled>
+                            </auto-deploy>
+                        </dependency>
+                    </dependencies>
+                    <commandScript>
+                        <script>scripts/indexing_master.py</script>
+                        <scriptType>PYTHON</scriptType>
+                        <timeout>600</timeout>
+                    </commandScript>
+                </component>
+            </components>
+
+            <osSpecifics>
+                <osSpecific>
+                    <osFamily>any</osFamily>
+                    <packages>
+                        <package>
+                            <name>metron-common</name>
+                        </package>
+                        <package>
+                            <name>metron-indexing</name>
+                        </package>
+                        <package>
+                            <name>metron-elasticsearch</name>
+                        </package>
+                    </packages>
+                </osSpecific>
+            </osSpecifics>
+
+            <commandScript>
+                <script>scripts/service_check.py</script>
+                <scriptType>PYTHON</scriptType>
+                <timeout>300</timeout>
+            </commandScript>
+
+            <requiredServices>
+                <service>HDFS</service>
+                <service>KAFKA</service>
+                <service>STORM</service>
+                <service>ZOOKEEPER</service>
+            </requiredServices>
+
+            <configuration-dependencies>
+                <config-type>metron-indexing</config-type>
+            </configuration-dependencies>
+
+            <quickLinksConfigurations>
+                <quickLinksConfiguration>
+                    <fileName>quicklinks.json</fileName>
+                    <default>true</default>
+                </quickLinksConfiguration>
+            </quickLinksConfigurations>
+        </service>
+    </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
new file mode 100755
index 0000000..a9a86c0
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/commands.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+import subprocess
+import time
+
+from resource_management.core.logger import Logger
+from resource_management.core.resources.system import Execute, File
+
+
+# Wrap major operations and functionality in this class
+class Commands:
+    __params = None
+    __configured = False
+
+    def __init__(self, params):
+        if params is None:
+            raise ValueError("params argument is required for initialization")
+        self.__params = params
+        self.__configured = os.path.isfile(self.__params.configured_flag_file)
+
+    def is_configured(self):
+        return self.__configured
+
+    def set_configured(self):
+        File(self.__params.configured_flag_file,
+             content="",
+             owner=self.__params.metron_user,
+             mode=0775)
+
+    def setup_repo(self):
+        def local_repo():
+            Logger.info("Setting up local repo")
+            Execute("yum -y install createrepo")
+            Execute("createrepo /localrepo")
+            Execute("chmod -R o-w+r /localrepo")
+            Execute("echo \"[METRON-0.2.0BETA]\n"
+                    "name=Metron 0.2.0BETA packages\n"
+                    "baseurl=file:///localrepo\n"
+                    "gpgcheck=0\n"
+                    "enabled=1\" > /etc/yum.repos.d/local.repo")
+
+        def remote_repo():
+            print('Using remote repo')
+
+        yum_repo_types = {
+            'local': local_repo,
+            'remote': remote_repo
+        }
+        repo_type = self.__params.yum_repo_type
+        if repo_type in yum_repo_types:
+            yum_repo_types[repo_type]()
+        else:
+            raise ValueError("Unsupported repo type '{}'".format(repo_type))
+
+    def init_kafka_topics(self):
+        Logger.info('Creating Kafka topics')
+        command_template = """{}/kafka-topics.sh \
+                                    --zookeeper {} \
+                                    --create \
+                                    --topic {} \
+                                    --partitions {} \
+                                    --replication-factor {} \
+                                    --config retention.bytes={}"""
+        num_partitions = 1
+        replication_factor = 1
+        retention_gigabytes = 10
+        retention_bytes = retention_gigabytes * 1024 * 1024 * 1024
+        Logger.info("Creating topics for indexing")
+
+        Logger.info("Creating topic'{}'".format(self.__params.metron_indexing_topology))
+        Execute(command_template.format(self.__params.kafka_bin_dir,
+                                        self.__params.zookeeper_quorum,
+                                        self.__params.metron_indexing_topology,
+                                        num_partitions,
+                                        replication_factor,
+                                        retention_bytes))
+        Logger.info("Done creating Kafka topics")
+
+    def start_indexing_topology(self):
+        Logger.info("Starting Metron indexing topology: {}".format(self.__params.metron_indexing_topology))
+        start_cmd_template = """{}/bin/start_elasticsearch_topology.sh \
+                                        -s {} \
+                                        -z {}"""
+        Logger.info('Starting ' + self.__params.metron_indexing_topology)
+        Execute(start_cmd_template.format(self.__params.metron_home, self.__params.metron_indexing_topology,
self.__params.zookeeper_quorum))
+
+        Logger.info('Finished starting indexing topology')
+
+    def stop_indexing_topology(self):
+        Logger.info('Stopping ' + self.__params.metron_indexing_topology)
+        stop_cmd = 'storm kill ' + self.__params.metron_indexing_topology
+        Execute(stop_cmd)
+        Logger.info('Done stopping indexing topologies')
+
+    def restart_indexing_topology(self):
+        Logger.info('Restarting the indexing topologies')
+        self.stop_indexing_topology()
+
+        # Wait for old topology to be cleaned up by Storm, before starting again.
+        retries = 0
+        topology_active = self.is_topology_active()
+        while topology_active and retries < 3:
+            Logger.info('Existing topology still active. Will wait and retry')
+            time.sleep(40)
+            topology_active = self.is_topology_active()
+            retries += 1
+
+        if not topology_active:
+            self.start_indexing_topology()
+            Logger.info('Done restarting the indexing topologies')
+        else:
+            Logger.warning('Retries exhausted. Existing topology not cleaned up.  Aborting
topology start.')
+
+    def is_topology_active(self):
+        cmd_retrieve = "storm list | grep 'indexing'"
+
+        proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
shell=True)
+        (stdout, stderr) = proc.communicate()
+        Logger.info("Retrieval response is: %s" % stdout)
+        Logger.warning("Error response is: %s" % stderr)
+
+        fields = stdout.split()
+        if len(fields) < 2:
+            Logger.warning("Indexing topology is not running")
+            return False
+
+        # Get the second column, which is status. We already know first column is indexing)
+        status = stdout.split()[1]
+        running_status_set = ['ACTIVE', 'REBALANCING']
+        return status in running_status_set

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
new file mode 100755
index 0000000..bfae19a
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/indexing_master.py
@@ -0,0 +1,71 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.logger import Logger
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class Indexing(Script):
+    def install(self, env):
+        import params
+        env.set_params(params)
+        commands = Commands(params)
+        commands.setup_repo()
+        Logger.info('Install RPM packages')
+        self.install_packages(env)
+
+    def start(self, env, upgrade_type=None):
+        import params
+        env.set_params(params)
+        commands = Commands(params)
+        if not commands.is_configured():
+            commands.init_kafka_topics()
+            commands.set_configured()
+
+        commands.start_indexing_topology()
+
+    def stop(self, env, upgrade_type=None):
+        import params
+        env.set_params(params)
+        commands = Commands(params)
+        commands.stop_indexing_topology()
+
+    def status(self, env):
+        import status_params
+        env.set_params(status_params)
+        commands = Commands(status_params)
+
+        if not commands.is_topology_active():
+            raise ComponentIsNotRunning()
+
+    def restart(self, env):
+        import params
+        env.set_params(params)
+        commands = Commands(params)
+        commands.restart_indexing_topology()
+
+    def kafkabuild(self, env, upgrade_type=None):
+        import params
+        env.set_params(params)
+        commands = Commands(params)
+        commands.init_kafka_topics()
+
+
+if __name__ == "__main__":
+    Indexing().execute()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
new file mode 100755
index 0000000..6440005
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.expect import expect
+
+if OSCheck.is_windows_family():
+    pass
+else:
+    from params_linux import *
+
+java_home = config['hostLevelParams']['java_home']
+java_version = expect("/hostLevelParams/java_version", int)
+
+host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
new file mode 100755
index 0000000..86a0359
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_linux.py
@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.script import Script
+
+# Server configurations
+config = Script.get_config()
+
+hostname = config['hostname']
+metron_home = config['configurations']['metron-indexing']['metron_home']
+metron_indexing_topology = config['configurations']['metron-indexing']['metron_indexing_topology']
+yum_repo_type = 'local'
+metron_user = config['configurations']['metron-indexing']['metron_user']
+
+metron_config_path = metron_home + '/config'
+configured_flag_file = metron_config_path + '/metron_indexing_is_configured'
+
+# Hadoop params
+hadoop_home_dir = stack_select.get_hadoop_dir("home")
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+
+# Zookeeper
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+has_zk_host = not len(zk_hosts) == 0
+zookeeper_quorum = None
+if has_zk_host:
+    if 'zoo.cfg' in config['configurations'] and 'clientPort' in config['configurations']['zoo.cfg']:
+        zookeeper_clientPort = config['configurations']['zoo.cfg']['clientPort']
+    else:
+        zookeeper_clientPort = '2181'
+    zookeeper_quorum = (':' + zookeeper_clientPort + ',').join(config['clusterHostInfo']['zookeeper_hosts'])
+    # last port config
+    zookeeper_quorum += ':' + zookeeper_clientPort
+
+# Kafka
+stack_root = Script.get_stack_root()
+kafka_home = os.path.join(stack_root, "current", "kafka-broker")
+kafka_bin_dir = os.path.join(kafka_home, "bin")
+metron_indexing_topic_retention = config['configurations']['metron-indexing']['metron_indexing_topic_retention']
+
+kafka_hosts = default("/clusterHostInfo/kafka_broker_hosts", [])
+has_kafka_host = not len(kafka_hosts) == 0
+kafka_brokers = None
+if has_kafka_host:
+    if 'port' in config['configurations']['kafka-broker']:
+        kafka_broker_port = config['configurations']['kafka-broker']['port']
+    else:
+        kafka_broker_port = '6667'
+    kafka_brokers = (':' + kafka_broker_port + ',').join(config['clusterHostInfo']['kafka_broker_hosts'])
+    kafka_brokers += ':' + kafka_broker_port

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
new file mode 100755
index 0000000..b5828d2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/params_windows.py
@@ -0,0 +1,17 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+raise NotImplementedError

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
new file mode 100755
index 0000000..c02d695
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/service_check.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from __future__ import print_function
+
+from resource_management.libraries.script import Script
+
+from commands import Commands
+
+
+class ServiceCheck(Script):
+    def service_check(self, env):
+        import params
+        env.set_params(params)
+
+        commands = Commands(params)
+        if commands.is_topology_active():
+            exit(0)
+        else:
+            exit(1)
+
+
+if __name__ == "__main__":
+    ServiceCheck().execute()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
new file mode 100755
index 0000000..30aed60
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/package/scripts/status_params.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+metron_indexing_topology = 'indexing'
+configured_flag_file = ""

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
new file mode 100755
index 0000000..ee1b225
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/common-services/INDEXING/0.2.0BETA/quicklinks/quicklinks.json
@@ -0,0 +1,28 @@
+{
+  "name": "default",
+  "description": "default quick links configuration",
+  "configuration": {
+    "protocol":
+    {
+      "type":"HTTP_ONLY"
+    },
+
+    "links": [
+      {
+        "name": "storm_ui",
+        "label": "Storm UI",
+        "requires_user_name": "false",
+        "component_name": "STORM_UI_SERVER",
+        "url":"%@://%@:%@/",
+        "port":{
+          "http_property": "ui.port",
+          "http_default_port": "8744",
+          "https_property": "ui.port",
+          "https_default_port": "8744",
+          "regex": "^(\\d+)$",
+          "site": "storm-site"
+        }
+      }
+    ]
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
new file mode 100755
index 0000000..e406780
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/metainfo.xml
@@ -0,0 +1,26 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+       http://www.apache.org/licenses/LICENSE-2.0
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<metainfo>
+    <schemaVersion>2.0</schemaVersion>
+    <services>
+        <service>
+            <name>INDEXING</name>
+            <version>0.2.0BETA</version>
+            <extends>common-services/INDEXING/0.2.0BETA</extends>
+        </service>
+    </services>
+</metainfo>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
new file mode 100755
index 0000000..f58c59c
--- /dev/null
+++ b/metron-deployment/packaging/ambari/src/main/resources/stacks/HDP/2.4/services/INDEXING/role_command_order.json
@@ -0,0 +1,9 @@
+{
+    "_comment" : "Record format:",
+    "_comment" : "blockedRole-blockedCommand: [blockerRole1-blockerCommand1, blockerRole2-blockerCommand2,
...]",
+    "general_deps" : {
+        "_comment" : "dependencies for all cases",
+        "INDEXING_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START",
"STORM_REST_API-START"],
+        "INDEXING_SERVICE_CHECK-SERVICE_CHECK" : ["INDEXING_MASTER-START"]
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e5538c4/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
index a1516d6..c2c10af 100644
--- a/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
+++ b/metron-platform/metron-elasticsearch/src/main/config/elasticsearch.properties
@@ -57,10 +57,13 @@ org.apache.metron.metrics.TelemetryParserBolt.fails=true
 
 bolt.hdfs.batch.size=5000
 bolt.hdfs.field.delimiter=|
+bolt.hdfs.rotation.policy=org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+bolt.hdfs.rotation.policy.units=DAYS
+bolt.hdfs.rotation.policy.count=1
+
 bolt.hdfs.file.rotation.size.in.mb=5
 bolt.hdfs.file.system.url=hdfs://iot01.cloud.hortonworks.com:8020
 bolt.hdfs.wip.file.path=/paloalto/wip
 bolt.hdfs.finished.file.path=/paloalto/rotated
 bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
 index.hdfs.output=/tmp/metron/enriched
-


Mime
View raw message