metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [6/6] incubator-metron git commit: METRON-671 Refactor existing Ansible deployment to use Ambari MPack (dlyle via justinleet) closes apache/incubator-metron#436
Date Fri, 10 Mar 2017 14:32:41 GMT
METRON-671 Refactor existing Ansible deployment to use Ambari MPack (dlyle via justinleet) closes apache/incubator-metron#436


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/68a334a8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/68a334a8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/68a334a8

Branch: refs/heads/master
Commit: 68a334a8717d2a6b82f7f9651c57bc75e256ba97
Parents: 8b6fc2e
Author: dlyle <dlyle65535@gmail.com>
Authored: Fri Mar 10 09:31:46 2017 -0500
Committer: leet <leet@apache.org>
Committed: Fri Mar 10 09:31:46 2017 -0500

----------------------------------------------------------------------
 Upgrading.md                                    |   8 +
 metron-deployment/README.md                     |  98 +---
 metron-deployment/amazon-ec2/conf/defaults.yml  |   7 -
 metron-deployment/amazon-ec2/playbook.yml       |  10 +-
 metron-deployment/amazon-ec2/run.sh             |   7 +-
 .../extra_modules/ambari_cluster_state.py       |   2 +-
 .../extra_modules/ambari_service_state.py       | 352 +++++++++++
 .../inventory/full-dev-platform/group_vars/all  |   6 -
 .../inventory/full-dev-platform/hosts           |  23 +-
 .../inventory/metron_example/group_vars/all     | 100 ----
 .../inventory/metron_example/hosts              |  63 --
 .../inventory/quick-dev-platform/group_vars/all |   6 -
 .../inventory/quick-dev-platform/hosts          |  23 +-
 .../METRON/CURRENT/role_command_order.json      |   6 +-
 .../2.3.3/configuration/elastic-env.xml         |  15 +-
 .../2.3.3/configuration/elastic-site.xml        |   8 +
 .../2.3.3/configuration/elastic-sysconfig.xml   |   2 +-
 .../ELASTICSEARCH/2.3.3/metainfo.xml            |   2 +-
 .../2.3.3/package/scripts/elastic.py            |   8 +-
 .../2.3.3/package/scripts/params.py             |   3 +-
 .../2.3.3/package/scripts/slave.py              |   7 +-
 .../templates/elasticsearch.master.yaml.j2      |   4 +-
 .../KIBANA/4.5.1/configuration/kibana-env.xml   |  19 +-
 .../KIBANA/4.5.1/configuration/kibana-site.xml  |   2 +-
 .../KIBANA/4.5.1/package/scripts/params.py      |   4 +-
 .../METRON/CURRENT/configuration/metron-env.xml |  14 +-
 .../package/scripts/enrichment_commands.py      |  46 +-
 .../package/scripts/enrichment_master.py        |  12 +-
 .../package/scripts/indexing_commands.py        |   2 +-
 .../package/scripts/params/params_linux.py      |   6 +-
 .../package/scripts/params/status_params.py     |   4 +-
 .../packaging/packer-build/.gitignore           |  12 +
 .../packaging/packer-build/README.md            |  51 ++
 .../packaging/packer-build/ansible.cfg          |  27 +
 .../packaging/packer-build/ansible/playbook.yml |  28 +
 .../packer-build/ansible/tasks/cleanup.yml      |  42 ++
 .../ansible/tasks/remove_services.yml           |  46 ++
 .../packaging/packer-build/base-centos-6.7.json | 123 ++++
 .../packaging/packer-build/bin/bento            | 475 +++++++++++++++
 .../packer-build/http/centos-6.7/ks.cfg         |  86 +++
 .../packer-build/quick-dev-centos-6.7.json      | 132 +++++
 .../packer-build/scripts/centos/cleanup.sh      |  25 +
 .../packer-build/scripts/centos/networking.sh   |  20 +
 .../packer-build/scripts/centos/update_all.sh   |  20 +
 .../packer-build/scripts/common/metadata.sh     |   6 +
 .../packer-build/scripts/common/minimize.sh     |  28 +
 .../packer-build/scripts/common/reboot.sh       |  20 +
 .../packer-build/scripts/common/sshd.sh         |  20 +
 .../packer-build/scripts/common/sudoers.sh      |   4 +
 .../packer-build/scripts/common/vagrant.sh      |  19 +
 .../packer-build/scripts/common/vmtools.sh      |  61 ++
 metron-deployment/packer-build/.gitignore       |  12 -
 metron-deployment/packer-build/README.md        |  52 --
 metron-deployment/packer-build/ansible.cfg      |  27 -
 .../packer-build/ansible/playbook.yml           |  36 --
 .../packer-build/ansible/tasks/cleanup.yml      |  49 --
 .../packer-build/ansible/tasks/clear_topics.yml |  29 -
 .../ansible/tasks/remove_services.yml           |  46 --
 .../packer-build/ansible/tasks/stop_storm.yml   |  28 -
 metron-deployment/packer-build/bin/bento        | 469 ---------------
 .../packer-build/hdp-centos-6.7.json            | 132 -----
 .../packer-build/http/centos-6.7/ks.cfg         |  86 ---
 .../packer-build/metron-centos-6.7.json         | 131 -----
 .../packer-build/scripts/centos/cleanup.sh      |  25 -
 .../packer-build/scripts/centos/networking.sh   |  20 -
 .../packer-build/scripts/centos/update_all.sh   |  20 -
 .../packer-build/scripts/common/metadata.sh     |   6 -
 .../packer-build/scripts/common/minimize.sh     |  28 -
 .../packer-build/scripts/common/reboot.sh       |  20 -
 .../packer-build/scripts/common/sshd.sh         |  20 -
 .../packer-build/scripts/common/sudoers.sh      |   4 -
 .../packer-build/scripts/common/vagrant.sh      |  19 -
 .../packer-build/scripts/common/vmtools.sh      |  61 --
 metron-deployment/playbooks/ambari_install.yml  |  14 +
 metron-deployment/playbooks/metron_build.yml    |  21 +
 .../playbooks/metron_full_install.yml           |   4 +
 metron-deployment/playbooks/metron_install.yml  |  80 +--
 metron-deployment/pom.xml                       |   2 +-
 .../roles/ambari_common/defaults/main.yml       |   3 +-
 .../roles/ambari_config/meta/main.yml           |   1 +
 .../roles/ambari_config/tasks/main.yml          |   1 -
 .../roles/ambari_config/tasks/start_hdp.yml     |  43 --
 .../vars/multi_vagrant_cluster.yml              |  99 ----
 .../roles/ambari_config/vars/single_node_vm.yml |  32 +-
 .../roles/ambari_config/vars/small_cluster.yml  |  54 +-
 .../roles/ambari_gather_facts/tasks/main.yml    |  32 +-
 .../roles/ambari_master/defaults/main.yml       |   2 +-
 .../roles/ambari_master/tasks/main.yml          |  12 +-
 .../roles/ambari_slave/meta/main.yml            |  19 +
 .../roles/ambari_slave/tasks/main.yml           |   5 +-
 .../roles/bro/tasks/bro-plugin-kafka.yml        |   3 +
 .../roles/deployment-report/tasks/main.yml      |   1 -
 .../roles/elasticsearch/defaults/main.yml       |  22 -
 .../elasticsearch/files/elasticsearch.repo      |  24 -
 .../roles/elasticsearch/meta/main.yml           |  24 -
 .../roles/elasticsearch/tasks/elasticsearch.yml |  73 ---
 .../roles/elasticsearch/tasks/main.yml          |  24 -
 .../metron-elasticsearch-logrotate.yml          |  27 -
 .../roles/java_jdk/defaults/main.yml            |   2 +-
 metron-deployment/roles/kibana/README.md        |  35 --
 .../roles/kibana/defaults/main.yml              |  22 -
 .../roles/kibana/tasks/dashboard.yml            |  22 -
 .../roles/kibana/tasks/dependencies.yml         |  27 -
 .../roles/kibana/tasks/elasticdump.yml          |  35 --
 metron-deployment/roles/kibana/tasks/kibana.yml |  47 --
 metron-deployment/roles/kibana/tasks/main.yml   |  21 -
 .../roles/kibana/templates/kibana-index.json    |  52 --
 .../roles/kibana/templates/kibana.repo.template |  22 -
 .../roles/load_web_templates/meta/main.yml      |  19 +
 .../roles/load_web_templates/tasks/main.yml     |  32 +
 .../roles/metron-builder/defaults/main.yml      |  18 +
 .../roles/metron-builder/tasks/main.yml         |  21 +
 .../roles/metron-rpms/defaults/main.yml         |  18 +
 .../roles/metron-rpms/tasks/main.yml            |  32 +
 .../roles/metron_common/defaults/main.yml       |  19 -
 .../roles/metron_common/meta/main.yml           |  20 -
 .../roles/metron_common/tasks/main.yml          |  36 --
 .../files/es_templates/bro_index.template       | 218 -------
 .../files/es_templates/snort_index.template     | 183 ------
 .../files/es_templates/yaf_index.template       | 205 -------
 .../tasks/load_templates.yml                    |  53 --
 .../tasks/main.yml                              |  18 -
 .../roles/metron_hbase_tables/defaults/main.yml |  22 -
 .../roles/metron_hbase_tables/meta/main.yml     |  19 -
 .../roles/metron_hbase_tables/tasks/main.yml    |  24 -
 .../roles/metron_kafka_topics/defaults/main.yml |  30 -
 .../roles/metron_kafka_topics/meta/main.yml     |  19 -
 .../roles/metron_kafka_topics/tasks/main.yml    |  30 -
 metron-deployment/roles/monit/defaults/main.yml |   3 -
 metron-deployment/roles/monit/tasks/main.yml    |   3 +-
 .../roles/monit/tasks/monit-definitions.yml     |  20 -
 metron-deployment/roles/monit/tasks/scripts.yml |  55 --
 .../monit/templates/monit/elasticsearch.monit   |  23 -
 .../monit/templates/monit/enrichment.monit      |  26 -
 .../monit/indexing-elasticsearch.monit          |  26 -
 .../monit/templates/monit/indexing-solr.monit   |  26 -
 .../roles/monit/templates/monit/kibana.monit    |  22 -
 .../roles/monit/templates/monit/parsers.monit   |  49 --
 .../scripts/start_enrichment_topology.sh        |  25 -
 .../monit/templates/scripts/start_topology.sh   |  25 -
 .../monit/templates/scripts/status_topology.sh  |  33 --
 .../monit/templates/scripts/stop_topology.sh    |  23 -
 metron-deployment/roles/quick_dev/meta/main.yml |  19 +
 .../roles/quick_dev/tasks/main.yml              |  70 +++
 metron-deployment/roles/solr/defaults/main.yml  |  29 -
 metron-deployment/roles/solr/files/schema.xml   | 191 ------
 metron-deployment/roles/solr/meta/main.yml      |  21 -
 metron-deployment/roles/solr/tasks/main.yml     |  74 ---
 metron-deployment/roles/solr/templates/solr.xml |  52 --
 .../roles/solr/templates/solrconfig.xml         | 583 -------------------
 .../vagrant/full-dev-platform/Vagrantfile       |   4 +-
 .../vagrant/quick-dev-platform/Vagrantfile      |   6 +-
 152 files changed, 2214 insertions(+), 4571 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/Upgrading.md
----------------------------------------------------------------------
diff --git a/Upgrading.md b/Upgrading.md
index 92e958b..b05acbc 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -152,3 +152,11 @@ When using Ansible to deploy the latest Metron bits to an existing installation,
 #### Workaround
 Set the `topology.classpath` property for storm in Ambari to `/etc/hbase/conf:/etc/hadoop/conf`
 
+## 0.3.1 to PLACEHOLDER
+
+### [METRON-671: Refactor existing Ansible deployment to use Ambari MPack](https://issues.apache.org/jira/browse/METRON-671)
+
+#### Description
+Since the Ansible Deployment uses the MPack, RPMs must be built prior to deployment. As a result,
+[Docker](https://www.docker.com/) is required to perform a Quick-Dev, Full-Dev or Ansible deployment.
+This effectively limits the build environment to Docker supported [platforms](https://docs.docker.com/engine/installation/#platform-support-matrix).

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/README.md b/metron-deployment/README.md
index 3c9b339..ee44a2f 100644
--- a/metron-deployment/README.md
+++ b/metron-deployment/README.md
@@ -1,97 +1,51 @@
 # Overview
-This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster, Metron services, or both using ansible
-playbooks. These playbooks currently only target RHEL/CentOS 6.x operating
+This set of playbooks can be used to deploy an Ambari-managed Hadoop cluster containing Metron services using Ansible. These playbooks target RHEL/CentOS 6.x operating
 systems.
 
-In addition, an Ambari Management Pack can be built which can be deployed in conjuction with RPMs detailed in this README.
+Installation consists of -
+- Building Metron tarballs, RPMs and the Ambari MPack
+- Deploying Ambari
+- Leveraging Ambari to install:
+  * The required Hadoop Components
+  * Core Metron (Parsing, Enrichment, Indexing)
+  * Elasticsearch
+  * Kibana
+- Starting All Services
 
 ## Prerequisites
 The following tools are required to run these scripts:
 
 - [Maven](https://maven.apache.org/)
 - [Git](https://git-scm.com/)
-- [Ansible](http://www.ansible.com/) (version 2.0 or greater)
-
-Currently Metron must be built from source.  Before running these scripts perform the following steps:
-
-1. Clone the Metron git repository with `git clone git@github.com:apache/incubator-metron.git`
-2. Navigate to `incubator-metron` and run `mvn clean package`
+- [Ansible](http://www.ansible.com/) (version 2.0.0.2)
+- [Docker](https://www.docker.com/) (Docker for Mac on OSX)
 
 These scripts depend on two files for configuration:
 
 - hosts - declares which Ansible roles will be run on which hosts
 - group_vars/all - various configuration settings needed to install Metron
 
-Examples can be found in the
-`incubator-metron/metron-deployment/inventory/metron_example` directory and are a good starting point.  Copy this directory
-into `incubator-metron/metron-deployment/inventory/` and rename it to your `project_name`.  More information about Ansible files and directory
-structure can be found at http://docs.ansible.com/ansible/playbooks_best_practices.html.
-
+For production use, it is recommended that Metron be installed on an existing cluster managed by Ambari as described in the Installing Management Pack section below.
 ## Ambari
-The Ambari playbook will install a Hadoop cluster with all the services and configuration required by Metron.  This
-section can be skipped if installing Metron on a pre-existing cluster.
-
-Currently, this playbook supports building a local development cluster running on one node but options for other types
- of clusters will be added in the future.
-
-### Setting up your inventory
-Make sure to update the hosts file in `incubator-metron/metron-deployment/inventory/project_name/hosts` or provide an
-alternate inventory file when you launch the playbooks, including the
-ssh user(s) and ssh keyfile location(s). These playbooks expect two
-host groups:
-
-- ambari_master
-- ambari_slaves
-
-### Running the playbook
-This playbook will install the Ambari server on the ambari_master, install the ambari agents on
-the ambari_slaves, and create a cluster in Ambari with a blueprint for the required
-Metron components.
-
-Navigate to `incubator-metron/metron-deployment/playbooks` and run:
-`ansible-playbook -i ../inventory/project_name ambari_install.yml`
-
-## Metron
-The Metron playbook will gather the necessary cluster settings from Ambari and install the Metron services.
+The Ambari playbook will install a Hadoop cluster including the Metron Services (Parsing, Enrichment, Indexing). Ambari will also install Elasticsearch and Kibana.
 
-### Setting up your inventory
-Edit the hosts file at `incubator-metron/metron-deployment/inventory/project_name/hosts`.  Declare where which hosts the
-Metron services will be installed on by updating these groups:
-
-- enrichment - submits the topology code to Storm and requires a storm client
-- search - host where Elasticsearch will be run
-- web - host where the Metron UI and underlying services will run
-- sensors - host where network data will be collected and published to Kafka
-
-The Metron topologies depend on Kafka topics and HBase tables being created beforehand.  Declare a host that has Kafka and HBase clients installed by updating these groups:
-
-- metron_kafka_topics
-- metron_hbase_tables
-
-If only installing Metron, these groups can be ignored:
-
-- ambari_master
-- ambari_slaves
-
-### Configuring group variables
-The Metron Ansible scripts depend on a set of variables.  These variables can be found in the file at
-`incubator-metron/metron-deployment/inventory/project_name/group_vars/all`.  Edit the ambari* variables to match your Ambari
-instance and update the java_home variable to match the java path on your hosts.
-
-### Running the playbook
-Navigate to `incubator-metron/metron-deployment/playbooks` and run:
-`ansible-playbook -i ../inventory/project_name metron_install.yml`
+Currently, the playbooks supports building a local development cluster running on one node or deploying to a 10 node cluster on AWS EC2.
 
 ## Vagrant
-A VagrantFile is included and will install a working version of the entire Metron stack.  The following is required to
-run this:
+There are current two Vagrant modes, full-dev and quick-dev. Full-dev installs the entire Ambari/Metron stack. This is useful in testing out changes to the installation procedure.
+Quick-dev re-installs the core Metron Services (Parsing, Enrichment, and Indexing)on a pre-built instance.
+Use quick-dev for testing out changes to core Metron services.
 
-- [Vagrant](https://www.vagrantup.com/)
-- Hostmanager plugin for vagrant - Run `vagrant plugin install vagrant-hostmanager` on the machine where Vagrant is
+### Prerequsities
+- Install [Vagrant](https://www.vagrantup.com/) (5.0.16+)
+- Install the Hostmanager plugin for vagrant - Run `vagrant plugin install vagrant-hostmanager` on the machine where Vagrant is
 installed
 
-Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`.  This also provides a good
-example of how to run a full end-to-end Metron install.
+### Full-Dev
+Navigate to `incubator-metron/metron-deployment/vagrant/full-dev-platform` and run `vagrant up`.
+
+### Quick-Dev
+Navigate to `incubator-metron/metron-deployment/vagrant/quick-dev-platform` and run `vagrant up`.
 
 ## Ambari Management Pack
 An Ambari Management Pack can be built in order to make the Metron service available on top of an existing stack, rather than needing a direct stack update.

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index 84dbcce..c9b435b 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -17,19 +17,12 @@
 ---
 # which services should be started?
 services_to_start:
-  - elasticsearch
   - pcap-service
-  - kibana
   - yaf
   - snort
   - snort-logs
   - bro
   - pcap-replay
-  - yaf-parser
-  - bro-parser
-  - snort-parser
-  - enrichment
-  - indexing
 
 # the ami for cent6 by region
 amis_by_region:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml
index a0b15dc..fb19d89 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -26,12 +26,12 @@
     - include: tasks/create-vpc.yml
     - include: tasks/create-open-inbound-security-group.yml
     - include: tasks/create-open-outbound-security-group.yml
-    - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,metron,ec2
+    - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,ec2,monit
     - include: tasks/create-hosts.yml host_count=4 host_type=ambari_slave,ec2
-    - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,metron_kafka_topics,metron_hbase_tables,metron,ec2,pcap_server
-    - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2
-    - include: tasks/create-hosts.yml host_count=2 host_type=search,metron,ec2
-    - include: tasks/create-hosts.yml host_count=1 host_type=web,metron,ec2
+    - include: tasks/create-hosts.yml host_count=1 host_type=pcap_server,monit,ec2
+    - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2,zeppelin
+    - include: tasks/create-hosts.yml host_count=2 host_type=ambari_slave,search,ec2
+    - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,web,ec2
   tags:
     - ec2
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/amazon-ec2/run.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/run.sh b/metron-deployment/amazon-ec2/run.sh
index cad4fb7..bdaabf0 100755
--- a/metron-deployment/amazon-ec2/run.sh
+++ b/metron-deployment/amazon-ec2/run.sh
@@ -58,15 +58,10 @@ echo "=============================================================" >> $LOGFILE
 echo "Launching Metron[$ENV] @ $NOW"... >> $LOGFILE
 $DEPLOYDIR/../scripts/platform-info.sh >> $LOGFILE
 
-# build metron
-cd ../..
-mvn package -DskipTests -PHDP-2.5.0.0
-RC=$?; if [[ $RC != 0 ]]; then exit $RC; fi
-
 # deploy metron
 cd $DEPLOYDIR
 export EC2_INI_PATH=conf/ec2.ini
 ansible-playbook -i ec2.py playbook.yml \
-  --skip-tags="solr, sensor-stubs" \
+  --skip-tags="quick_dev,sensor-stubs" \
   --extra-vars="env=$ENV" \
   $EXTRA_ARGS

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/extra_modules/ambari_cluster_state.py
----------------------------------------------------------------------
diff --git a/metron-deployment/extra_modules/ambari_cluster_state.py b/metron-deployment/extra_modules/ambari_cluster_state.py
index a61d0f1..691d48e 100644
--- a/metron-deployment/extra_modules/ambari_cluster_state.py
+++ b/metron-deployment/extra_modules/ambari_cluster_state.py
@@ -385,9 +385,9 @@ def blueprint_var_to_ambari_converter(blueprint_var):
         this_host_map['hosts'] = this_host_list
         host_map.append(this_host_map)
     blueprint = dict()
+    blueprint['configurations'] = blueprint_var['required_configurations']
     blueprint['host_groups'] = new_groups
     blueprint['Blueprints'] = {'stack_name': blueprint_var['stack_name'], 'stack_version': blueprint_var['stack_version']}
-
     return blueprint, host_map
 
 from ansible.module_utils.basic import *

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/extra_modules/ambari_service_state.py
----------------------------------------------------------------------
diff --git a/metron-deployment/extra_modules/ambari_service_state.py b/metron-deployment/extra_modules/ambari_service_state.py
new file mode 100644
index 0000000..6dbefa8
--- /dev/null
+++ b/metron-deployment/extra_modules/ambari_service_state.py
@@ -0,0 +1,352 @@
+#!/usr/bin/python
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+DOCUMENTATION = '''
+---
+module: ambari_service_state
+version_added: "2.1"
+author: Apache Metron (Incubating : https://github.com/apache/incubator-metron )
+short_description: Start/Stop/Change Service or Component State
+description:
+    - Start/Stop/Change Service or Component State
+options:
+  host:
+    description:
+      The hostname for the ambari web server
+  port:
+    description:
+      The port for the ambari web server
+  username:
+    description:
+      The username for the ambari web server
+  password:
+    description:
+      The name of the cluster in web server
+    required: yes
+  cluster_name:
+    description:
+      The name of the cluster in ambari
+    required: yes
+  service_name:
+    description:
+      The name of the service to alter
+    required: no
+  component_name:
+    description:
+      The name of the component to alter
+    required: no
+  component_host:
+    description:
+      The host running the targeted component. Required when component_name is used.
+    required: no
+  state:
+    description:
+      The desired service/component state.
+  wait_for_complete:
+    description:
+      Whether to wait for the request to complete before returning. Default is False.
+    required: no
+  requirements: [ 'requests']
+'''
+
+EXAMPLES = '''
+# must use full relative path to any files in stored in roles/role_name/files/
+- name: Create a new ambari cluster
+    ambari_cluster_state:
+      host: localhost
+      port: 8080
+      username: admin
+      password: admin
+      cluster_name: my_cluster
+      cluster_state: present
+      blueprint_var: roles/my_role/files/blueprint.yml
+      blueprint_name: hadoop
+      wait_for_complete: True
+- name: Start the ambari cluster
+  ambari_cluster_state:
+    host: localhost
+    port: 8080
+    username: admin
+    password: admin
+    cluster_name: my_cluster
+    cluster_state: started
+    wait_for_complete: True
+- name: Stop the ambari cluster
+  ambari_cluster_state:
+    host: localhost
+    port: 8080
+    username: admin
+    password: admin
+    cluster_name: my_cluster
+    cluster_state: stopped
+    wait_for_complete: True
+- name: Delete the ambari cluster
+  ambari_cluster_state:
+    host: localhost
+    port: 8080
+    username: admin
+    password: admin
+    cluster_name: my_cluster
+    cluster_state: absent
+'''
+
+RETURN = '''
+results:
+    description: The content of the requests object returned from the RESTful call
+    returned: success
+    type: string
+'''
+
+__author__ = 'apachemetron'
+
+import json
+
+try:
+    import requests
+except ImportError:
+    REQUESTS_FOUND = False
+else:
+    REQUESTS_FOUND = True
+
+
+def main():
+
+    argument_spec = dict(
+        host=dict(type='str', default=None, required=True),
+        port=dict(type='int', default=None, required=True),
+        username=dict(type='str', default=None, required=True),
+        password=dict(type='str', default=None, required=True),
+        cluster_name=dict(type='str', default=None, required=True),
+        state=dict(type='str', default=None, required=True,
+                           choices=['started', 'stopped', 'deleted']),
+        service_name=dict(type='str', required=False),
+        component_name=dict(type='str', default=None, required=False),
+        component_host=dict(type='str', default=None, required=False),
+        wait_for_complete=dict(default=False, required=False, choices=BOOLEANS),
+    )
+
+    required_together = ['component_name', 'component_host']
+
+    module = AnsibleModule(
+        argument_spec=argument_spec,
+        required_together=required_together
+    )
+
+    if not REQUESTS_FOUND:
+        module.fail_json(
+            msg='requests library is required for this module')
+
+    p = module.params
+
+    host = p.get('host')
+    port = p.get('port')
+    username = p.get('username')
+    password = p.get('password')
+    cluster_name = p.get('cluster_name')
+    state = p.get('state')
+    service_name = p.get('service_name')
+    component_name = p.get('component_name')
+    component_host = p.get('component_host')
+    wait_for_complete = p.get('wait_for_complete')
+    component_mode = False
+    ambari_url = 'http://{0}:{1}'.format(host, port)
+
+    if component_name:
+        component_mode = True
+
+    try:
+        if not cluster_exists(ambari_url, username, password, cluster_name):
+            module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name))
+
+        if state in ['started', 'stopped', 'installed']:
+            desired_state = ''
+
+            if state == 'started':
+                desired_state = 'STARTED'
+            elif state in ['stopped','installed']:
+                desired_state = 'INSTALLED'
+
+            if component_mode:
+                if desired_state == 'INSTALLED':
+                    if(can_add_component(ambari_url, username, password, cluster_name, component_name, component_host)):
+                        add_component_to_host(ambari_url, username, password, cluster_name, component_name, component_host)
+                request = set_component_state(ambari_url, username, password, cluster_name, component_name, component_host, desired_state)
+            else:
+                request = set_service_state(ambari_url,username,password,cluster_name,service_name, desired_state)
+            if wait_for_complete:
+                try:
+                    request_id = json.loads(request.content)['Requests']['id']
+                except ValueError:
+                    module.exit_json(changed=True, results=request.content)
+                status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
+                if status != 'COMPLETED':
+                    module.fail_json(msg="Request failed with status {0}".format(status))
+            module.exit_json(changed=True, results=request.content)
+
+        elif state == 'deleted':
+            if component_mode:
+                request = delete_component(ambari_url, username, password, cluster_name, component_name, component_host)
+            else:
+                request = delete_service(ambari_url,username,password,cluster_name,service_name)
+            module.exit_json(changed=True, results=request.content)
+
+    except requests.ConnectionError, e:
+        module.fail_json(msg="Could not connect to Ambari client: " + str(e.message))
+    except Exception, e:
+        module.fail_json(msg="Ambari client exception occurred: " + str(e.message))
+
+
+def get_clusters(ambari_url, user, password):
+    r = get(ambari_url, user, password, '/api/v1/clusters')
+    if r.status_code != 200:
+        msg = 'Could not get cluster list: request code {0}, \
+                    request message {1}'.format(r.status_code, r.content)
+        raise Exception(msg)
+    clusters = json.loads(r.content)
+    return clusters['items']
+
+
+def cluster_exists(ambari_url, user, password, cluster_name):
+    clusters = get_clusters(ambari_url, user, password)
+    return cluster_name in [item['Clusters']['cluster_name'] for item in clusters]
+
+
+def get_request_status(ambari_url, user, password, cluster_name, request_id):
+    path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id)
+    r = get(ambari_url, user, password, path)
+    if r.status_code != 200:
+        msg = 'Could not get cluster request status: request code {0}, \
+                    request message {1}'.format(r.status_code, r.content)
+        raise Exception(msg)
+    service = json.loads(r.content)
+    return service['Requests']['request_status']
+
+
+def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time):
+    while True:
+        status = get_request_status(ambari_url, user, password, cluster_name, request_id)
+        if status == 'COMPLETED':
+            return status
+        elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']:
+            return status
+        else:
+            time.sleep(sleep_time)
+
+
+def set_service_state(ambari_url, user, password, cluster_name, service_name, desired_state):
+    path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name)
+    request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(service_name,desired_state)},
+               "Body": {"ServiceInfo": {"state": "{0}".format(desired_state)}}}
+    payload = json.dumps(request)
+    r = put(ambari_url, user, password, path, payload)
+    if r.status_code not in [202, 200]:
+        msg = 'Could not set service state: request code {0}, \
+                    request message {1}'.format(r.status_code, r.content)
+        raise Exception(msg)
+    return r
+
+
+def set_component_state(ambari_url, user, password, cluster_name, component_name, component_host, desired_state):
+    path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+    request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(component_name,desired_state)},
+               "Body": {"HostRoles": {"state": "{0}".format(desired_state)}}}
+    payload = json.dumps(request)
+    r = put(ambari_url, user, password, path, payload)
+    if r.status_code not in [202, 200]:
+        msg = 'Could not set component state: request code {0}, \
+                    request message {1}'.format(r.status_code, r.content)
+        raise Exception(msg)
+    return r
+
+
+def delete_component(ambari_url, user, password, cluster_name, component_name, component_host):
+    enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host)
+    path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+    r = delete(ambari_url,user,password,path)
+    if r.status_code not in [202, 200]:
+        msg = 'Could not set service state: request code {0}, \
+                    request message {1}'.format(r.status_code, r.content)
+        raise Exception(msg)
+    return r
+
+
+def enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host):
+    path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+    request = {"RequestInfo":{"context":"Turn On Maintenance Mode for {0}".format(component_name)},
+               "Body":{"HostRoles":{"maintenance_state":"ON"}}}
+    payload = json.dumps(request)
+    r = put(ambari_url, user, password, path, payload)
+    if r.status_code not in [202, 200]:
+        msg = 'Could not set maintenance mode: request code {0}, \
+                    request message {1}'.format(r.status_code, r.content)
+        raise Exception(msg)
+    return r
+
+
+def delete_service(ambari_url, user, password, cluster_name, service_name):
+    path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name)
+    r = delete(ambari_url,user,password,path)
+    if r.status_code not in [202, 200]:
+        msg = 'Could not delete service: request code {0}, \
+                    request message {1}'.format(r.status_code, r.content)
+        raise Exception(msg)
+    return r
+
+
+def add_component_to_host(ambari_url, user, password, cluster_name, component_name, component_host):
+    path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+    r = post(ambari_url, user, password, path,'')
+    if r.status_code not in [202,201,200]:
+        msg = 'Could not add {0} to host {1}: request code {2}, \
+                    request message {3}'.format(component_name,component_host,r.status_code, r.content)
+        raise Exception(msg)
+    return r
+
+
+def can_add_component(ambari_url, user, password, cluster_name, component_name, component_host):
+    path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+    r = get(ambari_url, user, password, path)
+    return r.status_code == 404
+
+
+def get(ambari_url, user, password, path):
+    r = requests.get(ambari_url + path, auth=(user, password))
+    return r
+
+
+def put(ambari_url, user, password, path, data):
+    headers = {'X-Requested-By': 'ambari'}
+    r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers)
+    return r
+
+
+def post(ambari_url, user, password, path, data):
+    headers = {'X-Requested-By': 'ambari'}
+    r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers)
+    return r
+
+
+def delete(ambari_url, user, password, path):
+    headers = {'X-Requested-By': 'ambari'}
+    r = requests.delete(ambari_url + path, auth=(user, password), headers=headers)
+    return r
+
+
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+    main()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 2b4617f..56ad75d 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -17,17 +17,11 @@
 
 # which services should be started?
 services_to_start:
-  - elasticsearch
   - pcap-service
-  - kibana
   - snort
   - snort-logs
   - bro
   - pcap-replay
-  - bro-parser
-  - snort-parser
-  - enrichment
-  - indexing
 
 # ambari
 ambari_host: "{{ groups.ambari_master[0] }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/full-dev-platform/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/hosts b/metron-deployment/inventory/full-dev-platform/hosts
index 98e8681..9bd9ea1 100644
--- a/metron-deployment/inventory/full-dev-platform/hosts
+++ b/metron-deployment/inventory/full-dev-platform/hosts
@@ -21,32 +21,27 @@ node1
 [ambari_slave]
 node1
 
-[metron_hbase_tables]
+[metron]
 node1
 
-[metron_kafka_topics]
+[search]
 node1
 
-[enrichment]
+[sensors]
 node1
 
-[search]
+[pcap_server]
 node1
 
 [web]
 node1
 
-[sensors]
+[zeppelin]
 node1
 
-[pcap_server]
-node1
-
-[metron:children]
-enrichment
-search
-web
+[monit:children]
 sensors
-metron_kafka_topics
-metron_hbase_tables
 pcap_server
+
+[local]
+127.0.0.1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all
deleted file mode 100644
index aeefa44..0000000
--- a/metron-deployment/inventory/metron_example/group_vars/all
+++ /dev/null
@@ -1,100 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-# which services should be started?
-services_to_start:
-  - elasticsearch
-  - pcap-service
-  - kibana
-  - yaf
-  - snort
-  - snort-logs
-  - bro
-  - pcap-replay
-  - yaf-parser
-  - bro-parser
-  - snort-parser
-  - enrichment
-  - indexing
-
-#Ansible Variables
-ansible_ssh_private_key_file: /Path/to/private/key/file #Change This
-ansible_ssh_user: root
-
-#Ambari
-ambari_host: "{{ groups.ambari_master[0] }}"
-ambari_port: 8080
-ambari_user: admin
-ambari_password: admin
-cluster_type: small_cluster
-
-# hbase
-pcap_hbase_table: pcap
-tracker_hbase_table: access_tracker
-threatintel_hbase_table: threatintel
-enrichment_hbase_table: enrichment
-
-# kafka
-num_partitions: 3
-retention_in_gb: 25
-
-# metron variables
-metron_version: 0.3.1
-metron_directory: /usr/metron/{{ metron_version }}
-pcapservice_port: 8081
-
-# sensors
-sniff_interface: eth1
-bro_version: "2.4.1"
-fixbuf_version: "1.7.1"
-yaf_version: "2.8.0"
-daq_version: "2.0.6-1"
-iface: "eth0"
-pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
-pycapa_home: "/opt/pycapa"
-snort_version: "2.9.8.0-1"
-snort_alert_csv_path: "/var/log/snort/alert.csv"
-
-# pcap-replay
-install_pcap_replay: True
-install_tap: True
-tap_ip: 10.0.0.1
-sensor_test_mode: True
-pcap_replay_interface: tap0
-pcap_replay_home: /opt/pcap-replay
-
-# data directories
-zookeeper_data_dir: "/data1/hadoop/zookeeper"
-namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
-namenode_name_dir: "/data1/hadoop/hdfs/namenode"
-datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
-journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
-nodemanager_local_dirs: "/data1/hadoop/yarn/local"
-timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
-timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
-nodemanager_log_dirs: "/data1/hadoop/yarn/log"
-jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
-storm_local_dir: "/data1/hadoop/storm"
-kafka_log_dirs: "/data2/kafka-log"
-elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
-
-# search
-install_elasticsearch: True
-install_solr: False
-elasticsearch_transport_port: 9300
-elasticsearch_network_interface: eth0
-elasticsearch_web_port: 9200

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/metron_example/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/hosts b/metron-deployment/inventory/metron_example/hosts
deleted file mode 100644
index c53466d..0000000
--- a/metron-deployment/inventory/metron_example/hosts
+++ /dev/null
@@ -1,63 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-[ambari_master]
-node1
-
-#minimum of 3 - 6 from 12 node cluser
-[ambari_slave]
-node2
-node3
-node4
-node5
-node6
-node7
-node8
-
-[metron_hbase_tables]
-node9
-
-[metron_kafka_topics]
-node9
-
-[pcap_server]
-node9
-
-#3rd ambari_slave
-[enrichment]
-node1
-
-#1 or more
-[search]
-node10
-node11
-node12
-
-#1 only
-[sensors]
-node1
-
-[web]
-node12
-
-[metron:children]
-enrichment
-search
-web
-sensors
-metron_kafka_topics
-metron_hbase_tables

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/quick-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/quick-dev-platform/group_vars/all b/metron-deployment/inventory/quick-dev-platform/group_vars/all
index d62e74b..e835d79 100644
--- a/metron-deployment/inventory/quick-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/quick-dev-platform/group_vars/all
@@ -17,17 +17,11 @@
 
 # which services should be started?
 services_to_start:
-  - elasticsearch
   - pcap-service
-  - kibana
   - snort
   - snort-logs
   - bro
   - pcap-replay
-  - bro-parser
-  - snort-parser
-  - enrichment
-  - indexing
 
 # ambari
 ambari_host: "{{ groups.ambari_master[0] }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/inventory/quick-dev-platform/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/quick-dev-platform/hosts b/metron-deployment/inventory/quick-dev-platform/hosts
index 98e8681..9bd9ea1 100644
--- a/metron-deployment/inventory/quick-dev-platform/hosts
+++ b/metron-deployment/inventory/quick-dev-platform/hosts
@@ -21,32 +21,27 @@ node1
 [ambari_slave]
 node1
 
-[metron_hbase_tables]
+[metron]
 node1
 
-[metron_kafka_topics]
+[search]
 node1
 
-[enrichment]
+[sensors]
 node1
 
-[search]
+[pcap_server]
 node1
 
 [web]
 node1
 
-[sensors]
+[zeppelin]
 node1
 
-[pcap_server]
-node1
-
-[metron:children]
-enrichment
-search
-web
+[monit:children]
 sensors
-metron_kafka_topics
-metron_hbase_tables
 pcap_server
+
+[local]
+127.0.0.1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
index 0b04f12..9c71887 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
@@ -4,9 +4,9 @@
     "general_deps" : {
         "_comment" : "dependencies for all cases",
         "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
-        "METRON_ENRICHMENT-INSTALL" : ["METRON_INDEXING-INSTALL"],
-        "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
-        "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START", "METRON_INDEXING-START"],
+        "METRON_ENRICHMENT-INSTALL": ["METRON_INDEXING-INSTALL"],
+        "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_ENRICHMENT_MASTER-START"],
+        "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
         "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
         "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
         "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
index 2939c28..a4de039 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-env.xml
@@ -23,12 +23,21 @@
     <name>elastic_user</name>
     <value>elasticsearch</value>
     <property-type>USER</property-type>
-    <description>The user for Elasticsearch</description>
+    <description>Service user for Elasticsearch</description>
+    <value-attributes>
+      <type>user</type>
+      <overridable>false</overridable>
+    </value-attributes>
   </property>
   <property>
-    <name>user_group</name>
+    <name>elastic_group</name>
     <value>elasticsearch</value>
-    <description>The group for Elasticsearch</description>
+    <property-type>GROUP</property-type>
+    <description>Service group for Elasticsearch</description>
+    <value-attributes>
+      <type>user</type>
+      <overridable>false</overridable>
+    </value-attributes>
   </property>
   <property>
     <name>elastic_log_dir</name>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
index 3e7b6a5..cbcebdd 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-site.xml
@@ -27,6 +27,14 @@
         <description>Cluster name identifies your cluster</description>
     </property>
     <property>
+        <name>masters_also_are_datanodes</name>
+        <value>"false"</value>
+        <description>ES Masters and Slaves cannot be installed on the same nodes.  Set this to "true" if you want the ES master nodes to serve as combined master/datanodes. Note: surround value in quotes.</description>
+        <value-attributes>
+            <type>string</type>
+        </value-attributes>
+    </property>
+    <property>
         <name>zen_discovery_ping_unicast_hosts</name>
         <!--Ideally this gets populated by the list of master eligible nodes (as an acceptable default).  Unsure how to do this.-->
         <value></value>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
index 58e4916..6ef09ac 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/configuration/elastic-sysconfig.xml
@@ -90,7 +90,7 @@ CONF_DIR={{conf_dir}}
 ES_USER={{elastic_user}}
 
 # Additional Java OPTS
-ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled
+ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled
 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps
 -XX:ErrorFile={{log_dir}}elasticsearch_err.log -XX:ParallelGCThreads=8"
         </value>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
index 25f98e5..53909a2 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/metainfo.xml
@@ -39,7 +39,7 @@
                     <name>ES_SLAVE</name>
                     <displayName>Elasticsearch Data Node</displayName>
                     <category>SLAVE</category>
-                    <cardinality>3+</cardinality>
+                    <cardinality>0+</cardinality>
                     <commandScript>
                         <script>scripts/elastic_slave.py</script>
                         <scriptType>PYTHON</scriptType>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
index a472655..2e638de 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/elastic.py
@@ -22,12 +22,12 @@ from resource_management.core.resources.system import Directory
 from resource_management.core.resources.system import File
 from resource_management.core.source import InlineTemplate
 from resource_management.core.source import Template
-
+from resource_management.core.resources import User
 
 def elastic():
-    print "INSIDE THE %s" % __file__
     import params
 
+    User(params.elastic_user, action = "create", groups = params.elastic_group)
     params.path_data = params.path_data.replace('"', '')
     data_path = params.path_data.replace(' ', '').split(',')
     data_path[:] = [x.replace('"', '') for x in data_path]
@@ -37,15 +37,15 @@ def elastic():
 
     Directory(directories,
               create_parents=True,
-              # recursive=True,
               mode=0755,
               owner=params.elastic_user,
-              group=params.elastic_user
+              group=params.elastic_group
               )
 
     print "Master env: ""{0}/elastic-env.sh".format(params.conf_dir)
     File("{0}/elastic-env.sh".format(params.conf_dir),
          owner=params.elastic_user,
+         group=params.elastic_group,
          content=InlineTemplate(params.elastic_env_sh_template)
          )
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
index 8b890d4..aea81fb 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/params.py
@@ -23,6 +23,7 @@ from resource_management.libraries.script import Script
 # server configurations
 config = Script.get_config()
 
+masters_also_are_datanodes = config['configurations']['elastic-site']['masters_also_are_datanodes']
 elastic_home = config['configurations']['elastic-sysconfig']['elastic_home']
 data_dir = config['configurations']['elastic-sysconfig']['data_dir']
 work_dir = config['configurations']['elastic-sysconfig']['work_dir']
@@ -32,7 +33,7 @@ max_open_files = config['configurations']['elastic-sysconfig']['max_open_files']
 max_map_count = config['configurations']['elastic-sysconfig']['max_map_count']
 
 elastic_user = config['configurations']['elastic-env']['elastic_user']
-user_group = config['configurations']['elastic-env']['user_group']
+elastic_group = config['configurations']['elastic-env']['elastic_group']
 log_dir = config['configurations']['elastic-env']['elastic_log_dir']
 pid_dir = '/var/run/elasticsearch'
 pid_file = '/var/run/elasticsearch/elasticsearch.pid'

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
index 90d4d3d..15a5cf5 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/scripts/slave.py
@@ -22,11 +22,12 @@ from resource_management.core.resources.system import Directory
 from resource_management.core.resources.system import File
 from resource_management.core.source import InlineTemplate
 from resource_management.core.source import Template
-
+from resource_management.core.resources import User
 
 def slave():
     import params
 
+    User(params.elastic_user, action = "create", groups = params.elastic_group)
     params.path_data = params.path_data.replace('"', '')
     data_path = params.path_data.replace(' ', '').split(',')
     data_path[:] = [x.replace('"', '') for x in data_path]
@@ -38,7 +39,7 @@ def slave():
               create_parents=True,
               mode=0755,
               owner=params.elastic_user,
-              group=params.elastic_user,
+              group=params.elastic_group,
               cd_access="a"
               )
 
@@ -54,7 +55,7 @@ def slave():
              "elasticsearch.slave.yaml.j2",
              configurations=configurations),
          owner=params.elastic_user,
-         group=params.elastic_user
+         group=params.elastic_group
          )
 
     print "Master sysconfig: /etc/sysconfig/elasticsearch"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
index 8d92f1b..ca3630a 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/2.3.3/package/templates/elasticsearch.master.yaml.j2
@@ -28,12 +28,12 @@ discovery:
   zen:
     ping:
       multicast:
-        enabled: {{discovery_zen_ping_multicast_enabled}}
+        enabled: {{ discovery_zen_ping_multicast_enabled }}
       unicast:
         hosts: "{{zen_discovery_ping_unicast_hosts}}"
 
 node:
-  data: false
+  data: {{ masters_also_are_datanodes }}
   master: true
   name: {{hostname}}
 path:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
index 972945d..0adf32a 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-env.xml
@@ -23,13 +23,21 @@
     <name>kibana_user</name>
     <value>kibana</value>
     <property-type>USER</property-type>
-    <description></description>
+    <description>Service User for Kibana</description>
+    <value-attributes>
+      <type>user</type>
+      <overridable>false</overridable>
+    </value-attributes>
   </property>
   <property>
-    <name>user_group</name>
+    <name>kabana_group</name>
     <value>kibana</value>
     <property-type>GROUP</property-type>
-    <description></description>
+    <description>Service Group for Kibana</description>
+    <value-attributes>
+      <type>user</type>
+      <overridable>false</overridable>
+    </value-attributes>
   </property>
   <property require-input="true">
     <name>kibana_log_dir</name>
@@ -51,4 +59,9 @@
     <value>5000</value>
     <description>Kibana back end server port to use.</description>
   </property>
+  <property require-input="true">
+    <name>kibana_default_application</name>
+    <value>default</value>
+    <description>The default application to load.</description>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
index c546e2c..4373d14 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/configuration/kibana-site.xml
@@ -50,7 +50,7 @@ elasticsearch.url: {{ es_url }}
 # kibana.index: ".kibana"
 
 # The default application to load.
-# kibana.defaultAppId: "discover"
+kibana.defaultAppId: "{{ kibana_default_application }}"
 
 # If your Elasticsearch is protected with basic auth, these are the user credentials
 # used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
index ef1e597..a5e0ec9 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/4.5.1/package/scripts/params.py
@@ -33,7 +33,7 @@ kibana_bin = '/usr/share/kibana/bin/'
 
 conf_dir = "/opt/kibana/config/"
 kibana_user = config['configurations']['kibana-env']['kibana_user']
-user_group = config['configurations']['kibana-env']['user_group']
+kibana_group = config['configurations']['kibana-env']['kibana_group']
 log_dir = config['configurations']['kibana-env']['kibana_log_dir']
 pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
 pid_file = format("{pid_dir}/kibanasearch.pid")
@@ -42,6 +42,8 @@ parsed = urlparse(es_url)
 es_host = parsed.netloc.split(':')[0]
 es_port = parsed.netloc.split(':')[1]
 kibana_port = config['configurations']['kibana-env']['kibana_server_port']
+kibana_default_application = config['configurations']['kibana-env']['kibana_default_application']
 hostname = config['hostname']
 java64_home = config['hostLevelParams']['java_home']
 kibana_yml_template = config['configurations']['kibana-site']['content']
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index cbff4a9..277b636 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -46,15 +46,23 @@
         <name>metron_user</name>
         <value>metron</value>
         <property-type>USER</property-type>
-        <description>The user for Metron</description>
-        <display-name>Metron User</display-name>
+        <description>Service User for Metron</description>
+        <display-name>Metron User Name</display-name>
+        <value-attributes>
+            <type>user</type>
+            <overridable>false</overridable>
+        </value-attributes>
     </property>
     <property>
         <name>metron_group</name>
         <value>metron</value>
         <property-type>GROUP</property-type>
-        <description>The group for Metron</description>
+        <description>Service Group for Metron</description>
         <display-name>Metron Group Name</display-name>
+        <value-attributes>
+            <type>user</type>
+            <overridable>false</overridable>
+        </value-attributes>
     </property>
     <property>
         <name>metron_topic_retention</name>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index 817f266..5878dc6 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -28,7 +28,11 @@ class EnrichmentCommands:
     __params = None
     __enrichment_topology = None
     __enrichment_topic = None
-    __configured = False
+    __enrichment_error_topic = None
+    __threat_intel_error_topic = None
+    __kafka_configured = False
+    __hbase_configured = False
+    __geo_configured = False
 
     def __init__(self, params):
         if params is None:
@@ -36,13 +40,36 @@ class EnrichmentCommands:
         self.__params = params
         self.__enrichment_topology = params.metron_enrichment_topology
         self.__enrichment_topic = params.metron_enrichment_topic
-        self.__configured = os.path.isfile(self.__params.enrichment_configured_flag_file)
+        self.__kafka_configured = os.path.isfile(self.__params.enrichment_kafka_configured_flag_file)
+        self.__hbase_configured = os.path.isfile(self.__params.enrichment_hbase_configured_flag_file)
+        self.__geo_configured = os.path.isfile(self.__params.enrichment_geo_configured_flag_file)
 
-    def is_configured(self):
-        return self.__configured
+    def is_kafka_configured(self):
+        return self.__kafka_configured
 
-    def set_configured(self):
-        File(self.__params.enrichment_configured_flag_file,
+    def set_kafka_configured(self):
+        Logger.info("Setting Kafka Configured to True")
+        File(self.__params.enrichment_kafka_configured_flag_file,
+             content="",
+             owner=self.__params.metron_user,
+             mode=0775)
+
+    def is_hbase_configured(self):
+        return self.__hbase_configured
+
+    def set_hbase_configured(self):
+        Logger.info("Setting HBase Configured to True")
+        File(self.__params.enrichment_hbase_configured_flag_file,
+             content="",
+             owner=self.__params.metron_user,
+             mode=0775)
+
+    def is_geo_configured(self):
+        return self.__geo_configured
+
+    def set_geo_configured(self):
+        Logger.info("Setting GEO Configured to True")
+        File(self.__params.enrichment_geo_configured_flag_file,
              content="",
              owner=self.__params.metron_user,
              mode=0775)
@@ -84,7 +111,7 @@ class EnrichmentCommands:
                                    type="directory",
                                    action="create_on_execute",
                                    owner=self.__params.metron_user,
-                                   group=self.__params.user_group,
+                                   group=self.__params.metron_group,
                                    mode=0775,
                                    )
 
@@ -101,6 +128,7 @@ class EnrichmentCommands:
         Logger.info("Executing command " + command)
         Execute(command, user=self.__params.metron_user, tries=1, logoutput=True)
         Logger.info("Done intializing GeoIP data")
+        self.set_geo_configured()
 
     def init_kafka_topics(self):
         Logger.info('Creating Kafka topics')
@@ -128,6 +156,7 @@ class EnrichmentCommands:
                                             retention_bytes))
 
         Logger.info("Done creating Kafka topics")
+        self.set_kafka_configured()
 
     def start_enrichment_topology(self):
         Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
@@ -176,6 +205,7 @@ class EnrichmentCommands:
         return active
 
     def create_hbase_tables(self):
+        Logger.info("Creating HBase Tables")
         add_enrichment_cmd = "echo \"create '{0}','{1}'\" | hbase shell -n".format(self.__params.enrichment_table, self.__params.enrichment_cf)
         Execute(add_enrichment_cmd,
                 tries=3,
@@ -191,3 +221,5 @@ class EnrichmentCommands:
                 logoutput=False,
                 path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
                 )
+        Logger.info("Done creating HBase Tables")
+        self.set_hbase_configured()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index b02ba5c..362b2e2 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -48,14 +48,14 @@ class Enrichment(Script):
         from params import params
         env.set_params(params)
         commands = EnrichmentCommands(params)
-
         metron_service.load_global_config(params)
 
-        if not commands.is_configured():
+        if not commands.is_kafka_configured():
             commands.init_kafka_topics()
+        if not commands.is_hbase_configured():
             commands.create_hbase_tables()
+        if not commands.is_geo_configured():
             commands.init_geo()
-            commands.set_configured()
 
         commands.start_enrichment_topology()
 
@@ -79,12 +79,6 @@ class Enrichment(Script):
         commands = EnrichmentCommands(params)
         commands.restart_enrichment_topology(env)
 
-    def kafkabuild(self, env, upgrade_type=None):
-        from params import params
-        env.set_params(params)
-        commands = EnrichmentCommands(params)
-        commands.init_kafka_topics()
-
 
 if __name__ == "__main__":
     Enrichment().execute()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 8ed3f9a..50af3b2 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -101,7 +101,7 @@ class IndexingCommands:
                                    type="directory",
                                    action="create_on_execute",
                                    owner=self.__params.metron_user,
-                                   group=self.__params.user_group,
+                                   group=self.__params.metron_group,
                                    mode=0775,
                                    )
         Logger.info('Done creating HDFS indexing directory')

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 2b8276b..87a1f39 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -39,7 +39,7 @@ config = Script.get_config()
 tmp_dir = Script.get_tmp_dir()
 
 hostname = config['hostname']
-user_group = config['configurations']['cluster-env']['user_group']
+metron_group = config['configurations']['cluster-env']['metron_group']
 metron_home = status_params.metron_home
 parsers = status_params.parsers
 geoip_url = config['configurations']['metron-env']['geoip_url']
@@ -51,7 +51,9 @@ metron_config_path = metron_home + '/config'
 metron_zookeeper_config_dir = status_params.metron_zookeeper_config_dir
 metron_zookeeper_config_path = status_params.metron_zookeeper_config_path
 parsers_configured_flag_file = status_params.parsers_configured_flag_file
-enrichment_configured_flag_file = status_params.enrichment_configured_flag_file
+enrichment_kafka_configured_flag_file = status_params.enrichment_kafka_configured_flag_file
+enrichment_hbase_configured_flag_file = status_params.enrichment_hbase_configured_flag_file
+enrichment_geo_configured_flag_file = status_params.enrichment_geo_configured_flag_file
 indexing_configured_flag_file = status_params.indexing_configured_flag_file
 global_json_template = config['configurations']['metron-env']['global-json']
 global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 961102f..83b4fa4 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -45,7 +45,9 @@ metron_indexing_topology = config['configurations']['metron-env']['metron_indexi
 indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
 
 # Enrichment
-enrichment_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_configured'
+enrichment_kafka_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_kafka_configured'
+enrichment_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_hbase_configured'
+enrichment_geo_configured_flag_file = metron_zookeeper_config_path + '/../metron_enrichment_geo_configured'
 
 # Storm
 storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/.gitignore b/metron-deployment/packaging/packer-build/.gitignore
new file mode 100644
index 0000000..68bf87b
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/.gitignore
@@ -0,0 +1,12 @@
+.bundle/
+iso
+*.box
+virtualfloppy.vfd
+packer_cache
+packer.log
+.DS_Store
+/packer-*/
+*.variables.json
+/builds/
+.kitchen
+.kitchen.*.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/README.md b/metron-deployment/packaging/packer-build/README.md
new file mode 100644
index 0000000..d0c5484
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/README.md
@@ -0,0 +1,51 @@
+Build Metron Images
+=========================
+
+Based on the fantastic [Bento](https://github.com/chef/bento) project developed by Chef.
+
+Images Provided
+---------------------
+- base-centos-6.7: Centos 6.7 + HDP. Used in the full-dev-platform Vagrant image
+- quick-dev-centos-6.7: Centos 6.7 + HDP + Metron. Used for the quick-dev-platform Vagrant image.
+
+Prerequisites
+---------------------
+- [Packer](https://www.packer.io/) 0.12.2
+- [Virtualbox](https://www.virtualbox.org/) 5.0.16+ (Tested with 5.0.20)
+
+Build Both Images
+---------------------- 
+  Navigate to \<your-project-directory\>/metron-deployment/packer-build
+  Execute bin/bento build
+  
+  Packer will build both images and export .box files to the ./builds directory.
+  
+Build Single Images
+---------------------- 
+ Navigate to *your-project-directory*/metron-deployment/packer-build
+ * Base Centos (full-dev)
+ ```
+bin/bento build base-centos-6.7.json
+```
+ * Quick Dev
+ ```
+bin/bento build quick-dev-centos-6.7.json
+```
+
+Using Your New Box File
+---------------------- 
+Modify the relevant Vagrantfile (full-dev-platform or quick-dev-platform) replacing the lines:
+```
+<pre><code>config.vm.box = "<i>box_name</i>"
+config.ssh.insert_key = true</code></pre>
+```
+with
+```
+<pre></code>config.vm.box = "<i>test_box_name</i>"
+config.vm.box = "<i>PathToBoxfile/Boxfilename</i>"
+config.ssh.insert_key = true</code></pre>
+```
+Launch the image as usual.
+
+Node: Vagrant will cache boxes, you can force Vagrant to reload your box by running <code>vagrant box remove <i>test_box_name</i></code> before launching your new image.
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/ansible.cfg b/metron-deployment/packaging/packer-build/ansible.cfg
new file mode 100644
index 0000000..eeb5084
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/ansible.cfg
@@ -0,0 +1,27 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+[defaults]
+host_key_checking = false
+library = ../../extra_modules
+roles_path = ../../roles
+pipelining = True
+remote_user = vagrant
+log_path = ./ansible.log
+
+# fix for "ssh throws 'unix domain socket too long' " problem
+[ssh_connection]
+control_path =  ~/.ssh/ansible-ssh-%%C

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/ansible/playbook.yml b/metron-deployment/packaging/packer-build/ansible/playbook.yml
new file mode 100644
index 0000000..0aba491
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/ansible/playbook.yml
@@ -0,0 +1,28 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- include: ../../../playbooks/metron_full_install.yml
+
+- hosts: packer
+  become: true
+  roles:
+    - role: ambari_gather_facts
+  tasks:
+    - include: tasks/cleanup.yml
+  tags:
+    - packer-cleanup
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml b/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml
new file mode 100644
index 0000000..cdeab42
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/ansible/tasks/cleanup.yml
@@ -0,0 +1,42 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Stop Monit
+  service:
+    name: monit
+    state: stopped
+  register: command_result
+  failed_when: "command_result|failed and 'no service or tool found for: monit' not in command_result.msg"
+
+- name: Delete ES Indices
+  uri:
+    method: DELETE
+    url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/b*,s*,y*"
+
+- name: Stop the ambari cluster
+  ambari_cluster_state:
+    host: "{{ groups.ambari_master[0] }}"
+    port: "{{ ambari_port }}"
+    username: "{{ ambari_user }}"
+    password: "{{ ambari_password }}"
+    cluster_name: "{{ cluster_name }}"
+    cluster_state: stopped
+    wait_for_complete: true
+
+- include: remove_services.yml
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68a334a8/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml b/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml
new file mode 100644
index 0000000..9d2477c
--- /dev/null
+++ b/metron-deployment/packaging/packer-build/ansible/tasks/remove_services.yml
@@ -0,0 +1,46 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Remove Secondary Namenode
+  uri:
+    url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/hosts/node1/host_components/SECONDARY_NAMENODE"
+    method: DELETE
+    HEADER_X-Requested-By: "{{ ambari_user }}"
+    user: "{{ambari_user }}"
+    password: "{{ ambari_password }}"
+    force_basic_auth: yes
+    status_code: 200,201,404
+
+- name: Remove MapReduce2 Service
+  uri:
+    url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/services/MAPREDUCE2"
+    method: DELETE
+    HEADER_X-Requested-By: "{{ ambari_user }}"
+    user: "{{ambari_user }}"
+    password: "{{ ambari_password }}"
+    force_basic_auth: yes
+    status_code: 200,201,404
+
+- name: Remove YARN Service
+  uri:
+    url: "http://{{ambari_host}}:{{ambari_port}}/api/v1/clusters/metron_cluster/services/YARN"
+    method: DELETE
+    HEADER_X-Requested-By: "{{ ambari_user }}"
+    user: "{{ambari_user }}"
+    password: "{{ ambari_password }}"
+    force_basic_auth: yes
+    status_code: 200,201,404



Mime
View raw message