metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [40/43] incubator-metron git commit: METRON-76-Deployment Metron deploy has intermittent failures on EC2 closes apache/incubator-metron#51
Date Mon, 21 Mar 2016 17:06:22 GMT
METRON-76-Deployment Metron deploy has intermittent failures on EC2 closes apache/incubator-metron#51


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/68aab6e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/68aab6e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/68aab6e9

Branch: refs/heads/Metron_0.1BETA
Commit: 68aab6e9adc42b3a21701d1fca2c1a3a81e14237
Parents: c737aa9
Author: dlyle65535 <dlyle65535@gmail.com>
Authored: Sat Mar 19 14:11:38 2016 -0700
Committer: james-sirota <James.Sirota@yahoo.com>
Committed: Sat Mar 19 14:11:38 2016 -0700

----------------------------------------------------------------------
 deployment/amazon-ec2/playbook.yml              | 16 ++++++-
 deployment/amazon-ec2/tasks/check-hosts.yml     | 20 ++++++++
 deployment/amazon-ec2/tasks/create-hosts.yml    |  4 --
 deployment/amazon-ec2/tasks/expand-volume.yml   |  4 +-
 deployment/amazon-ec2/tasks/mount-volume.yml    |  9 +++-
 .../inventory/singlenode-vagrant/group_vars/all | 11 +++--
 deployment/playbooks/ambari_install.yml         |  1 +
 deployment/playbooks/metron_install.yml         |  4 --
 deployment/roles/ambari_common/meta/main.yml    | 22 +++++++++
 deployment/roles/ambari_common/tasks/main.yml   | 20 --------
 deployment/roles/ambari_config/meta/main.yml    | 21 +++++++++
 deployment/roles/ambari_config/tasks/main.yml   | 10 +++-
 .../ambari_config/tasks/start_services.yml      | 48 ++++++++++++++++++++
 .../vars/multi_vagrant_cluster.yml              |  2 +
 .../roles/ambari_config/vars/single_node_vm.yml |  1 +
 .../roles/ambari_config/vars/small_cluster.yml  |  2 +
 .../roles/ambari_gather_facts/meta/main.yml     | 21 +++++++++
 .../roles/ambari_gather_facts/tasks/main.yml    | 10 ----
 deployment/roles/ambari_master/tasks/main.yml   | 29 ++++++++----
 deployment/roles/ambari_slave/tasks/main.yml    |  8 +++-
 deployment/roles/bro/meta/main.yml              | 21 +++++++++
 deployment/roles/bro/tasks/dependencies.yml     |  5 +-
 deployment/roles/build-tools/meta/main.yml      | 19 ++++++++
 deployment/roles/build-tools/tasks/main.yml     | 34 ++++++++++++++
 .../elasticsearch/files/yaf_index.template      | 36 +++++++++++++++
 .../roles/elasticsearch/handlers/main.yml       | 19 --------
 deployment/roles/elasticsearch/meta/main.yml    | 24 ++++++++++
 .../elasticsearch/tasks/configure_index.yml     | 44 ++++++++++++++++++
 deployment/roles/elasticsearch/tasks/main.yml   | 17 +++----
 deployment/roles/epel/tasks/main.yml            | 30 ++++++++++++
 deployment/roles/flume/meta/main.yml            | 20 ++++++++
 deployment/roles/flume/tasks/main.yml           |  5 +-
 deployment/roles/hadoop_setup/meta/main.yml     | 20 ++++++++
 deployment/roles/httplib2/tasks/main.yml        | 20 ++++++++
 deployment/roles/java_jdk/defaults/main.yml     | 18 ++++++++
 deployment/roles/java_jdk/tasks/main.yml        | 34 ++++++++++++++
 .../roles/libselinux-python/tasks/main.yml      | 25 ++++++++++
 deployment/roles/metron_common/meta/main.yml    | 22 +++++++++
 deployment/roles/metron_common/tasks/main.yml   | 20 ++------
 .../roles/metron_pcapservice/meta/main.yml      | 19 ++++++++
 .../metron_pcapservice/tasks/config-java.yml    | 34 --------------
 .../roles/metron_pcapservice/tasks/main.yml     |  2 -
 deployment/roles/metron_streaming/meta/main.yml | 20 ++++++++
 .../metron_streaming/tasks/grok_upload.yml      | 37 +++++++++++++++
 .../metron_streaming/tasks/hdfs_filesystem.yml  | 41 +++++++++++++++++
 .../roles/metron_streaming/tasks/main.yml       | 27 +++--------
 deployment/roles/metron_ui/tasks/main.yml       |  9 +++-
 deployment/roles/mysql_server/tasks/main.yml    |  8 ++++
 deployment/roles/ntp/tasks/main.yml             | 31 +++++++++++++
 deployment/roles/pcap_replay/meta/main.yml      | 19 ++------
 deployment/roles/pcap_replay/tasks/main.yml     |  3 --
 .../roles/pcap_replay/tasks/tcpreplay.yml       |  7 ---
 deployment/roles/pycapa/meta/main.yml           | 21 +++++++++
 deployment/roles/pycapa/tasks/main.yml          | 22 +++++----
 deployment/roles/python-pip/tasks/main.yml      | 25 ++++++++++
 deployment/roles/snort/meta/main.yml            | 22 +++++++++
 deployment/roles/snort/tasks/daq.yml            |  8 +++-
 deployment/roles/snort/tasks/main.yml           | 16 -------
 deployment/roles/snort/tasks/snort.yml          | 28 +++++-------
 deployment/roles/tap_interface/tasks/main.yml   |  9 +++-
 deployment/roles/yaf/meta/main.yml              | 22 +++++++++
 deployment/roles/yaf/tasks/dependencies.yml     | 31 -------------
 deployment/roles/yaf/tasks/kafka-client.yml     |  9 ++--
 deployment/roles/yaf/tasks/main.yml             |  1 -
 deployment/roles/yum-update/tasks/main.yml      | 26 +++++++++++
 65 files changed, 926 insertions(+), 267 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/playbook.yml b/deployment/amazon-ec2/playbook.yml
index d76ac4d..5914b98 100644
--- a/deployment/amazon-ec2/playbook.yml
+++ b/deployment/amazon-ec2/playbook.yml
@@ -27,15 +27,27 @@
     - include: tasks/create-open-inbound-security-group.yml
     - include: tasks/create-open-outbound-security-group.yml
     - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,metron,ec2
-    - include: tasks/create-hosts.yml host_count=5 host_type=ambari_slave,ec2
+    - include: tasks/create-hosts.yml host_count=4 host_type=ambari_slave,ec2
     - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,hadoop_client,metron,ec2
     - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2
-    - include: tasks/create-hosts.yml host_count=3 host_type=search,metron,ec2
+    - include: tasks/create-hosts.yml host_count=2 host_type=search,metron,ec2
     - include: tasks/create-hosts.yml host_count=1 host_type=web,mysql,metron,ec2
   tags:
     - ec2
 
 #
+# wait for all ec2 hosts to come up
+#
+- hosts: ec2
+  become: True
+  vars_files:
+    - conf/defaults.yml
+  gather_facts: False
+  tasks:
+    - include: tasks/check-hosts.yml
+  tags:
+    - ec2
+#
 # mount additional data volumes on all ec2 hosts
 #
 - hosts: ec2

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/amazon-ec2/tasks/check-hosts.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/check-hosts.yml b/deployment/amazon-ec2/tasks/check-hosts.yml
new file mode 100644
index 0000000..1a4b2c7
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/check-hosts.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Wait for connectivity to host(s)
+  local_action: wait_for host={{ inventory_hostname }} state=started timeout=300 delay=10
+  become: False

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/amazon-ec2/tasks/create-hosts.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-hosts.yml b/deployment/amazon-ec2/tasks/create-hosts.yml
index bea6b24..e1ff3e9 100644
--- a/deployment/amazon-ec2/tasks/create-hosts.yml
+++ b/deployment/amazon-ec2/tasks/create-hosts.yml
@@ -53,7 +53,3 @@
   with_items: ec2.tagged_instances
   when: item.public_dns_name is defined
 
-- name: Wait for connectivity to host(s)
-  wait_for: host={{ item.public_dns_name }} port=22 search_regex=OpenSSH
-  with_items: ec2.tagged_instances
-  when: item.public_dns_name is defined

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/amazon-ec2/tasks/expand-volume.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/expand-volume.yml b/deployment/amazon-ec2/tasks/expand-volume.yml
index 41cd442..1e25e27 100644
--- a/deployment/amazon-ec2/tasks/expand-volume.yml
+++ b/deployment/amazon-ec2/tasks/expand-volume.yml
@@ -27,6 +27,4 @@
   poll: 0
   ignore_errors: True
 
-- name: Wait for connectivity to host(s)
-  local_action: wait_for host={{ inventory_hostname }} state=started
-  become: False
+- include: tasks/check-hosts.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/amazon-ec2/tasks/mount-volume.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/mount-volume.yml b/deployment/amazon-ec2/tasks/mount-volume.yml
index 9609816..11259a5 100644
--- a/deployment/amazon-ec2/tasks/mount-volume.yml
+++ b/deployment/amazon-ec2/tasks/mount-volume.yml
@@ -16,7 +16,14 @@
 #
 ---
 - name: Install xfsprogs
-  yum: name=xfsprogs
+  yum:
+    name: xfsprogs
+    state: present
+    update_cache: yes
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Format data volume(s)
   filesystem: fstype=xfs dev={{ vol_src }}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index 06acaa9..a5e0af7 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -28,11 +28,6 @@ pcap_hbase_table: pcap
 tracker_hbase_table: access_tracker
 threatintel_ip_hbase_table: malicious_ip
 
-#elasticsearch
-elasticsearch_transport_port: 9300
-elasticsearch_network_interface: eth1
-elasticsearch_web_port: 9200
-
 # metron variables
 metron_version: 0.1BETA
 metron_directory: /usr/metron/{{ metron_version }}
@@ -78,3 +73,9 @@ storm_topologies:
     - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
     - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
 pcapservice_port: 8081
+
+#Search
+install_elasticsearch: True
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth1
+elasticsearch_web_port: 9200

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/ambari_install.yml b/deployment/playbooks/ambari_install.yml
index e6d226b..685753c 100644
--- a/deployment/playbooks/ambari_install.yml
+++ b/deployment/playbooks/ambari_install.yml
@@ -47,6 +47,7 @@
     - hdp-install
 
 - hosts: ambari_master
+  become: true
   roles:
     - role: ambari_config
   tags:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index bfdde54..c8c181e 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -32,7 +32,6 @@
 - hosts: hadoop_client
   become: true
   roles:
-    - role: ambari_gather_facts
     - role: hadoop_setup
   tags:
     - metron-prereqs
@@ -63,7 +62,6 @@
 - hosts: sensors
   become: true
   roles:
-    - role: ambari_gather_facts
     - { role: tap_interface, when: install_tap | default(False) == True }
     - { role: pycapa, when: install_pycapa | default(True) == True }
     - { role: bro, when: install_bro | default(True) == True }
@@ -77,7 +75,6 @@
 - hosts: enrichment
   become: true
   roles:
-    - role: ambari_gather_facts
     - role: metron_streaming
   tags:
     - enrichment
@@ -85,7 +82,6 @@
 - hosts: web
   become: true
   roles:
-    - role: ambari_gather_facts
     - role: metron_ui
     - role: metron_pcapservice
   tags:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_common/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/meta/main.yml b/deployment/roles/ambari_common/meta/main.yml
new file mode 100644
index 0000000..8992ac1
--- /dev/null
+++ b/deployment/roles/ambari_common/meta/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - libselinux-python
+  - yum-update
+  - epel
+  - ntp

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/tasks/main.yml b/deployment/roles/ambari_common/tasks/main.yml
index 35f3fce..b1b1659 100644
--- a/deployment/roles/ambari_common/tasks/main.yml
+++ b/deployment/roles/ambari_common/tasks/main.yml
@@ -19,20 +19,8 @@
   fail: msg="Ambari HDP deployment supports CentOS 6 only."
   when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6")
 
-- name: Install libselinux-python
-  yum: name=libselinux-python
-
 - include: passwd_less_ssh.yml
 
-- name: upgrade all packages
-  yum: name=* state=latest
-
-- name: install ntp
-  yum: name=ntp state=present
-
-- name: Ensure ntp is running and enabled
-  service: name=ntpd state=started enabled=yes
-
 - name: Ensure iptables is stopped and is not running at boot time.
   ignore_errors: yes
   service: name=iptables state=stopped enabled=no
@@ -53,14 +41,6 @@
 - name: Add localhost to /etc/hosts
   lineinfile: dest=/etc/hosts line="127.0.0.1   localhost"
 
-- name: get epel-repo rpm
-  get_url: dest=/tmp/epel-release.rpm  url=http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
-
-- name: install epel-repo rpm
-  yum: pkg=/tmp/epel-release.rpm state=installed
-
 - name: Download Ambari repo
   get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
 
-- name: Clean yum
-  shell: yum clean all

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_config/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/meta/main.yml b/deployment/roles/ambari_config/meta/main.yml
new file mode 100644
index 0000000..61197e3
--- /dev/null
+++ b/deployment/roles/ambari_config/meta/main.yml
@@ -0,0 +1,21 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - epel
+  - python-pip
+  - httplib2

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_config/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/tasks/main.yml b/deployment/roles/ambari_config/tasks/main.yml
index f98c33b..f44f929 100644
--- a/deployment/roles/ambari_config/tasks/main.yml
+++ b/deployment/roles/ambari_config/tasks/main.yml
@@ -18,7 +18,13 @@
 - include_vars: "{{ cluster_type }}.yml"
 
 - name: Install python-requests
-  yum: name=python-requests
+  yum:
+    name: python-requests
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Deploy cluster with Ambari; http://{{ groups.ambari_master[0] }}:{{ ambari_port }}
   ambari_cluster_state:
@@ -32,3 +38,5 @@
     configurations: "{{ configurations }}"
     wait_for_complete: True
     blueprint_var: "{{ blueprint }}"
+
+- include: start_services.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_config/tasks/start_services.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/tasks/start_services.yml b/deployment/roles/ambari_config/tasks/start_services.yml
new file mode 100644
index 0000000..7c6e0a9
--- /dev/null
+++ b/deployment/roles/ambari_config/tasks/start_services.yml
@@ -0,0 +1,48 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Start All Hadoop Services {{ inventory_hostname }}
+  uri:
+    url: http://{{ inventory_hostname}}:{{ ambari_port}}/api/v1/clusters/{{ cluster_name }}/services/{{ item }}
+    HEADER_X-Requested-By: "{{ ambari_user }}"
+    method: PUT
+    body: "{ \"RequestInfo\": { \"context\": \"Start service via REST\" }, \"Body\": { \"ServiceInfo\": { \"state\": \"STARTED\" }}}"
+    body_format: json
+    status_code: 200,202
+    force_basic_auth: yes
+    user: "{{ ambari_user }}"
+    password: "{{ ambari_password }}"
+  with_items:
+    - "{{ metron_services }}"
+
+- name: Wait for Service Start
+  uri:
+    url: http://{{ inventory_hostname}}:{{ ambari_port}}/api/v1/clusters/{{ cluster_name }}/services/{{ item }}
+    HEADER_X-Requested-By: "{{ ambari_user }}"
+    method: GET
+    status_code: 200
+    force_basic_auth: yes
+    user: "{{ ambari_user }}"
+    password: "{{ ambari_password }}"
+    return_content: yes
+  with_items:
+    - "{{ metron_services }}"
+  register: result
+  until: result.content.find("STARTED") != -1
+  retries: 10
+  delay: 60
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml b/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
index 574268f..526661b 100644
--- a/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
+++ b/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
@@ -27,6 +27,8 @@ hbase_master: [HBASE_MASTER]
 hbase_slave: [HBASE_REGIONSERVER]
 hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
 
+metron_services: ["HDFS","YARN","MAPREDUCE2","ZOOKEEPER", "HBASE", "STORM", "KAFKA"]
+
 master_1_components: "{{ hadoop_master | union(hadoop_clients) }}"
 master_1_host:
   - "{{groups.ambari_slave[0]}}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/single_node_vm.yml b/deployment/roles/ambari_config/vars/single_node_vm.yml
index 6b18825..cb6fe4a 100644
--- a/deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -29,6 +29,7 @@ zookeeper_slave: [ZOOKEEPER_CLIENT]
 hbase_master: [HBASE_MASTER, HBASE_CLIENT]
 hbase_slave: [HBASE_REGIONSERVER]
 
+metron_services: ["HDFS","YARN","MAPREDUCE2","ZOOKEEPER", "HBASE", "STORM", "KAFKA"]
 metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(kafka_broker) | union(hbase_slave) }}"
 
 cluster_name: "metron_cluster"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/small_cluster.yml b/deployment/roles/ambari_config/vars/small_cluster.yml
index ff985dc..a3792e1 100644
--- a/deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/deployment/roles/ambari_config/vars/small_cluster.yml
@@ -27,6 +27,8 @@ hbase_master: [HBASE_MASTER]
 hbase_slave: [HBASE_REGIONSERVER]
 hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
 
+metron_services: ["HDFS","YARN","MAPREDUCE2","ZOOKEEPER", "HBASE", "STORM", "KAFKA"]
+
 master_1_components: "{{ hadoop_master | union(hadoop_clients) }}"
 master_1_host:
   - "{{groups.ambari_slave[0]}}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_gather_facts/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_gather_facts/meta/main.yml b/deployment/roles/ambari_gather_facts/meta/main.yml
new file mode 100644
index 0000000..61197e3
--- /dev/null
+++ b/deployment/roles/ambari_gather_facts/meta/main.yml
@@ -0,0 +1,21 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - epel
+  - python-pip
+  - httplib2

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_gather_facts/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_gather_facts/tasks/main.yml b/deployment/roles/ambari_gather_facts/tasks/main.yml
index 8ec7e40..db4927d 100644
--- a/deployment/roles/ambari_gather_facts/tasks/main.yml
+++ b/deployment/roles/ambari_gather_facts/tasks/main.yml
@@ -15,16 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Install EPEL repository
-  yum: name=epel-release
-
-- name: Install pip
-  yum: name=python-pip
-
-- name: Install python httplib2 dependency
-  pip:
-    name: httplib2
-
 - name: Ambari rest get cluster name
   uri:
     url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_master/tasks/main.yml b/deployment/roles/ambari_master/tasks/main.yml
index 021e377..25c3784 100644
--- a/deployment/roles/ambari_master/tasks/main.yml
+++ b/deployment/roles/ambari_master/tasks/main.yml
@@ -17,12 +17,14 @@
 ---
 # tasks file for ambari_master
 - name: Install ambari server
-  yum: name=ambari-server state=present
-
-- name: Setup ambari server
-  shell: ambari-server setup -s; touch /etc/ambari-server/configured creates=/etc/ambari-server/configured
-  register: ambari_server_setup
-  failed_when: ambari_server_setup.stderr
+  yum:
+    name: ambari-server
+    state: present
+    update_cache: yes
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Set Ambari Server Max Memory
   replace:
@@ -31,8 +33,19 @@
     replace: " -Xmx{{ ambari_server_mem }}m "
     backup: no
 
+- name: Setup ambari server
+  shell: ambari-server setup -s && touch /etc/ambari-server/configured creates=/etc/ambari-server/configured
+  register: ambari_server_setup
+  failed_when: ambari_server_setup.stderr
+
 - name: start ambari server
-  service: name=ambari-server state=restarted
+  service:
+    name: ambari-server
+    state: restarted
 
 - name : check if ambari-server is up on {{ ambari_host }}:{{ambari_port}}
-  wait_for : host={{ ambari_host }} port={{ ambari_port }} delay=10 timeout=120
+  wait_for :
+    host: "{{ ambari_host }}"
+    port: "{{ ambari_port }}"
+    delay: 120
+    timeout: 300

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ambari_slave/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_slave/tasks/main.yml b/deployment/roles/ambari_slave/tasks/main.yml
index eac0610..fc068da 100644
--- a/deployment/roles/ambari_slave/tasks/main.yml
+++ b/deployment/roles/ambari_slave/tasks/main.yml
@@ -17,7 +17,13 @@
 ---
 # tasks file for ambari_slave
 - name: Install ambari-agent
-  yum: name=ambari-agent
+  yum:
+    name: ambari-agent
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Create ambari-agent hostname script
   template:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/meta/main.yml b/deployment/roles/bro/meta/main.yml
new file mode 100644
index 0000000..4d10740
--- /dev/null
+++ b/deployment/roles/bro/meta/main.yml
@@ -0,0 +1,21 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - ambari_gather_facts
+  - libselinux-python
+  - build-tools

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/bro/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/dependencies.yml b/deployment/roles/bro/tasks/dependencies.yml
index 17e84df..431e861 100644
--- a/deployment/roles/bro/tasks/dependencies.yml
+++ b/deployment/roles/bro/tasks/dependencies.yml
@@ -18,7 +18,6 @@
 - name: Install prerequisites
   yum: name={{ item }}
   with_items:
-    - libselinux-python
     - cmake
     - make
     - gcc
@@ -32,3 +31,7 @@
     - swig
     - zlib-devel
     - perl
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/build-tools/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/build-tools/meta/main.yml b/deployment/roles/build-tools/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/deployment/roles/build-tools/meta/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/build-tools/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/build-tools/tasks/main.yml b/deployment/roles/build-tools/tasks/main.yml
new file mode 100644
index 0000000..c47ef43
--- /dev/null
+++ b/deployment/roles/build-tools/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install Build Tools
+  yum: name={{ item }}
+  with_items:
+    - "@Development tools"
+    - libdnet-devel
+    - rpm-build
+    - libpcap
+    - libpcap-devel
+    - pcre
+    - pcre-devel
+    - zlib
+    - zlib-devel
+    - glib2-devel
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/elasticsearch/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/files/yaf_index.template b/deployment/roles/elasticsearch/files/yaf_index.template
new file mode 100644
index 0000000..c8c1702
--- /dev/null
+++ b/deployment/roles/elasticsearch/files/yaf_index.template
@@ -0,0 +1,36 @@
+{
+    "template" : "yaf_index*",
+    "mappings" : {
+        "yaf_doc" : {
+            "properties": {
+                "message": {
+                    "properties": {
+                        "@timestamp":{"type":"date","format":"dateOptionalTime"},
+                        "end-time":{"type":"string"},
+                        "duration":{"type":"string"},
+                        "rtt":{"type":"string"},
+                        "proto":{"type":"string"},
+                        "sip":{"type":"string"},
+                        "sp":{"type":"string"},
+                        "dip":{"type":"string"},
+                        "dp":{"type":"string"},
+                        "iflags":{"type":"string"},
+                        "uflags":{"type":"string"},
+                        "riflags":{"type":"string"},
+                        "ruflags":{"type":"string"},
+                        "isn":{"type":"string"},
+                        "risn":{"type":"string"},
+                        "tag":{"type":"string"},
+                        "rtag":{"type":"string"},
+                        "pkt":{"type":"string"},
+                        "oct":{"type":"string"},
+                        "rpkt":{"type":"string"},
+                        "roct":{"type":"string"},
+                        "app":{"type":"string"},
+                        "end-reason":{"type":"string"}
+                    }
+                }
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/elasticsearch/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/handlers/main.yml b/deployment/roles/elasticsearch/handlers/main.yml
deleted file mode 100644
index 112c5ca..0000000
--- a/deployment/roles/elasticsearch/handlers/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-- name: restart elasticsearch
-  service: name=elasticsearch state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/elasticsearch/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/meta/main.yml b/deployment/roles/elasticsearch/meta/main.yml
new file mode 100644
index 0000000..f5f059a
--- /dev/null
+++ b/deployment/roles/elasticsearch/meta/main.yml
@@ -0,0 +1,24 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk
+  - epel
+  - python-pip
+  - httplib2
+  - libselinux-python
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/elasticsearch/tasks/configure_index.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/tasks/configure_index.yml b/deployment/roles/elasticsearch/tasks/configure_index.yml
new file mode 100644
index 0000000..09739be
--- /dev/null
+++ b/deployment/roles/elasticsearch/tasks/configure_index.yml
@@ -0,0 +1,44 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name : Wait for Elasticsearch Host to Start
+  wait_for:
+    host: "{{ groups.search[0] }}"
+    port: "{{ elasticsearch_web_port }}"
+    delay: 10
+    timeout: 300
+
+- name: Wait for Green Index Status
+  uri:
+    url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_cat/health"
+    method: GET
+    status_code: 200
+    return_content: yes
+  register: result
+  until: result.content.find("green") != -1
+  retries: 10
+  delay: 60
+  run_once: yes
+
+- name: Add Elasticsearch templates for topologies
+  uri:
+    url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
+    method: POST
+    body: "{{ lookup('file','yaf_index.template') }}"
+    status_code: 200
+    body_format: json
+  run_once: yes

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/tasks/main.yml b/deployment/roles/elasticsearch/tasks/main.yml
index fa8d4f3..d9b2f44 100644
--- a/deployment/roles/elasticsearch/tasks/main.yml
+++ b/deployment/roles/elasticsearch/tasks/main.yml
@@ -15,12 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Install libselinux-python and java
-  yum: name={{item}}
-  with_items:
-    - libselinux-python
-    - java-1.8.0-openjdk
-
 - name: Add Elasticsearch GPG key.
   rpm_key:
     key: https://packages.elastic.co/GPG-KEY-elasticsearch
@@ -33,7 +27,13 @@
     mode: 0644
 
 - name: Install Elasticsearch.
-  yum: pkg=elasticsearch state=installed
+  yum:
+    name: elasticsearch
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Create Data Directories
   file:
@@ -59,7 +59,8 @@
     - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
     line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
     - { regexp: '#path\.data', line: 'path.data: {{     elasticsearch_data_dir }}' }
-  notify: restart elasticsearch
 
 - name: Start Elasticsearch.
   service: name=elasticsearch state=started enabled=yes
+
+- include: configure_index.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/epel/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/epel/tasks/main.yml b/deployment/roles/epel/tasks/main.yml
new file mode 100644
index 0000000..db4e70b
--- /dev/null
+++ b/deployment/roles/epel/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Get epel-repo rpm
+  get_url:
+    dest: /tmp/epel-release.rpm
+    url: http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+
+- name: Install epel-repo rpm
+  yum:
+    pkg: /tmp/epel-release.rpm
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/flume/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/flume/meta/main.yml b/deployment/roles/flume/meta/main.yml
new file mode 100644
index 0000000..ff35a5a
--- /dev/null
+++ b/deployment/roles/flume/meta/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk
+  - libselinux-python

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/flume/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/flume/tasks/main.yml b/deployment/roles/flume/tasks/main.yml
index 7080b55..8576c3c 100644
--- a/deployment/roles/flume/tasks/main.yml
+++ b/deployment/roles/flume/tasks/main.yml
@@ -24,9 +24,12 @@
 - name: Install flume
   yum: name={{item}}
   with_items:
-    - libselinux-python
     - flume
     - flume-agent
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Create flume-env.sh
   shell: cp /etc/flume/conf/flume-env.sh.template /etc/flume/conf/flume-env.sh

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/hadoop_setup/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/hadoop_setup/meta/main.yml b/deployment/roles/hadoop_setup/meta/main.yml
new file mode 100644
index 0000000..8f0bf2b
--- /dev/null
+++ b/deployment/roles/hadoop_setup/meta/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - ambari_gather_facts
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/httplib2/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/httplib2/tasks/main.yml b/deployment/roles/httplib2/tasks/main.yml
new file mode 100644
index 0000000..5502cf4
--- /dev/null
+++ b/deployment/roles/httplib2/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install python httplib2 dependency
+  pip:
+    name: httplib2
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/java_jdk/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/java_jdk/defaults/main.yml b/deployment/roles/java_jdk/defaults/main.yml
new file mode 100644
index 0000000..28f6c71
--- /dev/null
+++ b/deployment/roles/java_jdk/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+java_home: /usr/jdk64/jdk1.8.0_40

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/java_jdk/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/java_jdk/tasks/main.yml b/deployment/roles/java_jdk/tasks/main.yml
new file mode 100644
index 0000000..999b9c1
--- /dev/null
+++ b/deployment/roles/java_jdk/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Check for java at "{{ java_home }}"
+  stat: path="{{ java_home }}"
+  register: jdk_dir
+
+- name: Alternatives link for java
+  alternatives: name={{ item.name }} link={{ item.link }}  path={{ item.path }}
+  with_items:
+    - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
+    - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
+  when: jdk_dir.stat.exists
+
+- name: Install openjdk
+  yum: name={{item}}
+  with_items:
+    - java-1.8.0-openjdk
+    - java-1.8.0-openjdk-devel
+  when: not jdk_dir.stat.exists

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/libselinux-python/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/libselinux-python/tasks/main.yml b/deployment/roles/libselinux-python/tasks/main.yml
new file mode 100644
index 0000000..78f5a27
--- /dev/null
+++ b/deployment/roles/libselinux-python/tasks/main.yml
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install libselinux-python
+  yum:
+    name: libselinux-python
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_common/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/meta/main.yml b/deployment/roles/metron_common/meta/main.yml
new file mode 100644
index 0000000..4db50aa
--- /dev/null
+++ b/deployment/roles/metron_common/meta/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - yum-update
+  - epel
+  - ntp
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/tasks/main.yml b/deployment/roles/metron_common/tasks/main.yml
index 29c0821..64e6ab9 100644
--- a/deployment/roles/metron_common/tasks/main.yml
+++ b/deployment/roles/metron_common/tasks/main.yml
@@ -27,21 +27,9 @@
   fail: msg="Unable to locate staged Metron jars at {{ metron_jar_path }}.  Did you run 'mvn package'?"
   when: metron_jars.stat.exists == True
 
-- name: upgrade all packages
-  yum: name=* state=latest
-
-- name: get epel-repo rpm
-  get_url: dest=/tmp/epel-release.rpm  url=http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
-
-- name: install epel-repo rpm
-  yum: pkg=/tmp/epel-release.rpm state=installed
-
-- name: install ntp
-  yum: name=ntp state=present
-
-- name: Ensure ntp is running and enabled
-  service: name=ntpd state=started enabled=yes
-
 - name: Ensure iptables is stopped and is not running at boot time.
   ignore_errors: yes
-  service: name=iptables state=stopped enabled=no
\ No newline at end of file
+  service:
+    name: iptables
+    state: stopped
+    enabled: no

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_pcapservice/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/meta/main.yml b/deployment/roles/metron_pcapservice/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/meta/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_pcapservice/tasks/config-java.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/config-java.yml b/deployment/roles/metron_pcapservice/tasks/config-java.yml
deleted file mode 100644
index 999b9c1..0000000
--- a/deployment/roles/metron_pcapservice/tasks/config-java.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-- name: Check for java at "{{ java_home }}"
-  stat: path="{{ java_home }}"
-  register: jdk_dir
-
-- name: Alternatives link for java
-  alternatives: name={{ item.name }} link={{ item.link }}  path={{ item.path }}
-  with_items:
-    - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
-    - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
-  when: jdk_dir.stat.exists
-
-- name: Install openjdk
-  yum: name={{item}}
-  with_items:
-    - java-1.8.0-openjdk
-    - java-1.8.0-openjdk-devel
-  when: not jdk_dir.stat.exists

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_pcapservice/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/main.yml b/deployment/roles/metron_pcapservice/tasks/main.yml
index 68da876..ee9cac2 100644
--- a/deployment/roles/metron_pcapservice/tasks/main.yml
+++ b/deployment/roles/metron_pcapservice/tasks/main.yml
@@ -15,8 +15,6 @@
 #  limitations under the License.
 #
 ---
-- include: config-java.yml
-
 - name: Check for hbase-site
   stat: path=/etc/hbase/conf/hbase-site.xml
   register: hbase_site

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_streaming/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/meta/main.yml b/deployment/roles/metron_streaming/meta/main.yml
new file mode 100644
index 0000000..6820b2c
--- /dev/null
+++ b/deployment/roles/metron_streaming/meta/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - ambari_gather_facts
+  - java_jdk

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_streaming/tasks/grok_upload.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/grok_upload.yml b/deployment/roles/metron_streaming/tasks/grok_upload.yml
new file mode 100644
index 0000000..1bcf39f
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/grok_upload.yml
@@ -0,0 +1,37 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create HDFS directory for grok patterns
+  command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns
+  become: yes
+  become_user: hdfs
+
+- name: Assign hfds user as owner of  {{ metron_hdfs_output_dir }}/patterns HDFS directory
+  command: hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns
+  become: yes
+  become_user: hdfs
+
+- name: Assign permissions of HDFS {{ metron_hdfs_output_dir }}/patterns directory
+  command: hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns
+  become: yes
+  become_user: hdfs
+
+- name: Upload Grok Patterns to hdfs://{{ metron_hdfs_output_dir }}
+  command: hdfs dfs -put {{ metron_directory }}/config/patterns  {{ metron_hdfs_output_dir }}
+  become: yes
+  become_user: hdfs
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml b/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
new file mode 100644
index 0000000..252e671
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
@@ -0,0 +1,41 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create root user HDFS directory
+  command: hdfs dfs -mkdir -p /user/root
+  become: yes
+  become_user: hdfs
+
+- name: Assign root as owner of /user/root HDFS directory
+  command: hdfs dfs -chown root:root /user/root
+  become: yes
+  become_user: hdfs
+
+- name: Create Metron HDFS output directory
+  command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}
+  become: yes
+  become_user: hdfs
+
+- name: Assign hdfs as owner of HDFS output directory
+  command: hdfs dfs -chown hdfs:hadoop {{ metron_hdfs_output_dir }}
+  become: yes
+  become_user: hdfs
+
+- name: Assign permissions of HDFS output directory
+  command: hdfs dfs -chmod 775 {{ metron_hdfs_output_dir }}
+  become: yes
+  become_user: hdfs
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index 27b087c..c0e0638 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -36,12 +36,6 @@
 - name: Unbundle Metron bundles
   shell: cd {{ metron_directory }} && tar xzvf Metron-Topologies*.tar.gz && tar xzvf Metron-DataLoads*.tar.gz && rm *.tar.gz
 
-- name: Alternatives link for "java"
-  alternatives: name={{ item.name }} link={{ item.link }}  path={{ item.path }}
-  with_items:
-    - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
-    - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
-
 - name: Add hbase-site.xml to topology jar
   shell: cd {{ hbase_config_path }} && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} hbase-site.xml
 
@@ -52,17 +46,14 @@
   shell: cd {{ hdfs_config_path }} && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} hdfs-site.xml
 
 - name: Get Default mysql passowrd
-  include_vars: "../roles/mysql_server/vars/main.yml"
+  include_vars: "../roles/mysql_server/defaults/main.yml"
   when: mysql_root_password is undefined
 
-- name: Create root user HDFS directory
-  command: su - hdfs -c "hdfs dfs -mkdir -p /user/root && hdfs dfs -chown root:root /user/root"
-
-- name: Create Metron HDFS output directory
-  command: su - hdfs -c "hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }} && hdfs dfs -chown hdfs:hadoop {{ metron_hdfs_output_dir }} && hdfs dfs -chmod 775 {{ metron_hdfs_output_dir }}"
+- include: hdfs_filesystem.yml
+  run_once: true
 
-- name: Upload grok patterns
-  command: su - hdfs -c "hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns && hdfs dfs -put {{ metron_directory }}/config/patterns/*  {{ metron_hdfs_output_dir }}/patterns && hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns && hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns"
+- include: grok_upload.yml
+  run_once: true
 
 - name: Configure Metron topologies
   lineinfile: >
@@ -87,16 +78,10 @@
     - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
     - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/enrichment/indexed" }
 
-- name: Add Elasticsearch templates for topologies
-  uri:
-    url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
-    method: POST
-    body: "{{ lookup('file','yaf_index.template') }}"
-    status_code: 200
-    body_format: json
 
 - include: source_config.yml
   run_once: true
+
 - include: threat_intel.yml
   run_once: true
   when: threat_intel_bulk_load == True

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/main.yml b/deployment/roles/metron_ui/tasks/main.yml
index 861abbd..01ffc17 100644
--- a/deployment/roles/metron_ui/tasks/main.yml
+++ b/deployment/roles/metron_ui/tasks/main.yml
@@ -16,12 +16,19 @@
 #
 ---
 - name: Install Metron UI dependencies
-  yum: pkg={{ item.package }} state=installed
+  yum:
+    pkg: "{{ item.package }}"
+    state: installed
   with_items:
       - { package: 'libpcap-devel'}
       - { package: 'wireshark'}
       - { package: 'nodejs'}
       - { package: 'npm'}
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
+
 
 - name: Copy Metron UI source
   copy:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/mysql_server/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/tasks/main.yml b/deployment/roles/mysql_server/tasks/main.yml
index e97b245..a484ed0 100644
--- a/deployment/roles/mysql_server/tasks/main.yml
+++ b/deployment/roles/mysql_server/tasks/main.yml
@@ -30,6 +30,10 @@
   yum:
     pkg: /tmp/{{ mysql_rpm_version }}.rpm
     state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Install MySQL
   yum:
@@ -38,6 +42,10 @@
   with_items:
     - "mysql-community-server"
     - "MySQL-python"
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Start MySQL
   service:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/ntp/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ntp/tasks/main.yml b/deployment/roles/ntp/tasks/main.yml
new file mode 100644
index 0000000..7b1b9a8
--- /dev/null
+++ b/deployment/roles/ntp/tasks/main.yml
@@ -0,0 +1,31 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install ntp
+  yum:
+    name: ntp
+    state: present
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
+
+- name: Ensure ntp is running and enabled
+  service:
+    name: ntpd
+    state: started
+    enabled: yes

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/pcap_replay/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/meta/main.yml b/deployment/roles/pcap_replay/meta/main.yml
index 2df379c..0c47853 100644
--- a/deployment/roles/pcap_replay/meta/main.yml
+++ b/deployment/roles/pcap_replay/meta/main.yml
@@ -15,20 +15,7 @@
 #  limitations under the License.
 #
 ---
-galaxy_info:
-  author: Nick Allen
-  description:
-  company: Metron
+dependencies:
+  - libselinux-python
+  - build-tools
 
-  license: license (Apache, CC-BY, Nick Allen)
-  min_ansible_version: 1.2
-
-  platforms:
-  - name: CentOS
-    versions:
-      - 6
-
-  categories:
-  - networking
-
-dependencies: []

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/pcap_replay/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/tasks/main.yml b/deployment/roles/pcap_replay/tasks/main.yml
index 449710e..68005b9 100644
--- a/deployment/roles/pcap_replay/tasks/main.yml
+++ b/deployment/roles/pcap_replay/tasks/main.yml
@@ -15,9 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Install prerequisites
-  yum: name=libselinux-python
-
 - include: tcpreplay.yml
 
 - include: service.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/pcap_replay/tasks/tcpreplay.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/tasks/tcpreplay.yml b/deployment/roles/pcap_replay/tasks/tcpreplay.yml
index 851a6ae..e24dcf1 100644
--- a/deployment/roles/pcap_replay/tasks/tcpreplay.yml
+++ b/deployment/roles/pcap_replay/tasks/tcpreplay.yml
@@ -15,13 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Install prerequisites
-  yum: name={{ item }}
-  with_items:
-    - "@Development tools"
-    - libpcap
-    - libpcap-devel
-
 - name: Download tcpreplay
   get_url:
     url: "https://github.com/appneta/tcpreplay/releases/download/v{{ tcpreplay_version }}/tcpreplay-{{ tcpreplay_version }}.tar.gz"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/pycapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/meta/main.yml b/deployment/roles/pycapa/meta/main.yml
new file mode 100644
index 0000000..85c04df
--- /dev/null
+++ b/deployment/roles/pycapa/meta/main.yml
@@ -0,0 +1,21 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - ambari_gather_facts
+  - epel
+  - python-pip

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/pycapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/tasks/main.yml b/deployment/roles/pycapa/tasks/main.yml
index 71e5964..8649f3a 100644
--- a/deployment/roles/pycapa/tasks/main.yml
+++ b/deployment/roles/pycapa/tasks/main.yml
@@ -15,18 +15,17 @@
 #  limitations under the License.
 #
 ---
-- name: Install EPEL repository
-  yum: name=epel-release
-
 - name: Install python and tcpdump
-  yum: name={{item}}
+  yum:
+    name: "{{item}}"
   with_items:
     - python
     - tcpdump
     - git
-
-- name: Install pip
-  easy_install: name=pip state=latest
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Clone pycapa repo
   git: repo={{ pycapa_repo }} dest={{ pycapa_home }}
@@ -43,8 +42,13 @@
 - name: Turn on promiscuous mode for {{ sniff_interface }}
   shell: "ip link set {{ sniff_interface }} promisc on"
 
-- name: Install prerequisites
-  yum: name=daemonize
+- name: Install Daemonize
+  yum:
+    name: daemonize
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - set_fact:
     pycapa: /opt/pycapa/pycapa/pycapa_cli.py

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/python-pip/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/python-pip/tasks/main.yml b/deployment/roles/python-pip/tasks/main.yml
new file mode 100644
index 0000000..809aca4
--- /dev/null
+++ b/deployment/roles/python-pip/tasks/main.yml
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install python-pip
+  yum:
+    name: python-pip
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/snort/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/meta/main.yml b/deployment/roles/snort/meta/main.yml
new file mode 100644
index 0000000..df99998
--- /dev/null
+++ b/deployment/roles/snort/meta/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - ambari_gather_facts
+  - epel
+  - libselinux-python
+  - build-tools

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/snort/tasks/daq.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/daq.yml b/deployment/roles/snort/tasks/daq.yml
index 8f0dcae..50d2049 100644
--- a/deployment/roles/snort/tasks/daq.yml
+++ b/deployment/roles/snort/tasks/daq.yml
@@ -27,4 +27,10 @@
     creates: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
 
 - name: Install daq
-  yum: name=/root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+  yum:
+    name: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/snort/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/main.yml b/deployment/roles/snort/tasks/main.yml
index 43b9c16..80755be 100644
--- a/deployment/roles/snort/tasks/main.yml
+++ b/deployment/roles/snort/tasks/main.yml
@@ -15,22 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Install EPEL repository
-  yum: name=epel-release
-
-- name: Install snort prerequisites
-  yum: name={{ item }}
-  with_items:
-    - "@Development tools"
-    - libdnet-devel
-    - rpm-build
-    - libpcap
-    - libpcap-devel
-    - pcre
-    - pcre-devel
-    - zlib-devel
-    - libselinux-python
-
 - include: daq.yml
 
 - include: snort.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/snort/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/snort.yml b/deployment/roles/snort/tasks/snort.yml
index c08bc93..b56ca74 100644
--- a/deployment/roles/snort/tasks/snort.yml
+++ b/deployment/roles/snort/tasks/snort.yml
@@ -15,20 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Install development tools
-  yum: name="@Development tools" state=present
-
-- name: Install snort prerequisites
-  yum: name={{ item }}
-  with_items:
-    - libdnet-devel
-    - rpm-build
-    - libpcap
-    - libpcap-devel
-    - pcre
-    - pcre-devel
-    - zlib-devel
-
 - name: Download daq
   get_url:
     url: "https://snort.org/downloads/snort/daq-{{ daq_version }}.src.rpm"
@@ -41,7 +27,12 @@
     creates: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
 
 - name: Install daq
-  yum: name=/root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+  yum:
+    name: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Download snort
   get_url:
@@ -55,7 +46,12 @@
     creates: /root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm
 
 - name: Install snort
-  yum: name=/root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm
+  yum:
+    name: /root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
 - name: Download snort community rules
   get_url:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/tap_interface/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/tap_interface/tasks/main.yml b/deployment/roles/tap_interface/tasks/main.yml
index d4590f7..1de3abe 100644
--- a/deployment/roles/tap_interface/tasks/main.yml
+++ b/deployment/roles/tap_interface/tasks/main.yml
@@ -16,9 +16,14 @@
 #
 ---
 - name: Install tunctl
-  yum: name=tunctl
+  yum:
+    name: tunctl
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
 
-#TODO - only run when tap_if does not exist
 - name: Create {{ tap_if }}
   command: tunctl -p
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/yaf/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/meta/main.yml b/deployment/roles/yaf/meta/main.yml
new file mode 100644
index 0000000..324f2ff
--- /dev/null
+++ b/deployment/roles/yaf/meta/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - ambari_gather_facts
+  - build-tools
+  - java_jdk
+  - libselinux-python

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/yaf/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/dependencies.yml b/deployment/roles/yaf/tasks/dependencies.yml
deleted file mode 100644
index 1a28b2a..0000000
--- a/deployment/roles/yaf/tasks/dependencies.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-- name: Install prerequisites
-  yum: name={{ item }}
-  with_items:
-    - "@Development tools"
-    - libselinux-python
-    - libpcap
-    - libpcap-devel
-    - pcre
-    - pcre-devel
-    - glib2-devel
-    - zlib
-    - zlib-devel
-    - java-1.8.0-openjdk
-    - java-1.8.0-openjdk-devel

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/yaf/tasks/kafka-client.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/kafka-client.yml b/deployment/roles/yaf/tasks/kafka-client.yml
index 729929a..1674225 100644
--- a/deployment/roles/yaf/tasks/kafka-client.yml
+++ b/deployment/roles/yaf/tasks/kafka-client.yml
@@ -22,6 +22,9 @@
       mode: 0644
 
   - name: Install kafka
-    yum: name={{ item }}
-    with_items:
-      - kafka
+    yum:
+      name: kafka
+    register: result
+    until: result.rc == 0
+    retries: 5
+    delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml
index e661f50..b583393 100644
--- a/deployment/roles/yaf/tasks/main.yml
+++ b/deployment/roles/yaf/tasks/main.yml
@@ -15,7 +15,6 @@
 #  limitations under the License.
 #
 ---
-- include: dependencies.yml
 - include: kafka-client.yml
 - include: fixbuf.yml
 - include: yaf.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/68aab6e9/deployment/roles/yum-update/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yum-update/tasks/main.yml b/deployment/roles/yum-update/tasks/main.yml
new file mode 100644
index 0000000..4db6297
--- /dev/null
+++ b/deployment/roles/yum-update/tasks/main.yml
@@ -0,0 +1,26 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Yum Update Packages
+  yum:
+    name: "*"
+    state: latest
+    update_cache: yes
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10


Mime
View raw message