metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rmerri...@apache.org
Subject [43/51] [partial] incubator-metron git commit: METRON-113 Project Reorganization (merrimanr) closes apache/incubator-metron#88
Date Tue, 26 Apr 2016 14:46:31 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/ambari_slave/files/hostname.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/files/hostname.sh b/metron-deployment/roles/ambari_slave/files/hostname.sh
new file mode 100644
index 0000000..cc8c1cd
--- /dev/null
+++ b/metron-deployment/roles/ambari_slave/files/hostname.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+echo {{ inventory_hostname }}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/ambari_slave/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/tasks/main.yml b/metron-deployment/roles/ambari_slave/tasks/main.yml
new file mode 100644
index 0000000..fc068da
--- /dev/null
+++ b/metron-deployment/roles/ambari_slave/tasks/main.yml
@@ -0,0 +1,51 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+# tasks file for ambari_slave
+- name: Install ambari-agent
+  yum:
+    name: ambari-agent
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
+
+- name: Create ambari-agent hostname script
+  template:
+    src: "../roles/ambari_slave/files/hostname.sh"
+    dest: "/var/lib/ambari-agent/hostname.sh"
+    mode: 0744
+    owner: "{{ ambari_user }}"
+    group: "{{ ambari_user }}"
+
+- name: Configure ambari-server hostname in ambari-agent configuration
+  lineinfile:
+    dest: /etc/ambari-agent/conf/ambari-agent.ini
+    regexp: "{{ item.regexp }}"
+    line: "{{ item.line }}"
+    insertafter: "{{ item.insertafter }}"
+    backup: yes
+  with_items:
+    - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' }
+    - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'}
+
+- name: Ensure ambari-agent is running
+  service: name=ambari-agent state=restarted
+
+- name : Wait for agent to register
+  command : sleep 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/ambari_slave/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/vars/main.yml b/metron-deployment/roles/ambari_slave/vars/main.yml
new file mode 100644
index 0000000..600b3fc
--- /dev/null
+++ b/metron-deployment/roles/ambari_slave/vars/main.yml
@@ -0,0 +1,24 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+#
+# TODO: duplicates from ambari-common.  need all of these moved to group_vars
+#
+rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.1.2.1/ambari.repo"
+ambari_user: "root"
+local_tmp_keygen_file: "/tmp/id_rsa.tmp"
+dest_tmp_keygen_file: "/tmp/id_rsa.tmp"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/meta/main.yml b/metron-deployment/roles/bro/meta/main.yml
new file mode 100644
index 0000000..9c9286f
--- /dev/null
+++ b/metron-deployment/roles/bro/meta/main.yml
@@ -0,0 +1,23 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - libselinux-python
+  - ambari_gather_facts
+  - build-tools
+  - kafka-client
+  - librdkafka

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml b/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml
new file mode 100644
index 0000000..d8e887d
--- /dev/null
+++ b/metron-deployment/roles/bro/tasks/bro-plugin-kafka.yml
@@ -0,0 +1,41 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Distribute bro-kafka plugin
+  copy: src=../../../metron-sensors/bro-plugin-kafka dest=/tmp/ mode=0755
+
+- name: Compile and install the plugin
+  shell: "{{ item }}"
+  args:
+    chdir: "/tmp/bro-plugin-kafka"
+    creates: "{{ bro_home }}/lib/bro/plugins/BRO_KAFKA"
+  with_items:
+    - rm -rf build/
+    - "./configure --bro-dist=/tmp/bro-{{ bro_version }} --install-root={{ bro_home }}/lib/bro/plugins/ --with-librdkafka={{ librdkafka_home }}"
+    - make
+    - make install
+
+- name: Configure bro-kafka plugin
+  lineinfile:
+    dest: "{{ bro_home }}/share/bro/site/local.bro"
+    line: "{{ item }}"
+  with_items:
+    - "@load Bro/Kafka/logs-to-kafka.bro"
+    - "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);"
+    - "redef Kafka::topic_name = \"{{ bro_topic }}\";"
+    - "redef Kafka::tag_json = T;"
+    - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/bro.yml b/metron-deployment/roles/bro/tasks/bro.yml
new file mode 100644
index 0000000..fb27ef9
--- /dev/null
+++ b/metron-deployment/roles/bro/tasks/bro.yml
@@ -0,0 +1,44 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Download bro
+  get_url:
+    url: "https://www.bro.org/downloads/release/bro-{{ bro_version }}.tar.gz"
+    dest: "/tmp/bro-{{ bro_version }}.tar.gz"
+
+- name: Extract bro tarball
+  unarchive:
+    src: "/tmp/bro-{{ bro_version }}.tar.gz"
+    dest: /tmp
+    copy: no
+    creates: "/tmp/bro-{{ bro_version }}"
+
+- name: Compile and Install bro
+  shell: "{{ item }}"
+  args:
+    chdir: "/tmp/bro-{{ bro_version }}"
+    creates: "{{ bro_home }}/bin/bro"
+  with_items:
+    - "./configure --prefix={{ bro_home }}"
+    - make
+    - make install
+
+- name: Configure bro
+  lineinfile:
+    dest: "{{ bro_home }}/etc/node.cfg"
+    regexp: '^interface=.*$'
+    line: 'interface={{ sniff_interface }}'

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/dependencies.yml b/metron-deployment/roles/bro/tasks/dependencies.yml
new file mode 100644
index 0000000..431e861
--- /dev/null
+++ b/metron-deployment/roles/bro/tasks/dependencies.yml
@@ -0,0 +1,37 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install prerequisites
+  yum: name={{ item }}
+  with_items:
+    - cmake
+    - make
+    - gcc
+    - gcc-c++
+    - flex
+    - bison
+    - libpcap
+    - libpcap-devel
+    - openssl-devel
+    - python-devel
+    - swig
+    - zlib-devel
+    - perl
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/librdkafka.yml b/metron-deployment/roles/bro/tasks/librdkafka.yml
new file mode 100644
index 0000000..652d319
--- /dev/null
+++ b/metron-deployment/roles/bro/tasks/librdkafka.yml
@@ -0,0 +1,39 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Download librdkafka
+  get_url:
+    url: "{{ librdkafka_url }}"
+    dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+
+- name: Extract librdkafka tarball
+  unarchive:
+    src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+    dest: /tmp
+    copy: no
+    creates: "/tmp/librdkafka-{{ librdkafka_version }}"
+
+- name: Compile and install librdkafka
+  shell: "{{ item }}"
+  args:
+    chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
+    creates: "{{ librdkafka_home }}/lib/librdkafka.so"
+  with_items:
+    - rm -rf build/
+    - "./configure --prefix={{ librdkafka_home }}"
+    - make
+    - make install

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/main.yml b/metron-deployment/roles/bro/tasks/main.yml
new file mode 100644
index 0000000..14426d6
--- /dev/null
+++ b/metron-deployment/roles/bro/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: librdkafka.yml
+- include: bro.yml
+- include: bro-plugin-kafka.yml
+- include: start-bro.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/tasks/start-bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/start-bro.yml b/metron-deployment/roles/bro/tasks/start-bro.yml
new file mode 100644
index 0000000..1a0b938
--- /dev/null
+++ b/metron-deployment/roles/bro/tasks/start-bro.yml
@@ -0,0 +1,31 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ sniff_interface }}
+  shell: "ip link set {{ sniff_interface }} promisc on"
+
+- name: Start bro
+  shell: "{{ bro_home }}/bin/broctl deploy"
+
+- name: Bro Cronjob
+  cron:
+    name: Bro Cron
+    minute: "{{ bro_crontab_minutes }}"
+    job: "{{ item }}"
+  with_items:
+    - "{{ bro_crontab_job }}"
+    - "{{ bro_clean_job }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/bro/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/vars/main.yml b/metron-deployment/roles/bro/vars/main.yml
new file mode 100644
index 0000000..2ff5177
--- /dev/null
+++ b/metron-deployment/roles/bro/vars/main.yml
@@ -0,0 +1,26 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+bro_home: /usr/local/bro
+bro_version: 2.4.1
+bro_daemon_log: /var/log/bro.log
+bro_topic: bro
+
+# bro cronjob
+bro_crontab_minutes: 0-59/5
+bro_crontab_job: "{{ bro_home }}/bin/broctl cron"
+bro_clean_job: "rm -rf {{ bro_home }}/spool/tmp/*"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/build-tools/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/build-tools/meta/main.yml b/metron-deployment/roles/build-tools/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/metron-deployment/roles/build-tools/meta/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/build-tools/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/build-tools/tasks/main.yml b/metron-deployment/roles/build-tools/tasks/main.yml
new file mode 100644
index 0000000..c47ef43
--- /dev/null
+++ b/metron-deployment/roles/build-tools/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install Build Tools
+  yum: name={{ item }}
+  with_items:
+    - "@Development tools"
+    - libdnet-devel
+    - rpm-build
+    - libpcap
+    - libpcap-devel
+    - pcre
+    - pcre-devel
+    - zlib
+    - zlib-devel
+    - glib2-devel
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/elasticsearch/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/defaults/main.yml b/metron-deployment/roles/elasticsearch/defaults/main.yml
new file mode 100644
index 0000000..0026717
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/defaults/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+elasticsearch_data_dir: /var/lib/elasticsearch
+elasticsearch_network_interface: eth0
+elasticsearch_logrotate_frequency: daily
+elasticsearch_logrotate_retention: 30
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/elasticsearch.repo b/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
new file mode 100644
index 0000000..f033ced
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/files/elasticsearch.repo
@@ -0,0 +1,23 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+[elasticsearch-1.7]
+name=Elasticsearch repository for 1.7.x packages
+baseurl=http://packages.elastic.co/elasticsearch/1.7/centos
+gpgcheck=1
+gpgkey=http://packages.elastic.co/GPG-KEY-elasticsearch
+enabled=1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/elasticsearch/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/files/yaf_index.template b/metron-deployment/roles/elasticsearch/files/yaf_index.template
new file mode 100644
index 0000000..15ff6f8
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/files/yaf_index.template
@@ -0,0 +1,82 @@
+{
+   "template": "yaf_index*",
+   "mappings": {
+      "yaf_doc": {
+         "properties": {
+            "timestamp": {
+               "type": "date",
+               "format": "dateOptionalTime"
+            },
+            "location_point": {
+               "type": "geo_point"
+            },
+            "end-time": {
+               "type": "string"
+            },
+            "duration": {
+               "type": "string"
+            },
+            "rtt": {
+               "type": "string"
+            },
+            "proto": {
+               "type": "string"
+            },
+            "sip": {
+               "type": "string"
+            },
+            "sp": {
+               "type": "string"
+            },
+            "dip": {
+               "type": "string"
+            },
+            "dp": {
+               "type": "string"
+            },
+            "iflags": {
+               "type": "string"
+            },
+            "uflags": {
+               "type": "string"
+            },
+            "riflags": {
+               "type": "string"
+            },
+            "ruflags": {
+               "type": "string"
+            },
+            "isn": {
+               "type": "string"
+            },
+            "risn": {
+               "type": "string"
+            },
+            "tag": {
+               "type": "string"
+            },
+            "rtag": {
+               "type": "string"
+            },
+            "pkt": {
+               "type": "string"
+            },
+            "oct": {
+               "type": "string"
+            },
+            "rpkt": {
+               "type": "string"
+            },
+            "roct": {
+               "type": "string"
+            },
+            "app": {
+               "type": "string"
+            },
+            "end-reason": {
+               "type": "string"
+            }
+         }
+      }
+   }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/elasticsearch/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/meta/main.yml b/metron-deployment/roles/elasticsearch/meta/main.yml
new file mode 100644
index 0000000..f5f059a
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/meta/main.yml
@@ -0,0 +1,24 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk
+  - epel
+  - python-pip
+  - httplib2
+  - libselinux-python
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/elasticsearch/tasks/configure_index.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/configure_index.yml b/metron-deployment/roles/elasticsearch/tasks/configure_index.yml
new file mode 100644
index 0000000..09739be
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/tasks/configure_index.yml
@@ -0,0 +1,44 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name : Wait for Elasticsearch Host to Start
+  wait_for:
+    host: "{{ groups.search[0] }}"
+    port: "{{ elasticsearch_web_port }}"
+    delay: 10
+    timeout: 300
+
+- name: Wait for Green Index Status
+  uri:
+    url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_cat/health"
+    method: GET
+    status_code: 200
+    return_content: yes
+  register: result
+  until: result.content.find("green") != -1
+  retries: 10
+  delay: 60
+  run_once: yes
+
+- name: Add Elasticsearch templates for topologies
+  uri:
+    url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
+    method: POST
+    body: "{{ lookup('file','yaf_index.template') }}"
+    status_code: 200
+    body_format: json
+  run_once: yes

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/tasks/main.yml b/metron-deployment/roles/elasticsearch/tasks/main.yml
new file mode 100644
index 0000000..26554aa
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/tasks/main.yml
@@ -0,0 +1,73 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Add Elasticsearch GPG key.
+  rpm_key:
+    key: https://packages.elastic.co/GPG-KEY-elasticsearch
+    state: present
+
+- name: Add Elasticsearch repository.
+  copy:
+    src: elasticsearch.repo
+    dest: /etc/yum.repos.d/elasticsearch.repo
+    mode: 0644
+
+- name: Install Elasticsearch.
+  yum:
+    name: elasticsearch
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
+
+- name: Create Data Directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    mode: 0755
+    owner: elasticsearch
+    group: elasticsearch
+  when: elasticsearch_data_dir is defined
+  with_items:
+     - '{{ elasticsearch_data_dir.split(",") }}'
+
+- name: Configure Elasticsearch.
+  lineinfile: >
+    dest=/etc/elasticsearch/elasticsearch.yml
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { regexp: '#cluster\.name', line: 'cluster.name: metron' }
+    - { regexp: '#network\.host:', line: 'network.host: _{{
+    elasticsearch_network_interface  }}:ipv4_' }
+    - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
+    line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
+    - { regexp: '#path\.data', line: 'path.data: {{     elasticsearch_data_dir }}' }
+
+- name: Start Elasticsearch.
+  service: name=elasticsearch state=started enabled=yes
+
+- include: configure_index.yml
+
+- name: Create Logrotate Script for Elasticsearch
+  template:
+    src: "metron-elasticsearch-logrotate.yml"
+    dest: "/etc/logrotate.d/metron-elasticsearch"
+    mode: 0644
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml b/metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml
new file mode 100644
index 0000000..5504ce1
--- /dev/null
+++ b/metron-deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml
@@ -0,0 +1,26 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+#Elasticsearch
+/var/log/elasticsearch/*.log {
+  {{ elasticsearch_logrotate_frequency }}
+  rotate {{ elasticsearch_logrotate_retention }}
+  missingok
+  notifempty
+  copytruncate
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/epel/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/epel/tasks/main.yml b/metron-deployment/roles/epel/tasks/main.yml
new file mode 100644
index 0000000..db4e70b
--- /dev/null
+++ b/metron-deployment/roles/epel/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Get epel-repo rpm
+  get_url:
+    dest: /tmp/epel-release.rpm
+    url: http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+
+- name: Install epel-repo rpm
+  yum:
+    pkg: /tmp/epel-release.rpm
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/flume/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/flume/meta/main.yml b/metron-deployment/roles/flume/meta/main.yml
new file mode 100644
index 0000000..ff35a5a
--- /dev/null
+++ b/metron-deployment/roles/flume/meta/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk
+  - libselinux-python

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/flume/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/flume/tasks/main.yml b/metron-deployment/roles/flume/tasks/main.yml
new file mode 100644
index 0000000..8576c3c
--- /dev/null
+++ b/metron-deployment/roles/flume/tasks/main.yml
@@ -0,0 +1,52 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Retrieve HDP repository definition
+  get_url:
+    url: "{{ hdp_repo_def }}"
+    dest: /etc/yum.repos.d/hdp.repo
+    mode: 0644
+
+- name: Install flume
+  yum: name={{item}}
+  with_items:
+    - flume
+    - flume-agent
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10
+
+- name: Create flume-env.sh
+  shell: cp /etc/flume/conf/flume-env.sh.template /etc/flume/conf/flume-env.sh
+
+- name: Configure flume-env.sh
+  lineinfile: >
+    dest=/etc/flume/conf/flume-env.sh
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { regexp: '^.*export JAVA_HOME=.*$', line: 'export JAVA_HOME={{ java_home }}' }
+
+- name: Create flume service
+  shell: "{{item}}"
+  with_items:
+    - cp /usr/hdp/current/flume-server/etc/rc.d/init.d/flume-agent /etc/init.d/
+
+- name: Remove default flume configuration
+  file: path=/etc/flume/conf/flume.conf state=absent

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/flume/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/flume/vars/main.yml b/metron-deployment/roles/flume/vars/main.yml
new file mode 100644
index 0000000..351d125
--- /dev/null
+++ b/metron-deployment/roles/flume/vars/main.yml
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/hadoop_setup/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/defaults/main.yml b/metron-deployment/roles/hadoop_setup/defaults/main.yml
new file mode 100644
index 0000000..99a55f6
--- /dev/null
+++ b/metron-deployment/roles/hadoop_setup/defaults/main.yml
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+num_partitions: 1
+retention_in_gb: 10
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: yaf
+snort_topic: snort
+enrichments_topic: enrichments
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/hadoop_setup/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/meta/main.yml b/metron-deployment/roles/hadoop_setup/meta/main.yml
new file mode 100644
index 0000000..8f0bf2b
--- /dev/null
+++ b/metron-deployment/roles/hadoop_setup/meta/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - ambari_gather_facts
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/hadoop_setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/tasks/main.yml b/metron-deployment/roles/hadoop_setup/tasks/main.yml
new file mode 100644
index 0000000..de01abf
--- /dev/null
+++ b/metron-deployment/roles/hadoop_setup/tasks/main.yml
@@ -0,0 +1,37 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+#must run on hadoop host
+- name: Create HBase tables
+  shell: echo "create '{{ item }}','t'" | hbase shell -n
+  ignore_errors: yes
+  with_items:
+    - "{{ pcap_hbase_table }}"
+    - "{{ tracker_hbase_table }}"
+    - "{{ threatintel_hbase_table }}"
+    - "{{ enrichment_hbase_table }}"
+
+#if kafka topic
+- name: Create Kafka topics
+  shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions {{ num_partitions }} --replication-factor 1 --config retention.bytes={{ retention_in_gb * 1024 * 1024 * 1024}}"
+  ignore_errors: yes
+  with_items:
+    - "{{ pycapa_topic }}"
+    - "{{ bro_topic }}"
+    - "{{ yaf_topic }}"
+    - "{{ snort_topic }}"
+    - "{{ enrichments_topic }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/hadoop_setup/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/hadoop_setup/vars/main.yml b/metron-deployment/roles/hadoop_setup/vars/main.yml
new file mode 100644
index 0000000..9747044
--- /dev/null
+++ b/metron-deployment/roles/hadoop_setup/vars/main.yml
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+kafka_home: /usr/hdp/current/kafka-broker/

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/httplib2/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/httplib2/tasks/main.yml b/metron-deployment/roles/httplib2/tasks/main.yml
new file mode 100644
index 0000000..5502cf4
--- /dev/null
+++ b/metron-deployment/roles/httplib2/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install python httplib2 dependency
+  pip:
+    name: httplib2
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/java_jdk/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/java_jdk/defaults/main.yml b/metron-deployment/roles/java_jdk/defaults/main.yml
new file mode 100644
index 0000000..28f6c71
--- /dev/null
+++ b/metron-deployment/roles/java_jdk/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+java_home: /usr/jdk64/jdk1.8.0_40

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/java_jdk/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/java_jdk/tasks/main.yml b/metron-deployment/roles/java_jdk/tasks/main.yml
new file mode 100644
index 0000000..999b9c1
--- /dev/null
+++ b/metron-deployment/roles/java_jdk/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Check for java at "{{ java_home }}"
+  stat: path="{{ java_home }}"
+  register: jdk_dir
+
+- name: Alternatives link for java
+  alternatives: name={{ item.name }} link={{ item.link }}  path={{ item.path }}
+  with_items:
+    - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
+    - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
+  when: jdk_dir.stat.exists
+
+- name: Install openjdk
+  yum: name={{item}}
+  with_items:
+    - java-1.8.0-openjdk
+    - java-1.8.0-openjdk-devel
+  when: not jdk_dir.stat.exists

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/kafka-broker/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-broker/defaults/main.yml b/metron-deployment/roles/kafka-broker/defaults/main.yml
new file mode 100644
index 0000000..351d125
--- /dev/null
+++ b/metron-deployment/roles/kafka-broker/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/kafka-broker/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-broker/meta/main.yml b/metron-deployment/roles/kafka-broker/meta/main.yml
new file mode 100644
index 0000000..9587e79
--- /dev/null
+++ b/metron-deployment/roles/kafka-broker/meta/main.yml
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+dependencies:
+  - libselinux-python

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/kafka-broker/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-broker/tasks/main.yml b/metron-deployment/roles/kafka-broker/tasks/main.yml
new file mode 100644
index 0000000..db05cb0
--- /dev/null
+++ b/metron-deployment/roles/kafka-broker/tasks/main.yml
@@ -0,0 +1,41 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Retrieve HDP repository definition
+  get_url:
+    url: "{{ hdp_repo_def }}"
+    dest: /etc/yum.repos.d/hdp.repo
+    mode: 0644
+
+- name: Install kafka
+  yum: name={{item}}
+  with_items:
+    - java-1.8.0-openjdk
+    - kafka
+    - zookeeper-server
+
+- name: Create pid directories
+  file: path={{ item }} state=directory mode=0755
+  with_items:
+    - /var/run/zookeeper
+    - /var/run/kafka
+
+- name: Start zookeeper
+  shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start
+
+- name: Start kafka
+  shell: /usr/hdp/current/kafka-broker/bin/kafka start

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/kafka-broker/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-broker/vars/main.yml b/metron-deployment/roles/kafka-broker/vars/main.yml
new file mode 100644
index 0000000..351d125
--- /dev/null
+++ b/metron-deployment/roles/kafka-broker/vars/main.yml
@@ -0,0 +1,18 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/kafka-client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-client/tasks/main.yml b/metron-deployment/roles/kafka-client/tasks/main.yml
new file mode 100644
index 0000000..1674225
--- /dev/null
+++ b/metron-deployment/roles/kafka-client/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+  - name: Retrieve HDP repository definition
+    get_url:
+      url: "{{ hdp_repo_def }}"
+      dest: /etc/yum.repos.d/hdp.repo
+      mode: 0644
+
+  - name: Install kafka
+    yum:
+      name: kafka
+    register: result
+    until: result.rc == 0
+    retries: 5
+    delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/defaults/main.yml b/metron-deployment/roles/librdkafka/defaults/main.yml
new file mode 100644
index 0000000..d920883
--- /dev/null
+++ b/metron-deployment/roles/librdkafka/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+librdkafka_version: 0.8.6
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/0.8.6.tar.gz
+librdkafka_home: /usr/local

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/librdkafka/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/tasks/dependencies.yml b/metron-deployment/roles/librdkafka/tasks/dependencies.yml
new file mode 100644
index 0000000..431e861
--- /dev/null
+++ b/metron-deployment/roles/librdkafka/tasks/dependencies.yml
@@ -0,0 +1,37 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install prerequisites
+  yum: name={{ item }}
+  with_items:
+    - cmake
+    - make
+    - gcc
+    - gcc-c++
+    - flex
+    - bison
+    - libpcap
+    - libpcap-devel
+    - openssl-devel
+    - python-devel
+    - swig
+    - zlib-devel
+    - perl
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/librdkafka/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/tasks/librdkafka.yml b/metron-deployment/roles/librdkafka/tasks/librdkafka.yml
new file mode 100644
index 0000000..652d319
--- /dev/null
+++ b/metron-deployment/roles/librdkafka/tasks/librdkafka.yml
@@ -0,0 +1,39 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Download librdkafka
+  get_url:
+    url: "{{ librdkafka_url }}"
+    dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+
+- name: Extract librdkafka tarball
+  unarchive:
+    src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+    dest: /tmp
+    copy: no
+    creates: "/tmp/librdkafka-{{ librdkafka_version }}"
+
+- name: Compile and install librdkafka
+  shell: "{{ item }}"
+  args:
+    chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
+    creates: "{{ librdkafka_home }}/lib/librdkafka.so"
+  with_items:
+    - rm -rf build/
+    - "./configure --prefix={{ librdkafka_home }}"
+    - make
+    - make install

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/librdkafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/tasks/main.yml b/metron-deployment/roles/librdkafka/tasks/main.yml
new file mode 100644
index 0000000..2144d7f
--- /dev/null
+++ b/metron-deployment/roles/librdkafka/tasks/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: librdkafka.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/libselinux-python/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/libselinux-python/tasks/main.yml b/metron-deployment/roles/libselinux-python/tasks/main.yml
new file mode 100644
index 0000000..78f5a27
--- /dev/null
+++ b/metron-deployment/roles/libselinux-python/tasks/main.yml
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install libselinux-python
+  yum:
+    name: libselinux-python
+    state: installed
+  register: result
+  until: result.rc == 0
+  retries: 5
+  delay: 10

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_common/defaults/main.yml b/metron-deployment/roles/metron_common/defaults/main.yml
new file mode 100644
index 0000000..e4a7735
--- /dev/null
+++ b/metron-deployment/roles/metron_common/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+metron_jar_name: metron-elasticsearch-{{ metron_version }}.jar
+metron_jar_path: "{{ playbook_dir }}/../../metron-platform/metron-elasticsearch/target/{{ metron_jar_name }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_common/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_common/meta/main.yml b/metron-deployment/roles/metron_common/meta/main.yml
new file mode 100644
index 0000000..4db50aa
--- /dev/null
+++ b/metron-deployment/roles/metron_common/meta/main.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - yum-update
+  - epel
+  - ntp
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_common/tasks/main.yml b/metron-deployment/roles/metron_common/tasks/main.yml
new file mode 100644
index 0000000..64e6ab9
--- /dev/null
+++ b/metron-deployment/roles/metron_common/tasks/main.yml
@@ -0,0 +1,35 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Check OS Version
+  fail: msg="Metron deployment supports CentOS 6 only."
+  when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6")
+
+- name: Check for Metron jar path
+  stat: path={{ metron_jar_path }}
+  register: metron_jars
+
+- name: Verify Metron jars exist
+  fail: msg="Unable to locate staged Metron jars at {{ metron_jar_path }}.  Did you run 'mvn package'?"
+  when: metron_jars.stat.exists == True
+
+- name: Ensure iptables is stopped and is not running at boot time.
+  ignore_errors: yes
+  service:
+    name: iptables
+    state: stopped
+    enabled: no

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/defaults/main.yml b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
new file mode 100644
index 0000000..2ff6c6b
--- /dev/null
+++ b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
@@ -0,0 +1,24 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+metron_version: 0.1BETA
+metron_directory: /usr/metron/{{ metron_version }}
+pcapservice_jar_name: metron-api-{{ metron_version }}-jar-with-dependencies.jar
+pcapservice_jar_src: "{{ playbook_dir }}/../../metron-platform/metron-api/target/{{ pcapservice_jar_name }}"
+pcapservice_jar_dst: "{{ metron_directory }}/lib/{{ pcapservice_jar_name }}"
+pcapservice_port: 8081
+hbase_config_path: "/etc/hbase/conf"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_pcapservice/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/meta/main.yml b/metron-deployment/roles/metron_pcapservice/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/metron-deployment/roles/metron_pcapservice/meta/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+dependencies:
+  - java_jdk

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml b/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml
new file mode 100644
index 0000000..b77c1ec
--- /dev/null
+++ b/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml
@@ -0,0 +1,26 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+ - name: Fetch hbase-site.xml
+   fetch: src=/etc/hbase/conf/hbase-site.xml dest=/tmp/hbase/conf/hbase-site.xml flat=yes
+   delegate_to: "{{ groups.ambari_slave[0] }}"
+
+ - name: Create hbase conf directory
+   file: path=/etc/hbase/conf state=directory mode=0755
+
+ - name: Copy hbase-site.xml
+   copy: src=/tmp/hbase/conf/hbase-site.xml dest=/etc/hbase/conf/hbase-site.xml mode=0644

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_pcapservice/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/tasks/main.yml b/metron-deployment/roles/metron_pcapservice/tasks/main.yml
new file mode 100644
index 0000000..ee9cac2
--- /dev/null
+++ b/metron-deployment/roles/metron_pcapservice/tasks/main.yml
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Check for hbase-site
+  stat: path=/etc/hbase/conf/hbase-site.xml
+  register: hbase_site
+
+- include: config-hbase.yml
+  when: not hbase_site.stat.exists
+
+- include: pcapservice.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml b/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
new file mode 100644
index 0000000..651f7fb
--- /dev/null
+++ b/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
@@ -0,0 +1,38 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create Metron streaming directories
+  file: path={{ metron_directory }}/{{ item.name }}  state=directory mode=0755
+  with_items:
+      - { name: 'lib'}
+      - { name: 'config'}
+
+- name: Copy Metron pcapservice jar
+  copy:
+    src: "{{ pcapservice_jar_src }}"
+    dest: "{{ pcapservice_jar_dst }}"
+
+- name: Add hbase-site.xml to pcapservice jar
+  shell: "jar -uf {{ pcapservice_jar_dst }} hbase-site.xml"
+  args:
+    chdir: "{{ hbase_config_path }}"
+
+- name: Install service script
+  template: src=pcapservice dest=/etc/init.d/pcapservice mode=0755
+
+- name: Start pcapservice
+  service: name=pcapservice state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_pcapservice/templates/pcapservice
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/templates/pcapservice b/metron-deployment/roles/metron_pcapservice/templates/pcapservice
new file mode 100644
index 0000000..a3ad92b
--- /dev/null
+++ b/metron-deployment/roles/metron_pcapservice/templates/pcapservice
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# metron pcap service
+# chkconfig: 345 20 80
+# description: Metron PCAP Service Daemon
+# processname: pcapservice
+#
+NAME=pcapservice
+DESC="Metron pcap service"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE="/var/log/metron_pcapservice.log"
+EXTRA_ARGS="${@:2}"
+DAEMON_PATH="/"
+DAEMON="/usr/bin/java"
+DAEMONOPTS="-cp {{ pcapservice_jar_dst }} org.apache.metron.pcapservice.rest.PcapService -port {{ pcapservice_port }}"
+
+case "$1" in
+  start)
+    printf "%-50s" "Starting $NAME..."
+
+    # kick-off the daemon
+    cd $DAEMON_PATH
+    PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!`
+    if [ -z $PID ]; then
+        printf "%s\n" "Fail"
+    else
+        echo $PID > $PIDFILE
+        printf "%s\n" "Ok"
+    fi
+  ;;
+
+  status)
+    printf "%-50s" "Checking $NAME..."
+    if [ -f $PIDFILE ]; then
+      PID=`cat $PIDFILE`
+      if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+        printf "%s\n" "Process dead but pidfile exists"
+      else
+        echo "Running"
+      fi
+    else
+      printf "%s\n" "Service not running"
+    fi
+  ;;
+
+  stop)
+    printf "%-50s" "Stopping $NAME"
+    PID=`cat $PIDFILE`
+    cd $DAEMON_PATH
+    if [ -f $PIDFILE ]; then
+        kill -HUP $PID
+        printf "%s\n" "Ok"
+        rm -f $PIDFILE
+    else
+        printf "%s\n" "pidfile not found"
+    fi
+  ;;
+
+  restart)
+    $0 stop
+    $0 start
+  ;;
+
+  *)
+    echo "Usage: $0 {status|start|stop|restart}"
+    exit 1
+esac

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/defaults/main.yml b/metron-deployment/roles/metron_streaming/defaults/main.yml
new file mode 100644
index 0000000..d799b33
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/defaults/main.yml
@@ -0,0 +1,81 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+metron_directory: /usr/metron/{{ metron_version }}
+metron_solr_jar_name: metron-solr-{{ metron_version }}.jar
+metron_elasticsearch_jar_name: metron-elasticsearch-{{ metron_version }}.jar
+metron_parsers_jar_name: metron-parsers-{{ metron_version }}.jar
+
+metron_common_bundle_name: metron-common-{{ metron_version }}-archive.tar.gz
+metron_data_management_bundle_name: metron-data-management-{{ metron_version }}-archive.tar.gz
+metron_enrichment_bundle_name: metron-enrichment-{{ metron_version }}-archive.tar.gz
+metron_solr_bundle_name: metron-solr-{{ metron_version }}-archive.tar.gz
+metron_elasticsearch_bundle_name: metron-elasticsearch-{{ metron_version }}-archive.tar.gz
+metron_parsers_bundle_name: metron-parsers-{{ metron_version }}-archive.tar.gz
+metron_common_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-common/target/{{ metron_common_bundle_name }}"
+metron_data_management_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-data-management/target/{{ metron_data_management_bundle_name }}"
+metron_enrichment_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-enrichment/target/{{ metron_enrichment_bundle_name }}"
+metron_solr_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-solr/target/{{ metron_solr_bundle_name }}"
+metron_elasticsearch_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-elasticsearch/target/{{ metron_elasticsearch_bundle_name }}"
+metron_parsers_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-parsers/target/{{ metron_parsers_bundle_name }}"
+
+
+config_path: "{{ metron_directory }}/config"
+zookeeper_config_path: "{{ config_path }}/zookeeper"
+zookeeper_global_config_path: "{{ zookeeper_config_path }}/global.json"
+metron_solr_properties_config_path: "{{ metron_directory }}/config/solr.properties"
+metron_elasticsearch_properties_config_path: "{{ metron_directory }}/config/elasticsearch.properties"
+metron_parsers_properties_config_path: "{{ metron_directory }}/config/parsers.properties"
+hbase_config_path: "/etc/hbase/conf"
+hdfs_config_path: "/etc/hadoop/conf"
+
+threat_intel_bulk_load: True
+threat_intel_bin: "{{ metron_directory }}/bin/threatintel_bulk_load.sh"
+threat_intel_work_dir: /tmp/ti_bulk
+threat_intel_csv_filename: "threat_ip.csv"
+threat_intel_csv_filepath: "{{ threat_intel_csv_filename }}"
+
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: yaf
+snort_topic: snort
+enrichments_topic: enrichments
+storm_enrichment_topology:
+    - "{{ metron_directory }}/flux/enrichment/remote.yaml"
+storm_parser_topologies:
+    - "{{ metron_directory }}/flux/bro/remote.yaml"
+    - "{{ metron_directory }}/flux/snort/remote.yaml"
+    - "{{ metron_directory }}/flux/yaf/remote.yaml"
+
+hdfs_retention_days: 30
+hdfs_bro_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/enrichment/indexed/bro_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/bro-purge/cron-hdfs-bro-purge.log 2>&1"
+hdfs_yaf_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/enrichment/indexed/yaf_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-hdfs-yaf-purge.log 2>&1"
+hdfs_snort_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/enrichment/indexed/snort_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-hdfs-yaf-purge.log 2>&1"
+
+elasticsearch_config_path: /etc/elasticsearch
+elasticsearch_cluster_name: metron
+elasticsearch_transport_port: 9300
+
+es_retention_days: 30
+es_bro_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p bro_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/bro-purge/cron-es-bro-purge.log 2>&1"
+es_yaf_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p yaf_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-es-yaf-purge.log 2>&1"
+es_snort_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p yaf_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/snort-purge/cron-es-snort-purge.log 2>&1"
+
+metron_hdfs_output_dir: "/apps/metron"
+metron_hdfs_rotation_policy: org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
+metron_hdfs_rotation_policy_count: 1
+metron_hdfs_rotation_policy_units: DAYS

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/0117987e/metron-deployment/roles/metron_streaming/files/config/sensors/bro.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/config/sensors/bro.json b/metron-deployment/roles/metron_streaming/files/config/sensors/bro.json
new file mode 100644
index 0000000..2b534b4
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/files/config/sensors/bro.json
@@ -0,0 +1,19 @@
+{
+  "index": "bro",
+  "batchSize": 5,
+  "enrichmentFieldMap":
+  {
+    "geo": ["ip_dst_addr", "ip_src_addr"],
+    "host": ["host"]
+  },
+  "threatIntelFieldMap":
+  {
+    "hbaseThreatIntel": ["ip_src_addr", "ip_dst_addr"]
+  },
+  "fieldToThreatIntelTypeMap":
+  {
+    "ip_src_addr" : ["malicious_ip"],
+    "ip_dst_addr" : ["malicious_ip"]
+  }
+}
+



Mime
View raw message