metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [11/30] incubator-metron git commit: Ansible and vagrant scripts to deploy Metron
Date Fri, 29 Jan 2016 20:04:24 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/flume/files/flume-ipfix.conf
----------------------------------------------------------------------
diff --git a/deployment/roles/flume/files/flume-ipfix.conf b/deployment/roles/flume/files/flume-ipfix.conf
new file mode 100644
index 0000000..fa97ca7
--- /dev/null
+++ b/deployment/roles/flume/files/flume-ipfix.conf
@@ -0,0 +1,27 @@
+
+ipfix.sources = netcat-source
+ipfix.channels = memory-channel
+ipfix.sinks = kafka-sink logger-sink
+
+# ipfix publishes ipfix to local tcp port 4739
+ipfix.sources.netcat-source.type = netcat
+ipfix.sources.netcat-source.bind = localhost
+ipfix.sources.netcat-source.port = 4739
+
+# data is sent to kafka
+ipfix.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
+ipfix.sinks.kafka-sink.brokerList = [[value overwritten by ansible]]
+ipfix.sinks.kafka-sink.topic = [[value overwritten by ansible]]
+
+# also log events
+ipfix.sinks.logger-sink.type = logger
+
+# buffer events in memory
+ipfix.channels.memory-channel.type = memory
+ipfix.channels.memory-channel.capacity = 1000
+ipfix.channels.memory-channel.transactionCapacity = 100
+
+# bind the source and sink to the channel
+ipfix.sources.netcat-source.channels = memory-channel
+ipfix.sinks.kafka-sink.channel = memory-channel
+ipfix.sinks.logger-sink.channel = memory-channel

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/flume/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/flume/tasks/main.yml b/deployment/roles/flume/tasks/main.yml
new file mode 100644
index 0000000..61af35a
--- /dev/null
+++ b/deployment/roles/flume/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+- name: Retrieve HDP repository definition
+  get_url:
+    url: "{{ hdp_repo_def }}"
+    dest: /etc/yum.repos.d/hdp.repo
+    mode: 0644
+
+- name: Install flume
+  yum: name={{item}} state=latest update_cache=yes
+  with_items:
+    - flume
+    - flume-agent
+
+- name: Create flume-env.sh
+  shell: cp /etc/flume/conf/flume-env.sh.template /etc/flume/conf/flume-env.sh
+
+- name: Configure flume-env.sh
+  lineinfile: >
+    dest=/etc/flume/conf/flume-env.sh
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { regexp: '^.*export JAVA_HOME=.*$', line: 'export JAVA_HOME=/usr/jdk64/jdk1.8.0_40/' }
+
+- name: Create flume service
+  shell: "{{item}}"
+  with_items:
+    - cp /usr/hdp/current/flume-server/etc/rc.d/init.d/flume-agent /etc/init.d/
+
+- name: Remove default flume configuration
+  shell: rm -f /etc/flume/conf/flume.conf

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/flume/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/flume/vars/main.yml b/deployment/roles/flume/vars/main.yml
new file mode 100644
index 0000000..d6af4b7
--- /dev/null
+++ b/deployment/roles/flume/vars/main.yml
@@ -0,0 +1,2 @@
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_pcapservice/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/README.md b/deployment/roles/metron_pcapservice/README.md
new file mode 100644
index 0000000..225dd44
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/README.md
@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/defaults/main.yml b/deployment/roles/metron_pcapservice/defaults/main.yml
new file mode 100644
index 0000000..a50f9ca
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for ambari_common

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_pcapservice/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/handlers/main.yml b/deployment/roles/metron_pcapservice/handlers/main.yml
new file mode 100644
index 0000000..a4c1162
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+- name: restart elasticsearch
+  service: name=elasticsearch state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_pcapservice/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/meta/main.yml b/deployment/roles/metron_pcapservice/meta/main.yml
new file mode 100644
index 0000000..62c7d35
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/meta/main.yml
@@ -0,0 +1,139 @@
+---
+galaxy_info:
+  author: your name
+  description: 
+  company: your company (optional)
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+  min_ansible_version: 1.2
+  #
+  # Below are all platforms currently available. Just uncomment
+  # the ones that apply to your role. If you don't see your 
+  # platform on this list, let us know and we'll get it added!
+  #
+  #platforms:
+  #- name: EL
+  #  versions:
+  #  - all
+  #  - 5
+  #  - 6
+  #  - 7
+  #- name: GenericUNIX
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Fedora
+  #  versions:
+  #  - all
+  #  - 16
+  #  - 17
+  #  - 18
+  #  - 19
+  #  - 20
+  #  - 21
+  #  - 22
+  #- name: Windows
+  #  versions:
+  #  - all
+  #  - 2012R2
+  #- name: SmartOS
+  #  versions:
+  #  - all
+  #  - any
+  #- name: opensuse
+  #  versions:
+  #  - all
+  #  - 12.1
+  #  - 12.2
+  #  - 12.3
+  #  - 13.1
+  #  - 13.2
+  #- name: Amazon
+  #  versions:
+  #  - all
+  #  - 2013.03
+  #  - 2013.09
+  #- name: GenericBSD
+  #  versions:
+  #  - all
+  #  - any
+  #- name: FreeBSD
+  #  versions:
+  #  - all
+  #  - 8.0
+  #  - 8.1
+  #  - 8.2
+  #  - 8.3
+  #  - 8.4
+  #  - 9.0
+  #  - 9.1
+  #  - 9.1
+  #  - 9.2
+  #- name: Ubuntu
+  #  versions:
+  #  - all
+  #  - lucid
+  #  - maverick
+  #  - natty
+  #  - oneiric
+  #  - precise
+  #  - quantal
+  #  - raring
+  #  - saucy
+  #  - trusty
+  #  - utopic
+  #  - vivid
+  #- name: SLES
+  #  versions:
+  #  - all
+  #  - 10SP3
+  #  - 10SP4
+  #  - 11
+  #  - 11SP1
+  #  - 11SP2
+  #  - 11SP3
+  #- name: GenericLinux
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Debian
+  #  versions:
+  #  - all
+  #  - etch
+  #  - jessie
+  #  - lenny
+  #  - squeeze
+  #  - wheezy
+  #
+  # Below are all categories currently available. Just as with
+  # the platforms above, uncomment those that apply to your role.
+  #
+  #categories:
+  #- cloud
+  #- cloud:ec2
+  #- cloud:gce
+  #- cloud:rax
+  #- clustering
+  #- database
+  #- database:nosql
+  #- database:sql
+  #- development
+  #- monitoring
+  #- networking
+  #- packaging
+  #- system
+  #- web
+dependencies: []
+  # List your role dependencies here, one per line.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.
+  

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_pcapservice/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/main.yml b/deployment/roles/metron_pcapservice/tasks/main.yml
new file mode 100644
index 0000000..060340a
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/tasks/main.yml
@@ -0,0 +1,38 @@
+---
+# tasks file for metron_pcapservice
+
+- set_fact:
+    pcapservice_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Pcap_Service/target/{{ pcapservice_jar_name }}"
+
+- name: Create Metron streaming directories
+  file: path={{ metron_directory }}/{{ item.name }}  state=directory mode=0755
+  with_items:
+      - { name: 'lib'}
+      - { name: 'config'}
+
+#- name: Dowloand Metron topology code
+#  get_url: url={{ pcapservice_jar_url }} dest={{ metron_directory }}/lib/
+
+- name: Copy Metron pcapservice jar
+  copy:
+    src: "{{ pcapservice_jar_path }}"
+    dest: "{{ metron_directory }}/lib/"
+
+- name: alternatives link for "java"
+  alternatives: name={{ item.name }} link={{ item.link }}  path={{ item.path }}
+  with_items:
+    - { name: java, link: /usr/bin/java, path: /usr/jdk64/jdk1.8.0_40/bin/java}
+    - { name: jar, link: /usr/bin/jar, path: /usr/jdk64/jdk1.8.0_40/bin/jar}
+
+- name: Add hbase-site.xml to pcapservice jar
+  shell: cd /etc/hbase/conf/ && jar -uf {{ metron_directory }}/lib/{{ pcapservice_jar_name }} hbase-site.xml
+
+- name: Install prerequisites
+  yum: name=daemonize state=latest
+
+- name: Start pcap_service
+  shell: "daemonize -c {{ metron_directory }} -e /var/log/metron_pcapservice.log -o /var/log/metron_pcapservice.log -l /var/lock/subsys/metron_pcapservice /usr/bin/java -cp {{ metron_directory }}/lib/{{ pcapservice_jar_name }} com.opensoc.pcapservice.rest.PcapService"
+  args:
+    creates: /var/lock/subsys/metron_pcapservice
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_pcapservice/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/vars/main.yml b/deployment/roles/metron_pcapservice/vars/main.yml
new file mode 100644
index 0000000..d129a49
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/vars/main.yml
@@ -0,0 +1,6 @@
+---
+# vars file for metron_pcapservice
+metron_version: 0.6BETA
+metron_directory: /usr/metron/{{ metron_version }}
+pcapservice_jar_name: Metron-Pcap_Service-{{ metron_version }}-jar-with-dependencies.jar
+pcapservice_jar_url: http://{{ pcapservice_host }}:8080/repository/internal/com/opensoc/OpenSOC-Pcap_Service/{{ metron_version }}/{{ pcapservice_jar_name }}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
new file mode 100644
index 0000000..a50f9ca
--- /dev/null
+++ b/deployment/roles/metron_streaming/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for ambari_common

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_streaming/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/handlers/main.yml b/deployment/roles/metron_streaming/handlers/main.yml
new file mode 100644
index 0000000..a4c1162
--- /dev/null
+++ b/deployment/roles/metron_streaming/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+- name: restart elasticsearch
+  service: name=elasticsearch state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_streaming/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/meta/main.yml b/deployment/roles/metron_streaming/meta/main.yml
new file mode 100644
index 0000000..62c7d35
--- /dev/null
+++ b/deployment/roles/metron_streaming/meta/main.yml
@@ -0,0 +1,139 @@
+---
+galaxy_info:
+  author: your name
+  description: 
+  company: your company (optional)
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+  min_ansible_version: 1.2
+  #
+  # Below are all platforms currently available. Just uncomment
+  # the ones that apply to your role. If you don't see your 
+  # platform on this list, let us know and we'll get it added!
+  #
+  #platforms:
+  #- name: EL
+  #  versions:
+  #  - all
+  #  - 5
+  #  - 6
+  #  - 7
+  #- name: GenericUNIX
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Fedora
+  #  versions:
+  #  - all
+  #  - 16
+  #  - 17
+  #  - 18
+  #  - 19
+  #  - 20
+  #  - 21
+  #  - 22
+  #- name: Windows
+  #  versions:
+  #  - all
+  #  - 2012R2
+  #- name: SmartOS
+  #  versions:
+  #  - all
+  #  - any
+  #- name: opensuse
+  #  versions:
+  #  - all
+  #  - 12.1
+  #  - 12.2
+  #  - 12.3
+  #  - 13.1
+  #  - 13.2
+  #- name: Amazon
+  #  versions:
+  #  - all
+  #  - 2013.03
+  #  - 2013.09
+  #- name: GenericBSD
+  #  versions:
+  #  - all
+  #  - any
+  #- name: FreeBSD
+  #  versions:
+  #  - all
+  #  - 8.0
+  #  - 8.1
+  #  - 8.2
+  #  - 8.3
+  #  - 8.4
+  #  - 9.0
+  #  - 9.1
+  #  - 9.1
+  #  - 9.2
+  #- name: Ubuntu
+  #  versions:
+  #  - all
+  #  - lucid
+  #  - maverick
+  #  - natty
+  #  - oneiric
+  #  - precise
+  #  - quantal
+  #  - raring
+  #  - saucy
+  #  - trusty
+  #  - utopic
+  #  - vivid
+  #- name: SLES
+  #  versions:
+  #  - all
+  #  - 10SP3
+  #  - 10SP4
+  #  - 11
+  #  - 11SP1
+  #  - 11SP2
+  #  - 11SP3
+  #- name: GenericLinux
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Debian
+  #  versions:
+  #  - all
+  #  - etch
+  #  - jessie
+  #  - lenny
+  #  - squeeze
+  #  - wheezy
+  #
+  # Below are all categories currently available. Just as with
+  # the platforms above, uncomment those that apply to your role.
+  #
+  #categories:
+  #- cloud
+  #- cloud:ec2
+  #- cloud:gce
+  #- cloud:rax
+  #- clustering
+  #- database
+  #- database:nosql
+  #- database:sql
+  #- development
+  #- monitoring
+  #- networking
+  #- packaging
+  #- system
+  #- web
+dependencies: []
+  # List your role dependencies here, one per line.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.
+  

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
new file mode 100644
index 0000000..f32df66
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -0,0 +1,76 @@
+---
+- set_fact:
+    metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
+
+# - name: Build Metron streaming
+#   shell: "{{ item }}"
+#   args:
+#     chdir: "{{ playbook_dir }}../../../opensoc-streaming"
+#     creates: "{{ metron_jar_path }}"
+#   with_items:
+#     - mvn package
+
+- name: Create Metron streaming directories
+  file: path="{{ metron_directory }}/{{ item.name }}"  state=directory mode=0755
+  with_items:
+      - { name: 'lib'}
+      - { name: 'config'}
+
+- name: Copy Metron streaming jars
+  copy:
+    src: "{{ metron_jar_path }}"
+    dest: "{{ metron_directory }}/lib/"
+
+- name: Alternatives link for "java"
+  alternatives: name={{ item.name }} link={{ item.link }}  path={{ item.path }}
+  with_items:
+    - { name: java, link: /usr/bin/java, path: /usr/jdk64/jdk1.8.0_40/bin/java}
+    - { name: jar, link: /usr/bin/jar, path: /usr/jdk64/jdk1.8.0_40/bin/jar}
+
+- name: Add hbase-site.xml to topology jar
+  shell: cd /etc/hbase/conf/ && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} hbase-site.xml
+
+- name: Add elasticsearch.ym to topology jar
+  shell: cd /etc/elasticsearch && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} elasticsearch.yml
+
+- name: Download Metron topology config templates
+  get_url: url={{ metron_source_url }} dest=/tmp/metron_source.zip
+
+- name: Unpack Metron source
+  unarchive: src=/tmp/metron_source.zip dest=/tmp copy=no
+
+- name: Copy unpacked environment configs to config directory
+  command: cp -r /tmp/incubator-metron-{{ metron_branch }}/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc {{ metron_directory }}/config
+
+- name: Copy unpacked topology configs to config directory
+  command: cp -r /tmp/incubator-metron-{{ metron_branch }}/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies {{ metron_directory }}/config
+
+- name: Cleanup unpacked topology configs
+  file:
+    path: "{{ item }}"
+    state: absent
+  with_items:
+    - "/tmp/metron_source.zip"
+    - "/tmp/incubator-metron-{{ metron_branch }}"
+
+- name: Configure config.properties
+  lineinfile: >
+    dest={{ metron_directory }}/config/etc/env/config.properties
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+    - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
+    - { regexp: "es.ip=", line: "es.ip={{ elasticsearch_ip }}" }
+    - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
+    - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
+    - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
+
+- name: Create HBase tables
+  shell: echo "create 'pcap_test','t'" | hbase shell -n
+  ignore_errors: yes
+
+- name: Submit Pcap topology
+  command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ metron_directory }}/config/topologies/pcap/remote.yaml --filter {{ metron_directory }}/config/etc/env/config.properties
+  ignore_errors: yes

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_streaming/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/vars/main.yml b/deployment/roles/metron_streaming/vars/main.yml
new file mode 100644
index 0000000..fa77ea4
--- /dev/null
+++ b/deployment/roles/metron_streaming/vars/main.yml
@@ -0,0 +1,8 @@
+---
+# vars file for elasticsearch
+metron_version: 0.6BETA
+metron_directory: /usr/metron/{{ metron_version }}
+metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
+metron_jar_url: http://52.90.121.195:8080/repository/internal/com/opensoc/OpenSOC-Topologies/{{ metron_version }}/{{ metron_jar_name }}
+metron_branch: topic_per_source
+metron_source_url: https://github.com/merrimanr/incubator-metron/archive/{{ metron_branch }}.zip

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_ui/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/defaults/main.yml b/deployment/roles/metron_ui/defaults/main.yml
new file mode 100644
index 0000000..a50f9ca
--- /dev/null
+++ b/deployment/roles/metron_ui/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for ambari_common

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_ui/files/opensoc-ui
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/files/opensoc-ui b/deployment/roles/metron_ui/files/opensoc-ui
new file mode 100644
index 0000000..16e3dbe
--- /dev/null
+++ b/deployment/roles/metron_ui/files/opensoc-ui
@@ -0,0 +1,6 @@
+{
+  "auth":false,
+  "secrent":"secret",
+  "elasticsearch": { "url": "http://host:port" },
+  "pcap": { "url": "http://host:port/pcap/pcapGetter","mock": false }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_ui/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/handlers/main.yml b/deployment/roles/metron_ui/handlers/main.yml
new file mode 100644
index 0000000..a4c1162
--- /dev/null
+++ b/deployment/roles/metron_ui/handlers/main.yml
@@ -0,0 +1,3 @@
+---
+- name: restart elasticsearch
+  service: name=elasticsearch state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_ui/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/meta/main.yml b/deployment/roles/metron_ui/meta/main.yml
new file mode 100644
index 0000000..62c7d35
--- /dev/null
+++ b/deployment/roles/metron_ui/meta/main.yml
@@ -0,0 +1,139 @@
+---
+galaxy_info:
+  author: your name
+  description: 
+  company: your company (optional)
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+  min_ansible_version: 1.2
+  #
+  # Below are all platforms currently available. Just uncomment
+  # the ones that apply to your role. If you don't see your 
+  # platform on this list, let us know and we'll get it added!
+  #
+  #platforms:
+  #- name: EL
+  #  versions:
+  #  - all
+  #  - 5
+  #  - 6
+  #  - 7
+  #- name: GenericUNIX
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Fedora
+  #  versions:
+  #  - all
+  #  - 16
+  #  - 17
+  #  - 18
+  #  - 19
+  #  - 20
+  #  - 21
+  #  - 22
+  #- name: Windows
+  #  versions:
+  #  - all
+  #  - 2012R2
+  #- name: SmartOS
+  #  versions:
+  #  - all
+  #  - any
+  #- name: opensuse
+  #  versions:
+  #  - all
+  #  - 12.1
+  #  - 12.2
+  #  - 12.3
+  #  - 13.1
+  #  - 13.2
+  #- name: Amazon
+  #  versions:
+  #  - all
+  #  - 2013.03
+  #  - 2013.09
+  #- name: GenericBSD
+  #  versions:
+  #  - all
+  #  - any
+  #- name: FreeBSD
+  #  versions:
+  #  - all
+  #  - 8.0
+  #  - 8.1
+  #  - 8.2
+  #  - 8.3
+  #  - 8.4
+  #  - 9.0
+  #  - 9.1
+  #  - 9.1
+  #  - 9.2
+  #- name: Ubuntu
+  #  versions:
+  #  - all
+  #  - lucid
+  #  - maverick
+  #  - natty
+  #  - oneiric
+  #  - precise
+  #  - quantal
+  #  - raring
+  #  - saucy
+  #  - trusty
+  #  - utopic
+  #  - vivid
+  #- name: SLES
+  #  versions:
+  #  - all
+  #  - 10SP3
+  #  - 10SP4
+  #  - 11
+  #  - 11SP1
+  #  - 11SP2
+  #  - 11SP3
+  #- name: GenericLinux
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Debian
+  #  versions:
+  #  - all
+  #  - etch
+  #  - jessie
+  #  - lenny
+  #  - squeeze
+  #  - wheezy
+  #
+  # Below are all categories currently available. Just as with
+  # the platforms above, uncomment those that apply to your role.
+  #
+  #categories:
+  #- cloud
+  #- cloud:ec2
+  #- cloud:gce
+  #- cloud:rax
+  #- clustering
+  #- database
+  #- database:nosql
+  #- database:sql
+  #- development
+  #- monitoring
+  #- networking
+  #- packaging
+  #- system
+  #- web
+dependencies: []
+  # List your role dependencies here, one per line.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.
+  

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/main.yml b/deployment/roles/metron_ui/tasks/main.yml
new file mode 100644
index 0000000..3a4372c
--- /dev/null
+++ b/deployment/roles/metron_ui/tasks/main.yml
@@ -0,0 +1,46 @@
+---
+# tasks file for metron_ui
+- name: Install Metron UI dependencies
+  yum: pkg={{ item.package }} state=installed
+  with_items:
+      - { package: 'libpcap-devel'}
+      - { package: 'wireshark'}
+      - { package: 'nodejs'}
+      - { package: 'npm'}
+
+- name: Download nvm installer
+  get_url: url=https://raw.githubusercontent.com/creationix/nvm/v0.30.1/install.sh dest=/tmp/nvm-install.sh
+
+- name: Change nvm installer permissions
+  file: path=/tmp/nvm-install.sh mode=0755
+
+- name: Install nvm
+  shell: /tmp/nvm-install.sh creates=~/.nvm/nvm.sh
+
+- name: Install Metron UI
+  shell: source ~/.bashrc && nvm install 0.10.35 && nvm use 0.10.35 && nvm alias default 0.10.35 && npm install -g opensoc-ui creates=/root/.nvm/v0.10.35/lib/node_modules/opensoc-ui/package.json
+
+- name: Add Metron UI config
+  copy:
+    src: opensoc-ui
+    dest: ~/.opensoc-ui
+    mode: 0644
+
+- name: Configure opensoc-ui
+  lineinfile: >
+    dest=~/.opensoc-ui
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { regexp: '"elasticsearch":', line: '"  elasticsearch": { "url": "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}" },' }
+
+- name: Fix pcap_all error
+  replace:
+    dest: /root/.nvm/v0.10.35/lib/node_modules/opensoc-ui/lib/static_dist/app/panels/pcap/module.js
+    regexp: "pcap_all"
+    replace: "pcap_*"
+    backup: yes
+
+- name: Start Metron UI service
+  shell: source ~/.bashrc && opensoc-ui start
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/metron_ui/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/vars/main.yml b/deployment/roles/metron_ui/vars/main.yml
new file mode 100644
index 0000000..3ddeeba
--- /dev/null
+++ b/deployment/roles/metron_ui/vars/main.yml
@@ -0,0 +1,6 @@
+---
+# vars file for elasticsearch
+metron_version: 0.6BETA
+metron_directory: /usr/metron/{{ metron_version }}
+metron_jar_url: http://192.168.1.174:8080/repository/internal/com/opensoc/OpenSOC-Topologies/{{ metron_version }}/OpenSOC-Topologies-{{ metron_version }}.jar
+metron_source_url: https://github.com/merrimanr/incubator-metron/archive/master.zip

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/pycapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/tasks/main.yml b/deployment/roles/pycapa/tasks/main.yml
new file mode 100644
index 0000000..13c67e9
--- /dev/null
+++ b/deployment/roles/pycapa/tasks/main.yml
@@ -0,0 +1,28 @@
+---
+- name: Install python and tcpdump
+  yum: name={{item}} state=latest update_cache=no
+  with_items:
+    - python
+    - python-pip
+    - tcpdump
+    - git
+
+- name: Clone pycapa repo
+  git: repo={{ pycapa_repo }} dest={{ pycapa_home }}
+
+- name: Build pycapa
+  shell: "{{ item }}"
+  args:
+    chdir: "{{ pycapa_home }}"
+  with_items:
+    - pip install -r requirements.txt
+    - pip install argparse
+    - python setup.py install
+
+- name: Install prerequisites
+  yum: name=daemonize state=latest
+
+- name: Start pycapa
+  shell: "daemonize -c {{ pycapa_home }} -e /var/log/pycapa.log -o /var/log/pycapa.log -l /var/lock/subsys/pycapa /usr/bin/python /opt/pycapa/pycapa/pycapa_cli.py  -z {{ zookeeper_url }} -t {{ pycapa_topic }} -i {{ iface }}"
+  args:
+    creates: /var/lock/subsys/pycapa

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/pycapa/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/vars/main.yml b/deployment/roles/pycapa/vars/main.yml
new file mode 100644
index 0000000..3d7ed3a
--- /dev/null
+++ b/deployment/roles/pycapa/vars/main.yml
@@ -0,0 +1,4 @@
+---
+pycapa_repo: https://github.com/OpenSOC/pycapa.git
+pycapa_home: /opt/pycapa
+pycapa_topic: pcap

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/snort/files/flume-snort.conf
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/files/flume-snort.conf b/deployment/roles/snort/files/flume-snort.conf
new file mode 100644
index 0000000..152228e
--- /dev/null
+++ b/deployment/roles/snort/files/flume-snort.conf
@@ -0,0 +1,28 @@
+
+snort.sources = exec-source
+snort.channels = memory-channel
+snort.sinks = kafka-sink logger-sink
+
+# snort alerts are logged to a file
+snort.sources.exec-source.type = exec
+snort.sources.exec-source.command = tail -F /var/log/snort/alert
+snort.sources.exec-source.restart = true
+snort.sources.exec-source.logStdErr = true
+
+# snort alerts are sent to kafka
+snort.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
+snort.sinks.kafka-sink.brokerList = localhost:9092
+snort.sinks.kafka-sink.topic = snort
+
+# also log events
+snort.sinks.logger-sink.type = logger
+
+# buffer events in memory
+snort.channels.memory-channel.type = memory
+snort.channels.memory-channel.capacity = 1000
+snort.channels.memory-channel.transactionCapacity = 100
+
+# bind the source and sink to the channel
+snort.sources.exec-source.channels = memory-channel
+snort.sinks.kafka-sink.channel = memory-channel
+snort.sinks.logger-sink.channel = memory-channel

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/snort/files/snort.conf
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/files/snort.conf b/deployment/roles/snort/files/snort.conf
new file mode 100644
index 0000000..35757c3
--- /dev/null
+++ b/deployment/roles/snort/files/snort.conf
@@ -0,0 +1,731 @@
+#--------------------------------------------------
+#   VRT Rule Packages Snort.conf
+#
+#   For more information visit us at:
+#     http://www.snort.org                   Snort Website
+#     http://vrt-blog.snort.org/    Sourcefire VRT Blog
+#
+#     Mailing list Contact:      snort-sigs@lists.sourceforge.net
+#     False Positive reports:    fp@sourcefire.com
+#     Snort bugs:                bugs@snort.org
+#
+#     Compatible with Snort Versions:
+#     VERSIONS : 2.9.6.2
+#
+#     Snort build options:
+#     OPTIONS : --enable-gre --enable-mpls --enable-targetbased --enable-ppm --enable-perfprofiling --enable-zlib --enable-active-response --enable-normalizer --enable-reload --enable-react --enable-flexresp3
+#
+#     Additional information:
+#     This configuration file enables active response, to run snort in
+#     test mode -T you are required to supply an interface -i <interface>
+#     or test mode will fail to fully validate the configuration and
+#     exit with a FATAL error
+#--------------------------------------------------
+
+###################################################
+# This file contains a sample snort configuration.
+# You should take the following steps to create your own custom configuration:
+#
+#  1) Set the network variables.
+#  2) Configure the decoder
+#  3) Configure the base detection engine
+#  4) Configure dynamic loaded libraries
+#  5) Configure preprocessors
+#  6) Configure output plugins
+#  7) Customize your rule set
+#  8) Customize preprocessor and decoder rule set
+#  9) Customize shared object rule set
+###################################################
+
+###################################################
+# Step #1: Set the network variables.  For more information, see README.variables
+###################################################
+
+# Setup the network addresses you are protecting
+ipvar HOME_NET 10.0.0.16
+
+# Set up the external network addresses. Leave as "any" in most situations
+ipvar EXTERNAL_NET any
+
+# List of DNS servers on your network
+ipvar DNS_SERVERS $HOME_NET
+
+# List of SMTP servers on your network
+ipvar SMTP_SERVERS $HOME_NET
+
+# List of web servers on your network
+ipvar HTTP_SERVERS $HOME_NET
+
+# List of sql servers on your network
+ipvar SQL_SERVERS $HOME_NET
+
+# List of telnet servers on your network
+ipvar TELNET_SERVERS $HOME_NET
+
+# List of ssh servers on your network
+ipvar SSH_SERVERS $HOME_NET
+
+# List of ftp servers on your network
+ipvar FTP_SERVERS $HOME_NET
+
+# List of sip servers on your network
+ipvar SIP_SERVERS $HOME_NET
+
+# List of ports you run web servers on
+portvar HTTP_PORTS [36,80,81,82,83,84,85,86,87,88,89,90,311,383,555,591,593,631,801,808,818,901,972,1158,1220,1414,1533,1741,1830,1942,2231,2301,2381,2578,2809,2980,3029,3037,3057,3128,3443,3702,4000,4343,4848,5000,5117,5250,5600,5814,6080,6173,6988,7000,7001,7005,7071,7144,7145,7510,7770,7777,7778,7779,8000,8001,8008,8014,8015,8020,8028,8040,8080,8081,8082,8085,8088,8090,8118,8123,8180,8181,8182,8222,8243,8280,8300,8333,8344,8400,8443,8500,8509,8787,8800,8888,8899,8983,9000,9002,9060,9080,9090,9091,9111,9290,9443,9447,9710,9788,9999,10000,11371,12601,13014,15489,19980,29991,33300,34412,34443,34444,40007,41080,44449,50000,50002,51423,53331,55252,55555,56712]
+
+# List of ports you want to look for SHELLCODE on.
+portvar SHELLCODE_PORTS !80
+
+# List of ports you might see oracle attacks on
+portvar ORACLE_PORTS 1024:
+
+# List of ports you want to look for SSH connections on:
+portvar SSH_PORTS 22
+
+# List of ports you run ftp servers on
+portvar FTP_PORTS [21,2100,3535]
+
+# List of ports you run SIP servers on
+portvar SIP_PORTS [5060,5061,5600]
+
+# List of file data ports for file inspection
+portvar FILE_DATA_PORTS [$HTTP_PORTS,110,143]
+
+# List of GTP ports for GTP preprocessor
+portvar GTP_PORTS [2123,2152,3386]
+
+# other variables, these should not be modified
+ipvar AIM_SERVERS [64.12.24.0/23,64.12.28.0/23,64.12.161.0/24,64.12.163.0/24,64.12.200.0/24,205.188.3.0/24,205.188.5.0/24,205.188.7.0/24,205.188.9.0/24,205.188.153.0/24,205.188.179.0/24,205.188.248.0/24]
+
+# Path to your rules files (this can be a relative path)
+# Note for Windows users:  You are advised to make this an absolute path,
+# such as:  c:\snort\rules
+var RULE_PATH rules
+var SO_RULE_PATH so_rules
+var PREPROC_RULE_PATH preproc_rules
+
+# If you are using reputation preprocessor set these
+var WHITE_LIST_PATH /etc/snort/rules
+var BLACK_LIST_PATH /etc/snort/rules
+
+###################################################
+# Step #2: Configure the decoder.  For more information, see README.decode
+###################################################
+
+# Stop generic decode events:
+config disable_decode_alerts
+
+# Stop Alerts on experimental TCP options
+config disable_tcpopt_experimental_alerts
+
+# Stop Alerts on obsolete TCP options
+config disable_tcpopt_obsolete_alerts
+
+# Stop Alerts on T/TCP alerts
+config disable_tcpopt_ttcp_alerts
+
+# Stop Alerts on all other TCPOption type events:
+config disable_tcpopt_alerts
+
+# Stop Alerts on invalid ip options
+config disable_ipopt_alerts
+
+# Alert if value in length field (IP, TCP, UDP) is greater th elength of the packet
+# config enable_decode_oversized_alerts
+
+# Same as above, but drop packet if in Inline mode (requires enable_decode_oversized_alerts)
+# config enable_decode_oversized_drops
+
+# Configure IP / TCP checksum mode
+config checksum_mode: all
+
+# Configure maximum number of flowbit references.  For more information, see README.flowbits
+# config flowbits_size: 64
+
+# Configure ports to ignore
+# config ignore_ports: tcp 21 6667:6671 1356
+# config ignore_ports: udp 1:17 53
+
+# Configure active response for non inline operation. For more information, see REAMDE.active
+# config response: eth0 attempts 2
+
+# Configure DAQ related options for inline operation. For more information, see README.daq
+#
+# config daq: <type>
+# config daq_dir: <dir>
+# config daq_mode: <mode>
+# config daq_var: <var>
+#
+# <type> ::= pcap | afpacket | dump | nfq | ipq | ipfw
+# <mode> ::= read-file | passive | inline
+# <var> ::= arbitrary <name>=<value passed to DAQ
+# <dir> ::= path as to where to look for DAQ module so's
+
+# Configure specific UID and GID to run snort as after dropping privs. For more information see snort -h command line options
+#
+# config set_gid:
+# config set_uid:
+
+# Configure default snaplen. Snort defaults to MTU of in use interface. For more information see README
+#
+# config snaplen:
+#
+
+# Configure default bpf_file to use for filtering what traffic reaches snort. For more information see snort -h command line options (-F)
+#
+# config bpf_file:
+#
+
+# Configure default log directory for snort to log to.  For more information see snort -h command line options (-l)
+#
+# config logdir:
+
+
+###################################################
+# Step #3: Configure the base detection engine.  For more information, see  README.decode
+###################################################
+
+# Configure PCRE match limitations
+config pcre_match_limit: 3500
+config pcre_match_limit_recursion: 1500
+
+# Configure the detection engine  See the Snort Manual, Configuring Snort - Includes - Config
+config detection: search-method ac-split search-optimize max-pattern-len 20
+
+# Configure the event queue.  For more information, see README.event_queue
+config event_queue: max_queue 8 log 5 order_events content_length
+
+###################################################
+## Configure GTP if it is to be used.
+## For more information, see README.GTP
+####################################################
+
+# config enable_gtp
+
+###################################################
+# Per packet and rule latency enforcement
+# For more information see README.ppm
+###################################################
+
+# Per Packet latency configuration
+#config ppm: max-pkt-time 250, \
+#   fastpath-expensive-packets, \
+#   pkt-log
+
+# Per Rule latency configuration
+#config ppm: max-rule-time 200, \
+#   threshold 3, \
+#   suspend-expensive-rules, \
+#   suspend-timeout 20, \
+#   rule-log alert
+
+###################################################
+# Configure Perf Profiling for debugging
+# For more information see README.PerfProfiling
+###################################################
+
+#config profile_rules: print all, sort avg_ticks
+#config profile_preprocs: print all, sort avg_ticks
+
+###################################################
+# Configure protocol aware flushing
+# For more information see README.stream5
+###################################################
+config paf_max: 16000
+
+###################################################
+# Step #4: Configure dynamic loaded libraries.
+# For more information, see Snort Manual, Configuring Snort - Dynamic Modules
+###################################################
+
+# path to dynamic preprocessor libraries
+dynamicpreprocessor directory /usr/lib64/snort-2.9.8.0_dynamicpreprocessor
+
+# path to base preprocessor engine
+dynamicengine /usr/lib64/snort-2.9.8.0_dynamicengine/libsf_engine.so
+
+# path to dynamic rules libraries
+#dynamicdetection directory /usr/local/lib/snort_dynamicrules
+
+###################################################
+# Step #5: Configure preprocessors
+# For more information, see the Snort Manual, Configuring Snort - Preprocessors
+###################################################
+
+# GTP Control Channle Preprocessor. For more information, see README.GTP
+# preprocessor gtp: ports { 2123 3386 2152 }
+
+# Inline packet normalization. For more information, see README.normalize
+# Does nothing in IDS mode
+preprocessor normalize_ip4
+preprocessor normalize_tcp: ips ecn stream
+preprocessor normalize_icmp4
+preprocessor normalize_ip6
+preprocessor normalize_icmp6
+
+# Target-based IP defragmentation.  For more inforation, see README.frag3
+preprocessor frag3_global: max_frags 65536
+preprocessor frag3_engine: policy windows detect_anomalies overlap_limit 10 min_fragment_length 100 timeout 180
+
+# Target-Based stateful inspection/stream reassembly.  For more inforation, see README.stream5
+preprocessor stream5_global: track_tcp yes, \
+   track_udp yes, \
+   track_icmp no, \
+   max_tcp 262144, \
+   max_udp 131072, \
+   max_active_responses 2, \
+   min_response_seconds 5
+preprocessor stream5_tcp: policy windows, detect_anomalies, require_3whs 180, \
+   overlap_limit 10, small_segments 3 bytes 150, timeout 180, \
+    ports client 21 22 23 25 42 53 70 79 109 110 111 113 119 135 136 137 139 143 \
+        161 445 513 514 587 593 691 1433 1521 1741 2100 3306 6070 6665 6666 6667 6668 6669 \
+        7000 8181 32770 32771 32772 32773 32774 32775 32776 32777 32778 32779, \
+    ports both 36 80 81 82 83 84 85 86 87 88 89 90 110 311 383 443 465 563 555 591 593 631 636 801 808 818 901 972 989 992 993 994 995 1158 1220 1414 1533 1741 1830 1942 2231 2301 2381 2578 2809 2980 3029 3037 3057 3128 3443 3702 4000 4343 4848 5000 5117 5250 5600 5814 6080 6173 6988 7907 7000 7001 7005 7071 7144 7145 7510 7802 7770 7777 7778 7779 \
+        7801 7900 7901 7902 7903 7904 7905 7906 7908 7909 7910 7911 7912 7913 7914 7915 7916 \
+        7917 7918 7919 7920 8000 8001 8008 8014 8015 8020 8028 8040 8080 8081 8082 8085 8088 8090 8118 8123 8180 8181 8182 8222 8243 8280 8300 8333 8344 8400 8443 8500 8509 8787 8800 8888 8899 8983 9000 9002 9060 9080 9090 9091 9111 9290 9443 9447 9710 9788 9999 10000 11371 12601 13014 15489 19980 29991 33300 34412 34443 34444 40007 41080 44449 50000 50002 51423 53331 55252 55555 56712
+preprocessor stream5_udp: timeout 180
+
+# performance statistics.  For more information, see the Snort Manual, Configuring Snort - Preprocessors - Performance Monitor
+# preprocessor perfmonitor: time 300 file /var/snort/snort.stats pktcnt 10000
+
+# HTTP normalization and anomaly detection.  For more information, see README.http_inspect
+preprocessor http_inspect: global iis_unicode_map unicode.map 1252 compress_depth 65535 decompress_depth 65535
+preprocessor http_inspect_server: server default \
+    http_methods { GET POST PUT SEARCH MKCOL COPY MOVE LOCK UNLOCK NOTIFY POLL BCOPY BDELETE BMOVE LINK UNLINK OPTIONS HEAD DELETE TRACE TRACK CONNECT SOURCE SUBSCRIBE UNSUBSCRIBE PROPFIND PROPPATCH BPROPFIND BPROPPATCH RPC_CONNECT PROXY_SUCCESS BITS_POST CCM_POST SMS_POST RPC_IN_DATA RPC_OUT_DATA RPC_ECHO_DATA } \
+    chunk_length 500000 \
+    server_flow_depth 0 \
+    client_flow_depth 0 \
+    post_depth 65495 \
+    oversize_dir_length 500 \
+    max_header_length 750 \
+    max_headers 100 \
+    max_spaces 200 \
+    small_chunk_length { 10 5 } \
+    ports { 36 80 81 82 83 84 85 86 87 88 89 90 311 383 555 591 593 631 801 808 818 901 972 1158 1220 1414 1533 1741 1830 1942 2231 2301 2381 2578 2809 2980 3029 3037 3057 3128 3443 3702 4000 4343 4848 5000 5117 5250 5600 5814 6080 6173 6988 7000 7001 7005 7071 7144 7145 7510 7770 7777 7778 7779 8000 8001 8008 8014 8015 8020 8028 8040 8080 8081 8082 8085 8088 8090 8118 8123 8180 8181 8182 8222 8243 8280 8300 8333 8344 8400 8443 8500 8509 8787 8800 8888 8899 8983 9000 9002 9060 9080 9090 9091 9111 9290 9443 9447 9710 9788 9999 10000 11371 12601 13014 15489 19980 29991 33300 34412 34443 34444 40007 41080 44449 50000 50002 51423 53331 55252 55555 56712 } \
+    non_rfc_char { 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 } \
+    enable_cookie \
+    extended_response_inspection \
+    inspect_gzip \
+    normalize_utf \
+    unlimited_decompress \
+    normalize_javascript \
+    apache_whitespace no \
+    ascii no \
+    bare_byte no \
+    directory no \
+    double_decode no \
+    iis_backslash no \
+    iis_delimiter no \
+    iis_unicode no \
+    multi_slash no \
+    utf_8 no \
+    u_encode yes \
+    webroot no
+
+# ONC-RPC normalization and anomaly detection.  For more information, see the Snort Manual, Configuring Snort - Preprocessors - RPC Decode
+preprocessor rpc_decode: 111 32770 32771 32772 32773 32774 32775 32776 32777 32778 32779 no_alert_multiple_requests no_alert_large_fragments no_alert_incomplete
+
+# Back Orifice detection.
+preprocessor bo
+
+# FTP / Telnet normalization and anomaly detection.  For more information, see README.ftptelnet
+preprocessor ftp_telnet: global inspection_type stateful encrypted_traffic no check_encrypted
+preprocessor ftp_telnet_protocol: telnet \
+    ayt_attack_thresh 20 \
+    normalize ports { 23 } \
+    detect_anomalies
+preprocessor ftp_telnet_protocol: ftp server default \
+    def_max_param_len 100 \
+    ports { 21 2100 3535 } \
+    telnet_cmds yes \
+    ignore_telnet_erase_cmds yes \
+    ftp_cmds { ABOR ACCT ADAT ALLO APPE AUTH CCC CDUP } \
+    ftp_cmds { CEL CLNT CMD CONF CWD DELE ENC EPRT } \
+    ftp_cmds { EPSV ESTA ESTP FEAT HELP LANG LIST LPRT } \
+    ftp_cmds { LPSV MACB MAIL MDTM MIC MKD MLSD MLST } \
+    ftp_cmds { MODE NLST NOOP OPTS PASS PASV PBSZ PORT } \
+    ftp_cmds { PROT PWD QUIT REIN REST RETR RMD RNFR } \
+    ftp_cmds { RNTO SDUP SITE SIZE SMNT STAT STOR STOU } \
+    ftp_cmds { STRU SYST TEST TYPE USER XCUP XCRC XCWD } \
+    ftp_cmds { XMAS XMD5 XMKD XPWD XRCP XRMD XRSQ XSEM } \
+    ftp_cmds { XSEN XSHA1 XSHA256 } \
+    alt_max_param_len 0 { ABOR CCC CDUP ESTA FEAT LPSV NOOP PASV PWD QUIT REIN STOU SYST XCUP XPWD } \
+    alt_max_param_len 200 { ALLO APPE CMD HELP NLST RETR RNFR STOR STOU XMKD } \
+    alt_max_param_len 256 { CWD RNTO } \
+    alt_max_param_len 400 { PORT } \
+    alt_max_param_len 512 { SIZE } \
+    chk_str_fmt { ACCT ADAT ALLO APPE AUTH CEL CLNT CMD } \
+    chk_str_fmt { CONF CWD DELE ENC EPRT EPSV ESTP HELP } \
+    chk_str_fmt { LANG LIST LPRT MACB MAIL MDTM MIC MKD } \
+    chk_str_fmt { MLSD MLST MODE NLST OPTS PASS PBSZ PORT } \
+    chk_str_fmt { PROT REST RETR RMD RNFR RNTO SDUP SITE } \
+    chk_str_fmt { SIZE SMNT STAT STOR STRU TEST TYPE USER } \
+    chk_str_fmt { XCRC XCWD XMAS XMD5 XMKD XRCP XRMD XRSQ } \
+    chk_str_fmt { XSEM XSEN XSHA1 XSHA256 } \
+    cmd_validity ALLO < int [ char R int ] > \
+    cmd_validity EPSV < [ { char 12 | char A char L char L } ] > \
+    cmd_validity MACB < string > \
+    cmd_validity MDTM < [ date nnnnnnnnnnnnnn[.n[n[n]]] ] string > \
+    cmd_validity MODE < char ASBCZ > \
+    cmd_validity PORT < host_port > \
+    cmd_validity PROT < char CSEP > \
+    cmd_validity STRU < char FRPO [ string ] > \
+    cmd_validity TYPE < { char AE [ char NTC ] | char I | char L [ number ] } >
+preprocessor ftp_telnet_protocol: ftp client default \
+    max_resp_len 256 \
+    bounce yes \
+    ignore_telnet_erase_cmds yes \
+    telnet_cmds yes
+
+
+# SMTP normalization and anomaly detection.  For more information, see README.SMTP
+preprocessor smtp: ports { 25 465 587 691 } \
+    inspection_type stateful \
+    b64_decode_depth 0 \
+    qp_decode_depth 0 \
+    bitenc_decode_depth 0 \
+    uu_decode_depth 0 \
+    log_mailfrom \
+    log_rcptto \
+    log_filename \
+    log_email_hdrs \
+    normalize cmds \
+    normalize_cmds { ATRN AUTH BDAT CHUNKING DATA DEBUG EHLO EMAL ESAM ESND ESOM ETRN EVFY } \
+    normalize_cmds { EXPN HELO HELP IDENT MAIL NOOP ONEX QUEU QUIT RCPT RSET SAML SEND SOML } \
+    normalize_cmds { STARTTLS TICK TIME TURN TURNME VERB VRFY X-ADAT X-DRCP X-ERCP X-EXCH50 } \
+    normalize_cmds { X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
+    max_command_line_len 512 \
+    max_header_line_len 1000 \
+    max_response_line_len 512 \
+    alt_max_command_line_len 260 { MAIL } \
+    alt_max_command_line_len 300 { RCPT } \
+    alt_max_command_line_len 500 { HELP HELO ETRN EHLO } \
+    alt_max_command_line_len 255 { EXPN VRFY ATRN SIZE BDAT DEBUG EMAL ESAM ESND ESOM EVFY IDENT NOOP RSET } \
+    alt_max_command_line_len 246 { SEND SAML SOML AUTH TURN ETRN DATA RSET QUIT ONEX QUEU STARTTLS TICK TIME TURNME VERB X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
+    valid_cmds { ATRN AUTH BDAT CHUNKING DATA DEBUG EHLO EMAL ESAM ESND ESOM ETRN EVFY } \
+    valid_cmds { EXPN HELO HELP IDENT MAIL NOOP ONEX QUEU QUIT RCPT RSET SAML SEND SOML } \
+    valid_cmds { STARTTLS TICK TIME TURN TURNME VERB VRFY X-ADAT X-DRCP X-ERCP X-EXCH50 } \
+    valid_cmds { X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
+    xlink2state { enabled }
+
+# Portscan detection.  For more information, see README.sfportscan
+preprocessor sfportscan: proto  { all } memcap { 10000000 } sense_level { low }
+
+# ARP spoof detection.  For more information, see the Snort Manual - Configuring Snort - Preprocessors - ARP Spoof Preprocessor
+# preprocessor arpspoof
+# preprocessor arpspoof_detect_host: 192.168.40.1 f0:0f:00:f0:0f:00
+
+# SSH anomaly detection.  For more information, see README.ssh
+preprocessor ssh: server_ports { 22 } \
+                  autodetect \
+                  max_client_bytes 19600 \
+                  max_encrypted_packets 20 \
+                  max_server_version_len 100 \
+                  enable_respoverflow enable_ssh1crc32 \
+                  enable_srvoverflow enable_protomismatch
+
+# SMB / DCE-RPC normalization and anomaly detection.  For more information, see README.dcerpc2
+preprocessor dcerpc2: memcap 102400, events [co ]
+preprocessor dcerpc2_server: default, policy WinXP, \
+    detect [smb [139,445], tcp 135, udp 135, rpc-over-http-server 593], \
+    autodetect [tcp 1025:, udp 1025:, rpc-over-http-server 1025:], \
+    smb_max_chain 3, smb_invalid_shares ["C$", "D$", "ADMIN$"]
+
+# DNS anomaly detection.  For more information, see README.dns
+preprocessor dns: ports { 53 } enable_rdata_overflow
+
+# SSL anomaly detection and traffic bypass.  For more information, see README.ssl
+preprocessor ssl: ports { 443 465 563 636 989 992 993 994 995 5061 7801 7802 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 }, trustservers, noinspect_encrypted
+
+# SDF sensitive data preprocessor.  For more information see README.sensitive_data
+preprocessor sensitive_data: alert_threshold 25
+
+# SIP Session Initiation Protocol preprocessor.  For more information see README.sip
+preprocessor sip: max_sessions 40000, \
+   ports { 5060 5061 5600 }, \
+   methods { invite \
+             cancel \
+             ack \
+             bye \
+             register \
+             options \
+             refer \
+             subscribe \
+             update \
+             join \
+             info \
+             message \
+             notify \
+             benotify \
+             do \
+             qauth \
+             sprack \
+             publish \
+             service \
+             unsubscribe \
+             prack }, \
+   max_uri_len 512, \
+   max_call_id_len 80, \
+   max_requestName_len 20, \
+   max_from_len 256, \
+   max_to_len 256, \
+   max_via_len 1024, \
+   max_contact_len 512, \
+   max_content_len 2048
+
+# IMAP preprocessor.  For more information see README.imap
+preprocessor imap: \
+   ports { 143 } \
+   b64_decode_depth 0 \
+   qp_decode_depth 0 \
+   bitenc_decode_depth 0 \
+   uu_decode_depth 0
+
+# POP preprocessor. For more information see README.pop
+preprocessor pop: \
+   ports { 110 } \
+   b64_decode_depth 0 \
+   qp_decode_depth 0 \
+   bitenc_decode_depth 0 \
+   uu_decode_depth 0
+
+# Modbus preprocessor. For more information see README.modbus
+preprocessor modbus: ports { 502 }
+
+# DNP3 preprocessor. For more information see README.dnp3
+preprocessor dnp3: ports { 20000 } \
+   memcap 262144 \
+   check_crc
+
+# Reputation preprocessor. For more information see README.reputation
+preprocessor reputation: \
+   memcap 500, \
+   priority whitelist, \
+   nested_ip inner, \
+   whitelist $WHITE_LIST_PATH/white_list.rules, \
+   blacklist $BLACK_LIST_PATH/black_list.rules
+
+###################################################
+# Step #6: Configure output plugins
+# For more information, see Snort Manual, Configuring Snort - Output Modules
+###################################################
+
+# unified2
+# Recommended for most installs
+# output unified2: filename merged.log, limit 128, nostamp, mpls_event_types, vlan_event_types
+
+# Additional configuration for specific types of installs
+# output alert_unified2: filename snort.alert, limit 128, nostamp
+# output log_unified2: filename snort.log, limit 128, nostamp
+
+# syslog
+# output alert_syslog: LOG_AUTH LOG_ALERT
+
+# pcap
+# output log_tcpdump: tcpdump.log
+
+# metadata reference data.  do not modify these lines
+include classification.config
+include reference.config
+
+
+###################################################
+# Step #7: Customize your rule set
+# For more information, see Snort Manual, Writing Snort Rules
+#
+# NOTE: All categories are enabled in this conf file
+###################################################
+
+# site specific rules
+#include $RULE_PATH/local.rules
+
+include $RULE_PATH/app-detect.rules
+include $RULE_PATH/attack-responses.rules
+include $RULE_PATH/backdoor.rules
+include $RULE_PATH/bad-traffic.rules
+include $RULE_PATH/blacklist.rules
+include $RULE_PATH/botnet-cnc.rules
+include $RULE_PATH/browser-chrome.rules
+include $RULE_PATH/browser-firefox.rules
+include $RULE_PATH/browser-ie.rules
+include $RULE_PATH/browser-other.rules
+include $RULE_PATH/browser-plugins.rules
+include $RULE_PATH/browser-webkit.rules
+include $RULE_PATH/chat.rules
+include $RULE_PATH/content-replace.rules
+include $RULE_PATH/ddos.rules
+include $RULE_PATH/dns.rules
+include $RULE_PATH/dos.rules
+include $RULE_PATH/experimental.rules
+include $RULE_PATH/exploit-kit.rules
+include $RULE_PATH/exploit.rules
+include $RULE_PATH/file-executable.rules
+include $RULE_PATH/file-flash.rules
+include $RULE_PATH/file-identify.rules
+include $RULE_PATH/file-image.rules
+include $RULE_PATH/file-java.rules
+include $RULE_PATH/file-multimedia.rules
+include $RULE_PATH/file-office.rules
+include $RULE_PATH/file-other.rules
+include $RULE_PATH/file-pdf.rules
+include $RULE_PATH/finger.rules
+include $RULE_PATH/ftp.rules
+include $RULE_PATH/icmp-info.rules
+include $RULE_PATH/icmp.rules
+include $RULE_PATH/imap.rules
+include $RULE_PATH/indicator-compromise.rules
+include $RULE_PATH/indicator-obfuscation.rules
+include $RULE_PATH/indicator-scan.rules
+include $RULE_PATH/indicator-shellcode.rules
+include $RULE_PATH/info.rules
+include $RULE_PATH/malware-backdoor.rules
+include $RULE_PATH/malware-cnc.rules
+include $RULE_PATH/malware-other.rules
+include $RULE_PATH/malware-tools.rules
+include $RULE_PATH/misc.rules
+include $RULE_PATH/multimedia.rules
+include $RULE_PATH/mysql.rules
+include $RULE_PATH/netbios.rules
+include $RULE_PATH/nntp.rules
+include $RULE_PATH/oracle.rules
+include $RULE_PATH/os-linux.rules
+include $RULE_PATH/os-mobile.rules
+include $RULE_PATH/os-other.rules
+include $RULE_PATH/os-solaris.rules
+include $RULE_PATH/os-windows.rules
+include $RULE_PATH/other-ids.rules
+include $RULE_PATH/p2p.rules
+include $RULE_PATH/phishing-spam.rules
+include $RULE_PATH/policy-multimedia.rules
+include $RULE_PATH/policy-other.rules
+include $RULE_PATH/policy.rules
+include $RULE_PATH/policy-social.rules
+include $RULE_PATH/policy-spam.rules
+include $RULE_PATH/pop2.rules
+include $RULE_PATH/pop3.rules
+include $RULE_PATH/protocol-dns.rules
+include $RULE_PATH/protocol-finger.rules
+include $RULE_PATH/protocol-ftp.rules
+include $RULE_PATH/protocol-icmp.rules
+include $RULE_PATH/protocol-imap.rules
+include $RULE_PATH/protocol-nntp.rules
+include $RULE_PATH/protocol-other.rules
+include $RULE_PATH/protocol-pop.rules
+include $RULE_PATH/protocol-rpc.rules
+include $RULE_PATH/protocol-scada.rules
+include $RULE_PATH/protocol-services.rules
+include $RULE_PATH/protocol-snmp.rules
+include $RULE_PATH/protocol-telnet.rules
+include $RULE_PATH/protocol-tftp.rules
+include $RULE_PATH/protocol-voip.rules
+include $RULE_PATH/pua-adware.rules
+include $RULE_PATH/pua-other.rules
+include $RULE_PATH/pua-p2p.rules
+include $RULE_PATH/pua-toolbars.rules
+include $RULE_PATH/rpc.rules
+include $RULE_PATH/rservices.rules
+include $RULE_PATH/scada.rules
+include $RULE_PATH/scan.rules
+include $RULE_PATH/server-apache.rules
+include $RULE_PATH/server-iis.rules
+include $RULE_PATH/server-mail.rules
+include $RULE_PATH/server-mssql.rules
+include $RULE_PATH/server-mysql.rules
+include $RULE_PATH/server-oracle.rules
+include $RULE_PATH/server-other.rules
+include $RULE_PATH/server-samba.rules
+include $RULE_PATH/server-webapp.rules
+include $RULE_PATH/shellcode.rules
+include $RULE_PATH/smtp.rules
+include $RULE_PATH/snmp.rules
+include $RULE_PATH/specific-threats.rules
+include $RULE_PATH/spyware-put.rules
+include $RULE_PATH/sql.rules
+include $RULE_PATH/telnet.rules
+include $RULE_PATH/tftp.rules
+include $RULE_PATH/virus.rules
+include $RULE_PATH/voip.rules
+include $RULE_PATH/web-activex.rules
+include $RULE_PATH/web-attacks.rules
+include $RULE_PATH/web-cgi.rules
+include $RULE_PATH/web-client.rules
+include $RULE_PATH/web-coldfusion.rules
+include $RULE_PATH/web-frontpage.rules
+include $RULE_PATH/web-iis.rules
+include $RULE_PATH/web-misc.rules
+include $RULE_PATH/web-php.rules
+include $RULE_PATH/x11.rules
+
+###################################################
+# Step #8: Customize your preprocessor and decoder alerts
+# For more information, see README.decoder_preproc_rules
+###################################################
+
+# decoder and preprocessor event rules
+include $PREPROC_RULE_PATH/preprocessor.rules
+include $PREPROC_RULE_PATH/decoder.rules
+include $PREPROC_RULE_PATH/sensitive-data.rules
+
+###################################################
+# Step #9: Customize your Shared Object Snort Rules
+# For more information, see http://vrt-blog.snort.org/2009/01/using-vrt-certified-shared-object-rules.html
+###################################################
+
+# dynamic library rules
+include $SO_RULE_PATH/browser-ie.rules
+include $SO_RULE_PATH/browser-other.rules
+include $SO_RULE_PATH/exploit-kit.rules
+include $SO_RULE_PATH/file-flash.rules
+include $SO_RULE_PATH/file-image.rules
+include $SO_RULE_PATH/file-java.rules
+include $SO_RULE_PATH/file-multimedia.rules
+include $SO_RULE_PATH/file-office.rules
+include $SO_RULE_PATH/file-other.rules
+include $SO_RULE_PATH/file-pdf.rules
+include $SO_RULE_PATH/indicator-shellcode.rules
+include $SO_RULE_PATH/malware-cnc.rules
+include $SO_RULE_PATH/malware-other.rules
+include $SO_RULE_PATH/netbios.rules
+include $SO_RULE_PATH/os-linux.rules
+include $SO_RULE_PATH/os-other.rules
+include $SO_RULE_PATH/os-windows.rules
+include $SO_RULE_PATH/policy-social.rules
+include $SO_RULE_PATH/protocol-dns.rules
+include $SO_RULE_PATH/protocol-nntp.rules
+include $SO_RULE_PATH/protocol-other.rules
+include $SO_RULE_PATH/protocol-snmp.rules
+include $SO_RULE_PATH/protocol-voip.rules
+include $SO_RULE_PATH/pua-p2p.rules
+include $SO_RULE_PATH/server-apache.rules
+include $SO_RULE_PATH/server-iis.rules
+include $SO_RULE_PATH/server-mail.rules
+include $SO_RULE_PATH/server-mysql.rules
+include $SO_RULE_PATH/server-oracle.rules
+include $SO_RULE_PATH/server-other.rules
+include $SO_RULE_PATH/server-webapp.rules
+
+# legacy dynamic library rule files
+# include $SO_RULE_PATH/bad-traffic.rules
+# include $SO_RULE_PATH/browser-ie.rules
+# include $SO_RULE_PATH/chat.rules
+# include $SO_RULE_PATH/dos.rules
+# include $SO_RULE_PATH/exploit.rules
+# include $SO_RULE_PATH/file-flash.rules
+# include $SO_RULE_PATH/icmp.rules
+# include $SO_RULE_PATH/imap.rules
+# include $SO_RULE_PATH/misc.rules
+# include $SO_RULE_PATH/multimedia.rules
+# include $SO_RULE_PATH/netbios.rules
+# include $SO_RULE_PATH/nntp.rules
+# include $SO_RULE_PATH/p2p.rules
+# include $SO_RULE_PATH/smtp.rules
+# include $SO_RULE_PATH/snmp.rules
+# include $SO_RULE_PATH/specific-threats.rules
+# include $SO_RULE_PATH/web-activex.rules
+# include $SO_RULE_PATH/web-client.rules
+# include $SO_RULE_PATH/web-iis.rules
+# include $SO_RULE_PATH/web-misc.rules
+
+# Event thresholding or suppression commands. See threshold.conf
+include threshold.conf

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/snort/files/snortrules-snapshot-2962.tar.gz
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/files/snortrules-snapshot-2962.tar.gz b/deployment/roles/snort/files/snortrules-snapshot-2962.tar.gz
new file mode 100644
index 0000000..6b0af89
Binary files /dev/null and b/deployment/roles/snort/files/snortrules-snapshot-2962.tar.gz differ

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/snort/tasks/flume.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/flume.yml b/deployment/roles/snort/tasks/flume.yml
new file mode 100644
index 0000000..df93b52
--- /dev/null
+++ b/deployment/roles/snort/tasks/flume.yml
@@ -0,0 +1,27 @@
+---
+- name: Install flume configurations
+  copy: src={{ item.src }} dest={{ item.dest }}
+  with_items:
+    - { src: flume-snort.conf, dest: /etc/flume/conf/flume-snort.conf }
+
+- name: Set kafka broker in flume
+  lineinfile: >
+    dest="{{ item.dest }}"
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { dest: /etc/flume/conf/flume-snort.conf,
+        regexp: '^snort\.sinks\.kafka-sink\.brokerList.*$',
+        line: 'snort.sinks.kafka-sink.brokerList = {{ kafka_broker_url }}' }
+
+- name: Set kafka topics in flume
+  lineinfile: >
+    dest="{{ item.dest }}"
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { dest: /etc/flume/conf/flume-snort.conf,
+        regexp: '^snort\.sinks\.kafka-sink\.topic.*$',
+        line: 'snort.sinks.kafka-sink.topic = {{ snort_topic }}'}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/snort/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/main.yml b/deployment/roles/snort/tasks/main.yml
new file mode 100644
index 0000000..651a8a8
--- /dev/null
+++ b/deployment/roles/snort/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+- include: snort.yml
+
+- include: flume.yml
+
+- name: Start snort
+  service: name=snortd state=restarted
+
+- name: Start flume service to consume snort alerts
+  service: name=flume-agent state=restarted args=snort

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/snort/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/snort.yml b/deployment/roles/snort/tasks/snort.yml
new file mode 100644
index 0000000..2a32a66
--- /dev/null
+++ b/deployment/roles/snort/tasks/snort.yml
@@ -0,0 +1,63 @@
+---
+- name: Install development tools
+  yum: name="@Development tools" state=present
+
+- name: Install snort prerequisites
+  yum: name={{ item }} state=latest update_cache=yes
+  with_items:
+    - libdnet-devel
+    - rpm-build
+    - libpcap
+    - libpcap-devel
+    - pcre
+    - pcre-devel
+
+- name: Download daq
+  get_url: url="https://snort.org/downloads/snort/daq-{{ daq_version }}.src.rpm" dest=/tmp
+
+- name: Build daq
+  shell: rpmbuild --rebuild "daq-{{ daq_version }}.src.rpm" chdir=/tmp
+
+- name: Install daq
+  yum: name=/root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm state=present
+
+- name: Download snort
+  get_url: url="https://snort.org/downloads/snort/snort-{{ snort_version }}.src.rpm" dest=/tmp
+
+- name: Build snort
+  shell: rpmbuild --rebuild "snort-{{ snort_version }}.src.rpm" chdir=/tmp
+
+- name: Install snort
+  yum: name=/root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm state=present
+
+#
+# TODO download latest rules from snort website - requires credentials
+#
+- name: Download snort rules
+  unarchive: src=snortrules-snapshot-2962.tar.gz dest=/tmp
+
+- name: Install snort rules
+  shell: "{{ item }}"
+  args:
+    chdir: /tmp
+  with_items:
+    - cp -r etc/* /etc/snort/
+    - cp -r rules /etc/snort/
+    - cp -r so_rules /etc/snort/
+    - cp -r preproc_rules /etc/snort/
+    - touch /etc/snort/rules/white_list.rules
+    - touch /etc/snort/rules/black_list.rules
+    - touch /var/log/snort/alerts
+    - chown -R snort:snort /etc/snort
+
+- name: Download snort configuration
+  copy: src=snort.conf dest=/etc/snort/snort.conf
+
+- name: Configure snort
+  lineinfile: >
+    dest=/etc/snort/snort.conf
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { regexp: "^ipvar HOME_NET.*$", line: "ipvar HOME_NET {{ ansible_eth0.ipv4.address }}" }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/snort/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/vars/main.yml b/deployment/roles/snort/vars/main.yml
new file mode 100644
index 0000000..784430e
--- /dev/null
+++ b/deployment/roles/snort/vars/main.yml
@@ -0,0 +1,3 @@
+---
+snort_version: 2.9.8.0-1
+daq_version: 2.0.6-1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/yaf/files/flume-ipfix.conf
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/files/flume-ipfix.conf b/deployment/roles/yaf/files/flume-ipfix.conf
new file mode 100644
index 0000000..fa97ca7
--- /dev/null
+++ b/deployment/roles/yaf/files/flume-ipfix.conf
@@ -0,0 +1,27 @@
+
+ipfix.sources = netcat-source
+ipfix.channels = memory-channel
+ipfix.sinks = kafka-sink logger-sink
+
+# ipfix publishes ipfix to local tcp port 4739
+ipfix.sources.netcat-source.type = netcat
+ipfix.sources.netcat-source.bind = localhost
+ipfix.sources.netcat-source.port = 4739
+
+# data is sent to kafka
+ipfix.sinks.kafka-sink.type = org.apache.flume.sink.kafka.KafkaSink
+ipfix.sinks.kafka-sink.brokerList = [[value overwritten by ansible]]
+ipfix.sinks.kafka-sink.topic = [[value overwritten by ansible]]
+
+# also log events
+ipfix.sinks.logger-sink.type = logger
+
+# buffer events in memory
+ipfix.channels.memory-channel.type = memory
+ipfix.channels.memory-channel.capacity = 1000
+ipfix.channels.memory-channel.transactionCapacity = 100
+
+# bind the source and sink to the channel
+ipfix.sources.netcat-source.channels = memory-channel
+ipfix.sinks.kafka-sink.channel = memory-channel
+ipfix.sinks.logger-sink.channel = memory-channel

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/yaf/tasks/flume.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/flume.yml b/deployment/roles/yaf/tasks/flume.yml
new file mode 100644
index 0000000..8c8db29
--- /dev/null
+++ b/deployment/roles/yaf/tasks/flume.yml
@@ -0,0 +1,27 @@
+---
+- name: Install flume configurations
+  copy: src={{ item.src }} dest={{ item.dest }}
+  with_items:
+    - { src: flume-ipfix.conf, dest: /etc/flume/conf/flume-ipfix.conf }
+
+- name: Set kafka broker in flume
+  lineinfile: >
+    dest="{{ item.dest }}"
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { dest: /etc/flume/conf/flume-ipfix.conf,
+        regexp: '^ipfix\.sinks\.kafka-sink\.brokerList.*$',
+        line: 'ipfix.sinks.kafka-sink.brokerList = {{ kafka_broker_url }}' }
+
+- name: Set kafka topics in flume
+  lineinfile: >
+    dest="{{ item.dest }}"
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { dest: /etc/flume/conf/flume-ipfix.conf,
+        regexp: '^ipfix\.sinks\.kafka-sink\.topic.*$',
+        line: 'ipfix.sinks.kafka-sink.topic = {{ yaf_topic }}'}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml
new file mode 100644
index 0000000..64eb08c
--- /dev/null
+++ b/deployment/roles/yaf/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+- include: yaf.yml
+
+- include: flume.yml
+
+- name: Start flume service to consume ipfix
+  service: name=flume-agent state=restarted args=ipfix
+
+- name: Start yaf service to produce ipfix
+  service: name=yaf state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/yaf/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/yaf.yml b/deployment/roles/yaf/tasks/yaf.yml
new file mode 100644
index 0000000..b7939de
--- /dev/null
+++ b/deployment/roles/yaf/tasks/yaf.yml
@@ -0,0 +1,77 @@
+---
+- name: Install development tools
+  yum: name="@Development tools" state=present
+
+- name: Install prerequisites
+  yum: name={{item}} state=latest update_cache=yes
+  with_items:
+    - libpcap
+    - libpcap-devel
+    - pcre
+    - pcre-devel
+    - glib2-devel
+    - zlib
+    - zlib-devel
+
+- name: Download fixbuf
+  get_url:
+    url: "http://tools.netsa.cert.org/releases/libfixbuf-{{fixbuf_version}}.tar.gz"
+    dest: /tmp
+
+- name: Extract fixbuf tarball
+  unarchive:
+    src: "/tmp/libfixbuf-{{fixbuf_version}}.tar.gz"
+    dest: /tmp
+    copy: no
+    creates: "/tmp/libfixbuf-{{fixbuf_version}}"
+
+- name: Compile and Install fixbuf
+  shell: "{{item}}"
+  args:
+    chdir: "/tmp/libfixbuf-{{fixbuf_version}}"
+  with_items:
+    - ./configure
+    - make
+    - make install
+
+- name: Download yaf
+  get_url:
+    url: "http://tools.netsa.cert.org/releases/yaf-{{yaf_version}}.tar.gz"
+    dest: /tmp
+
+- name: Extract yaf tarball
+  unarchive:
+    src: "/tmp/yaf-{{yaf_version}}.tar.gz"
+    dest: /tmp
+    copy: no
+    creates: /usr/local/bin/yaf
+
+- name: Compile and install yaf
+  shell: "{{item}}"
+  args:
+    chdir: "/tmp/yaf-{{yaf_version}}"
+    creates: /usr/local/bin/yaf
+  with_items:
+    - ./configure --enable-applabel --enable-plugins
+    - make
+    - make install
+
+- name: Create yaf service
+  shell: "{{item}}"
+  args:
+    creates: /usr/local/etc/yaf.conf
+  with_items:
+    - cp /tmp/yaf-{{yaf_version}}/etc/init.d/yaf /etc/init.d/yaf
+    - chmod +x /etc/init.d/yaf
+    - cp /tmp/yaf-{{yaf_version}}/etc/yaf.conf /usr/local/etc/yaf.conf
+
+- name: Configure yaf
+  lineinfile: >
+    dest=/usr/local/etc/yaf.conf
+    regexp="{{ item.regexp }}"
+    line="{{ item.line }}"
+    state=present
+  with_items:
+    - { regexp: '^ENABLED=.*$', line: 'ENABLED=1' }
+    - { regexp: '^YAF_EXTRAFLAGS=.*$', line: 'YAF_EXTRAFLAGS='}
+    - { regexp: '^YAF_CAP_IF=.*$', line: 'YAF_CAP_IF={{ iface }}'}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/roles/yaf/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/vars/main.yml b/deployment/roles/yaf/vars/main.yml
new file mode 100644
index 0000000..b73b8d1
--- /dev/null
+++ b/deployment/roles/yaf/vars/main.yml
@@ -0,0 +1,3 @@
+---
+fixbuf_version: 1.7.1
+yaf_version: 2.8.0

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/vagrant/multinode-vagrant/.gitignore
----------------------------------------------------------------------
diff --git a/deployment/vagrant/multinode-vagrant/.gitignore b/deployment/vagrant/multinode-vagrant/.gitignore
new file mode 100644
index 0000000..8000dd9
--- /dev/null
+++ b/deployment/vagrant/multinode-vagrant/.gitignore
@@ -0,0 +1 @@
+.vagrant

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/vagrant/multinode-vagrant/Vagrantfile
----------------------------------------------------------------------
diff --git a/deployment/vagrant/multinode-vagrant/Vagrantfile b/deployment/vagrant/multinode-vagrant/Vagrantfile
new file mode 100644
index 0000000..052168c
--- /dev/null
+++ b/deployment/vagrant/multinode-vagrant/Vagrantfile
@@ -0,0 +1,45 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+hosts = [
+  { hostname: "node1", ip: "192.168.66.101", memory: "2048", cpus: 2 },
+  { hostname: "node2", ip: "192.168.66.102", memory: "2048", cpus: 2 },
+  { hostname: "node3", ip: "192.168.66.103", memory: "2048", cpus: 2 },
+  { hostname: "node4", ip: "192.168.66.104", memory: "2048", cpus: 2 }
+]
+
+Vagrant.configure(2) do |config|
+
+  # all hosts built on centos 6
+  config.vm.box = "bento/centos-6.7"
+  config.ssh.insert_key = "true"
+
+  # enable the hostmanager plugin
+  config.hostmanager.enabled = true
+  config.hostmanager.manage_host = true
+
+  # define each host
+  hosts.each_with_index do |hosts, index|
+    config.vm.define hosts[:hostname] do |node|
+
+      # host settings
+      node.vm.hostname = hosts[:hostname]
+      node.vm.network "private_network", ip: hosts[:ip]
+
+      # vm settings
+      node.vm.provider "virtualbox" do |vb|
+        vb.memory = hosts[:memory]
+        vb.cpus = hosts[:cpus]
+      end
+
+      # provisioning; only after all hosts created
+      if index == hosts.size - 1
+        node.vm.provision :ansible do |ansible|
+          ansible.playbook = "../../playbooks/metron_full_install.yml"
+          ansible.sudo = true
+          ansible.inventory_path = "../../inventory/singlenode-vagrant"
+        end
+      end
+    end
+  end
+end

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/vagrant/multinode-vagrant/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/multinode-vagrant/ansible.cfg b/deployment/vagrant/multinode-vagrant/ansible.cfg
new file mode 100644
index 0000000..45e5dc4
--- /dev/null
+++ b/deployment/vagrant/multinode-vagrant/ansible.cfg
@@ -0,0 +1,4 @@
+[defaults]
+host_key_checking = false
+library = ../../extra_modules
+roles_path = ../../roles

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/vagrant/singlenode-vagrant/.gitignore
----------------------------------------------------------------------
diff --git a/deployment/vagrant/singlenode-vagrant/.gitignore b/deployment/vagrant/singlenode-vagrant/.gitignore
new file mode 100644
index 0000000..8000dd9
--- /dev/null
+++ b/deployment/vagrant/singlenode-vagrant/.gitignore
@@ -0,0 +1 @@
+.vagrant

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/vagrant/singlenode-vagrant/Vagrantfile
----------------------------------------------------------------------
diff --git a/deployment/vagrant/singlenode-vagrant/Vagrantfile b/deployment/vagrant/singlenode-vagrant/Vagrantfile
new file mode 100644
index 0000000..955535c
--- /dev/null
+++ b/deployment/vagrant/singlenode-vagrant/Vagrantfile
@@ -0,0 +1,40 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+hosts = [
+  { hostname: "node1", ip: "192.168.66.121", memory: "8192", cpus: 2 }
+]
+
+Vagrant.configure(2) do |config|
+
+  # all hosts built on centos 6
+  config.vm.box = "bento/centos-6.7"
+  config.ssh.insert_key = "true"
+
+  # enable the hostmanager plugin
+  config.hostmanager.enabled = true
+  config.hostmanager.manage_host = true
+
+  # host definition
+  hosts.each_with_index do |hosts, index|
+    config.vm.define hosts[:hostname] do |node|
+
+      # host settings
+      node.vm.hostname = hosts[:hostname]
+      node.vm.network "private_network", ip: hosts[:ip]
+
+      # vm settings
+      node.vm.provider "virtualbox" do |vb|
+        vb.memory = hosts[:memory]
+        vb.cpus = hosts[:cpus]
+      end
+    end
+  end
+
+  # provisioning
+  config.vm.provision :ansible do |ansible|
+    ansible.playbook = "../../playbooks/metron_full_install.yml"
+    ansible.sudo = true
+    ansible.inventory_path = "../../inventory/singlenode-vagrant"
+  end
+end

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/aba9e9c5/deployment/vagrant/singlenode-vagrant/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/singlenode-vagrant/ansible.cfg b/deployment/vagrant/singlenode-vagrant/ansible.cfg
new file mode 100644
index 0000000..45e5dc4
--- /dev/null
+++ b/deployment/vagrant/singlenode-vagrant/ansible.cfg
@@ -0,0 +1,4 @@
+[defaults]
+host_key_checking = false
+library = ../../extra_modules
+roles_path = ../../roles


Mime
View raw message