metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ceste...@apache.org
Subject [3/3] incubator-metron git commit: METRON-58 Remediate Deployment Integration Testing Issues (dlyle65535 via cestella) closes apache/incubator-metron#36
Date Mon, 07 Mar 2016 23:25:09 GMT
METRON-58 Remediate Deployment Integration Testing Issues (dlyle65535 via cestella) closes apache/incubator-metron#36


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/2e9f2c6c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/2e9f2c6c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/2e9f2c6c

Branch: refs/heads/master
Commit: 2e9f2c6ceac70fb19d38281a1abe1d3ee0d088bb
Parents: 6638a71
Author: dlyle65535 <dlyle65535@gmail.com>
Authored: Mon Mar 7 18:24:31 2016 -0500
Committer: cstella <cestella@gmail.com>
Committed: Mon Mar 7 18:24:31 2016 -0500

----------------------------------------------------------------------
 .../inventory/metron_example/group_vars/all     |   6 -
 .../inventory/multinode-vagrant/group_vars/all  |   6 -
 .../inventory/singlenode-vagrant/group_vars/all |  11 +-
 deployment/playbooks/ambari_install.yml         |  14 +-
 deployment/playbooks/metron_full_install.yml    |   4 +
 deployment/playbooks/metron_install.yml         |  17 +-
 deployment/roles/ambari_common/tasks/main.yml   |   2 +-
 .../roles/ambari_config/defaults/main.yml       |  30 +++
 .../roles/ambari_config/vars/single_node_vm.yml |  40 ++--
 .../roles/ambari_master/defaults/main.yml       |  19 ++
 deployment/roles/ambari_master/tasks/main.yml   |   7 +
 deployment/roles/bro/defaults/main.yml          |  20 ++
 deployment/roles/bro/tasks/main.yml             |   8 +
 .../roles/elasticsearch/defaults/main.yml       |  20 ++
 deployment/roles/elasticsearch/tasks/main.yml   |   4 +-
 deployment/roles/hadoop_setup/defaults/main.yml |  25 ++
 deployment/roles/hadoop_setup/tasks/main.yml    |   3 +-
 .../roles/metron_common/defaults/main.yml       |  19 ++
 deployment/roles/metron_common/vars/main.yml    |  19 --
 .../roles/metron_streaming/defaults/main.yml    |  31 +++
 .../roles/metron_streaming/files/extractor.json |  11 +
 .../files/source/bro-config.json                |  14 ++
 .../files/source/pcap-config.json               |  14 ++
 .../files/source/snort-config.json              |  14 ++
 .../files/source/yaf-config.json                |  14 ++
 .../roles/metron_streaming/handlers/main.yml    |   4 +-
 .../metron_streaming/tasks/full_topology.yml    |  26 +++
 .../roles/metron_streaming/tasks/main.yml       |  24 +-
 .../metron_streaming/tasks/small_topology.yml   |  26 +++
 .../metron_streaming/tasks/source_config.yml    |  31 +++
 .../metron_streaming/tasks/threat_intel.yml     |  48 ++++
 .../metron_streaming/templates/threat_ip.csv    |  37 +++
 deployment/roles/mysql/files/geoip_ddl.sql      |  49 ----
 deployment/roles/mysql/files/mylogin.cnf        |  19 --
 .../mysql57-community-release-el6-7.noarch.rpm  | Bin 8848 -> 0 bytes
 deployment/roles/mysql/handlers/main.yml        |  19 --
 deployment/roles/mysql/tasks/main.yml           |  85 -------
 deployment/roles/mysql/templates/.my.cnf        |  20 --
 deployment/roles/mysql/vars/main.yml            |  20 --
 deployment/roles/mysql_client/tasks/main.yml    |  34 +++
 .../roles/mysql_client/templates/db_config.sql  |  21 ++
 .../roles/mysql_server/files/geoip_ddl.sql      |  49 ++++
 .../mysql57-community-release-el6-7.noarch.rpm  | Bin 0 -> 8848 bytes
 deployment/roles/mysql_server/handlers/main.yml |  19 ++
 deployment/roles/mysql_server/tasks/main.yml    |  86 +++++++
 deployment/roles/mysql_server/templates/.my.cnf |  20 ++
 deployment/roles/mysql_server/vars/main.yml     |  20 ++
 deployment/roles/pcap_replay/defaults/main.yml  |  21 ++
 .../roles/pcap_replay/templates/pcap-replay     |   2 +-
 deployment/roles/pcap_replay/vars/main.yml      |  21 --
 .../roles/tap_interface/defaults/main.yml       |  19 ++
 deployment/roles/tap_interface/tasks/main.yml   |  30 +++
 deployment/roles/yaf/defaults/main.yml          |  29 +++
 deployment/roles/yaf/tasks/main.yml             |   9 +-
 deployment/roles/yaf/vars/main.yml              |  22 --
 metron-streaming/Metron-Common/pom.xml          |   5 +
 .../metron/bolt/BulkMessageWriterBolt.java      |   5 +-
 .../java/org/apache/metron/bolt/JoinBolt.java   |   1 +
 .../org/apache/metron/domain/Enrichment.java    |  11 +
 .../java/org/apache/metron/utils/JSONUtils.java |  70 ++++++
 .../resources/config/source/bro-config.json     |   9 +-
 .../resources/config/source/snort-config.json   |   9 +-
 .../resources/config/source/yaf-config.json     |   9 +-
 .../dataloads/bulk/ThreatIntelBulkLoader.java   |   2 +-
 .../enrichment/bolt/EnrichmentJoinBolt.java     |  19 +-
 .../enrichment/bolt/EnrichmentSplitterBolt.java |   2 +
 .../enrichment/bolt/GenericEnrichmentBolt.java  |   6 +-
 .../enrichment/bolt/ThreatIntelJoinBolt.java    |  13 +-
 .../metron/threatintel/ThreatIntelAdapter.java  |   2 +-
 .../metron/indexing/TelemetryIndexingBolt.java  |  11 +
 .../metron/writer/ElasticsearchWriter.java      |  10 +-
 .../metron/parsing/parsers/BasicBroParser.java  |   7 +-
 .../parsing/parsers/BasicSnortParser.java       |   1 +
 .../metron/parsing/test/BasicBroParserTest.java |  22 ++
 .../src/test/resources/BroParserTest.log        |   3 +-
 .../util/integration/ComponentRunner.java       |  33 ++-
 metron-streaming/Metron-Topologies/pom.xml      |  17 +-
 .../apache/metron/utils/SourceConfigUtils.java  |  37 ++-
 .../Metron_Configs/etc/env/config.properties    |   3 +-
 .../Metron_Configs/topologies/bro/remote.yaml   |  15 +-
 .../Metron_Configs/topologies/bro/test.yaml     |  10 -
 .../topologies/enrichment/remote.yaml           |  13 +-
 .../topologies/paloalto/test.yaml               |   2 +-
 .../Metron_Configs/topologies/pcap/parse.yaml   |   4 +-
 .../Metron_Configs/topologies/pcap/remote.yaml  |  97 +++-----
 .../Metron_Configs/topologies/snort/remote.yaml |  10 -
 .../Metron_Configs/topologies/snort/test.yaml   |  12 +-
 .../Metron_Configs/topologies/yaf/remote.yaml   |  15 +-
 .../Metron_Configs/topologies/yaf/test.yaml     |  10 -
 .../src/main/resources/SampleIndexed/YafIndexed |  20 +-
 .../src/main/resources/SampleParsed/SnortParsed |   6 +-
 .../integration/EnrichmentIntegrationTest.java  | 226 +++++++++++++++++--
 .../integration/ParserIntegrationTest.java      |  15 +-
 .../metron/integration/util/TestUtils.java      |   1 -
 .../integration/util/mock/MockGeoAdapter.java   |  26 ++-
 pom.xml                                         |   3 +-
 96 files changed, 1382 insertions(+), 592 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index b8cf9dc..3a26769 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -31,12 +31,6 @@ pcap_hbase_table: pcap
 tracker_hbase_table: access_tracker
 threatintel_ip_hbase_table: malicious_ip
 
-# kafka
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: ipfix
-snort_topic: snort
-
 #elasticsearch
 elasticsearch_transport_port: 9300
 elasticsearch_network_interface: eth0

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/multinode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/multinode-vagrant/group_vars/all b/deployment/inventory/multinode-vagrant/group_vars/all
index bb41e89..fc3b56d 100644
--- a/deployment/inventory/multinode-vagrant/group_vars/all
+++ b/deployment/inventory/multinode-vagrant/group_vars/all
@@ -28,12 +28,6 @@ pcap_hbase_table: pcap
 tracker_hbase_table: access_tracker
 threatintel_ip_hbase_table: malicious_ip
 
-# kafka
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: ipfix
-snort_topic: snort
-
 #elasticsearch
 elasticsearch_transport_port: 9300
 elasticsearch_network_interface: eth1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index 1e08a6a..6405eea 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -28,12 +28,6 @@ pcap_hbase_table: pcap
 tracker_hbase_table: access_tracker
 threatintel_ip_hbase_table: malicious_ip
 
-# kafka
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: ipfix
-snort_topic: snort
-
 #elasticsearch
 elasticsearch_transport_port: 9300
 elasticsearch_network_interface: eth1
@@ -55,7 +49,7 @@ snort_version: "2.9.8.0-1"
 snort_alert_csv_path: "/var/log/snort/alert.csv"
 
 #PCAP Replay
-pcap_replay: True
+pcap_replay: False
 pcap_replay_interface: eth1
 
 #data directories - only required to override defaults
@@ -73,3 +67,6 @@ storm_local_dir: "/data1/hadoop/storm"
 kafka_log_dirs: "/data1/kafka-log"
 elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
 
+ambari_server_mem: 512
+topology_name: small_topology.yml
+threat_intel_bulk_load: False

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/ambari_install.yml b/deployment/playbooks/ambari_install.yml
index e1da427..c7f8249 100644
--- a/deployment/playbooks/ambari_install.yml
+++ b/deployment/playbooks/ambari_install.yml
@@ -19,17 +19,29 @@
   sudo: yes
   roles:
     - role: ambari_common
+  tags:
+    - ambari-prereqs
+    - hdp-install
 
 - hosts: ambari_master
   sudo: yes
   roles:
-    - role: ambari_master
+    - role:  ambari_master
+  tags:
+    - ambari-server
+    - hdp-install
 
 - hosts: ambari_slave
   sudo: yes
   roles:
     - role: ambari_slave
+  tags:
+    - ambari-agent
+    - hdp-install
 
 - hosts: ambari_master
   roles:
     - role: ambari_config
+  tags:
+    - hdp-install
+    - hdp-deploy

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/metron_full_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_full_install.yml b/deployment/playbooks/metron_full_install.yml
index 38203da..26ffd62 100644
--- a/deployment/playbooks/metron_full_install.yml
+++ b/deployment/playbooks/metron_full_install.yml
@@ -16,4 +16,8 @@
 #
 ---
 - include: ambari_install.yml
+  tags:
+    - ambari
 - include: metron_install.yml
+  tags:
+    - metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index ad070c9..b8646fc 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -19,12 +19,16 @@
   sudo: yes
   roles:
     - role: metron_common
+  tags:
+    - metron-prereqs
 
 - hosts: hadoop_client
   sudo: yes
   roles:
     - role: ambari_gather_facts
     - role: hadoop_setup
+  tags:
+    - metron-prereqs
 
 - hosts: search
   sudo: yes
@@ -38,13 +42,22 @@
 - hosts: mysql
   sudo: yes
   roles:
-    - role: mysql
+    - role: mysql_server
   tags:
-    - mysql
+    - mysql-server
+
+- hosts: ambari_slave
+  sudo: yes
+  roles:
+    - role: mysql_client
+  tags:
+    - mysql-client
+
 
 - hosts: sensors
   sudo: yes
   roles:
+    - { role: tap_interface, when: install_tap | default(False) == True }
     - role: ambari_gather_facts
     - role: flume
     - role: pycapa

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/tasks/main.yml b/deployment/roles/ambari_common/tasks/main.yml
index 992468e..35f3fce 100644
--- a/deployment/roles/ambari_common/tasks/main.yml
+++ b/deployment/roles/ambari_common/tasks/main.yml
@@ -59,7 +59,7 @@
 - name: install epel-repo rpm
   yum: pkg=/tmp/epel-release.rpm state=installed
 
-- name: Download HDP repo
+- name: Download Ambari repo
   get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
 
 - name: Clean yum

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_config/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/defaults/main.yml b/deployment/roles/ambari_config/defaults/main.yml
new file mode 100644
index 0000000..507b6e3
--- /dev/null
+++ b/deployment/roles/ambari_config/defaults/main.yml
@@ -0,0 +1,30 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+zookeeper_data_dir: /hadoop/zookeeper
+namenode_checkpoint_dir: /hadoop/hdfs/namesecondary
+namenode_name_dir: /hadoop/hdfs/namenode
+datanode_data_dir: /hadoop/hdfs/data
+journalnode_edits_dir: /hadoop/hdfs/journalnode
+jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs
+nodemanager_local_dirs: /hadoop/yarn/local
+timeline_ldb_store_path: /hadoop/yarn/timeline
+timeline_ldb_state_path: /hadoop/yarn/timeline
+nodemanager_log_dirs: /hadoop/yarn/log
+storm_local_dir: /hadoop/storm
+kafka_log_dirs: /kafka-log
+cluster_type: small_cluster

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/single_node_vm.yml b/deployment/roles/ambari_config/vars/single_node_vm.yml
index d0d3b78..6b18825 100644
--- a/deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -29,47 +29,49 @@ zookeeper_slave: [ZOOKEEPER_CLIENT]
 hbase_master: [HBASE_MASTER, HBASE_CLIENT]
 hbase_slave: [HBASE_REGIONSERVER]
 
-metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(spark_slave) | union(kafka_broker) | union(hbase_slave) }}"
+metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(kafka_broker) | union(hbase_slave) }}"
 
 cluster_name: "metron_cluster"
 blueprint_name: "metron_blueprint"
 
 configurations:
   - zoo.cfg:
-      dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
+      dataDir: '{{ zookeeper_data_dir }}'
   - hadoop-env:
-      namenode_heapsize: 1024
-      dtnode_heapsize: 1024
+      hadoop_heapsize: 1024
+      namenode_heapsize: 512
+      dtnode_heapsize: 512
+      namenode_opt_permsize: 128m
   - hbase-env:
-      hbase_regionserver_heapsize: 1024
-      hbase_master_heapsize: 1024
+      hbase_regionserver_heapsize: 512
+      hbase_master_heapsize: 512
+      hbase_regionserver_xmn_max: 512
   - hdfs-site:
-      dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
-      dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
-      dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
-      dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
+      dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir  }}'
+      dfs.namenode.name.dir: '{{ namenode_name_dir }}'
+      dfs.datanode.data.dir: '{{ datanode_data_dir }}'
+      dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}'
   - yarn-env:
       nodemanager_heapsize: 512
       yarn_heapsize: 512
       apptimelineserver_heapsize : 512
+      resourcemanager_heapsize: 1024
   - mapred-env:
       jobhistory_heapsize: 256
   - mapred-site:
-      mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
+      mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}'
   - yarn-site:
-      yarn.nodemanager.resource.memory-mb: 1024
-      yarn.scheduler.maximum-allocation-mb: 1024
-      yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
-      yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
-      yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
-      yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
+      yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}'
+      yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}'
+      yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}'
+      yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}'
   - storm-site:
       supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
-      storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
+      storm.local.dir: '{{ storm_local_dir }}'
   - kafka-env:
       content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n  export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n  export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n   . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
   - kafka-broker:
-      log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
+      log.dirs: '{{ kafka_log_dirs }}'
 
 blueprint:
   stack_name: HDP

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_master/defaults/main.yml b/deployment/roles/ambari_master/defaults/main.yml
new file mode 100644
index 0000000..3b8cc73
--- /dev/null
+++ b/deployment/roles/ambari_master/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+ambari_server_mem: 2048
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_master/tasks/main.yml b/deployment/roles/ambari_master/tasks/main.yml
index 8c78f06..daf4e41 100644
--- a/deployment/roles/ambari_master/tasks/main.yml
+++ b/deployment/roles/ambari_master/tasks/main.yml
@@ -24,6 +24,13 @@
   register: ambari_server_setup
   failed_when: ambari_server_setup.stderr
 
+- name: Set Ambari Server Max Memory
+  replace:
+    dest: /var/lib/ambari-server/ambari-env.sh
+    regexp:  "\ -Xmx2048m\ "
+    replace: " -Xmx{{ ambari_server_mem }}m "
+    backup: no
+
 - name: start ambari server
   service: name=ambari-server state=restarted
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/bro/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/defaults/main.yml b/deployment/roles/bro/defaults/main.yml
new file mode 100644
index 0000000..c7a2c1f
--- /dev/null
+++ b/deployment/roles/bro/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+bro_crontab_minutes: 0-59/5
+bro_crontab_job: /usr/local/bro/bin/broctl cron
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/tasks/main.yml b/deployment/roles/bro/tasks/main.yml
index 04dfe8f..0191052 100644
--- a/deployment/roles/bro/tasks/main.yml
+++ b/deployment/roles/bro/tasks/main.yml
@@ -31,6 +31,7 @@
     - python-devel
     - swig
     - zlib-devel
+    - perl
 
 - include: librdkafka.yml
 
@@ -46,3 +47,10 @@
 
 - name: Start bro
   shell: /usr/local/bro/bin/broctl start
+
+- name: Bro Cronjob
+  cron:
+    name: Bro Cron
+    minute: "{{ bro_crontab_minutes }}"
+    job: "{{ bro_crontab_job }}"
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/elasticsearch/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/defaults/main.yml b/deployment/roles/elasticsearch/defaults/main.yml
new file mode 100644
index 0000000..d91fa1a
--- /dev/null
+++ b/deployment/roles/elasticsearch/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+elasticsearch_data_dir: /var/lib/elasticsearch
+elasticsearch_network_interface: eth0
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/tasks/main.yml b/deployment/roles/elasticsearch/tasks/main.yml
index 555666b..fa8d4f3 100644
--- a/deployment/roles/elasticsearch/tasks/main.yml
+++ b/deployment/roles/elasticsearch/tasks/main.yml
@@ -55,10 +55,10 @@
   with_items:
     - { regexp: '#cluster\.name', line: 'cluster.name: metron' }
     - { regexp: '#network\.host:', line: 'network.host: _{{
-    elasticsearch_network_interface | default("eth0") }}:ipv4_' }
+    elasticsearch_network_interface  }}:ipv4_' }
     - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
     line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
-    - { regexp: '#path\.data', line: 'path.data: {{     elasticsearch_data_dir | default("/var/lib/elasticsearch")}}' }
+    - { regexp: '#path\.data', line: 'path.data: {{     elasticsearch_data_dir }}' }
   notify: restart elasticsearch
 
 - name: Start Elasticsearch.

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/hadoop_setup/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/hadoop_setup/defaults/main.yml b/deployment/roles/hadoop_setup/defaults/main.yml
new file mode 100644
index 0000000..c783cea
--- /dev/null
+++ b/deployment/roles/hadoop_setup/defaults/main.yml
@@ -0,0 +1,25 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+num_partitions: 1
+retention_in_gb: 10
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: ipfix
+snort_topic: snort
+enrichments_topic: enrichments
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/hadoop_setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/hadoop_setup/tasks/main.yml b/deployment/roles/hadoop_setup/tasks/main.yml
index 5e77b99..5b6c47c 100644
--- a/deployment/roles/hadoop_setup/tasks/main.yml
+++ b/deployment/roles/hadoop_setup/tasks/main.yml
@@ -26,10 +26,11 @@
 
 #if kafka topic
 - name: Create Kafka topics
-  shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions 1 --replication-factor 1"
+  shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions {{ num_partitions }} --replication-factor 1 --config retention.bytes={{ retention_in_gb * 1024 * 1024 * 1024}}"
   ignore_errors: yes
   with_items:
     - "{{ pycapa_topic }}"
     - "{{ bro_topic }}"
     - "{{ yaf_topic }}"
     - "{{ snort_topic }}"
+    - "{{ enrichments_topic }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/defaults/main.yml b/deployment/roles/metron_common/defaults/main.yml
new file mode 100644
index 0000000..50aaefd
--- /dev/null
+++ b/deployment/roles/metron_common/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
+metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_common/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/vars/main.yml b/deployment/roles/metron_common/vars/main.yml
deleted file mode 100644
index 50aaefd..0000000
--- a/deployment/roles/metron_common/vars/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
-metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
new file mode 100644
index 0000000..cb425f9
--- /dev/null
+++ b/deployment/roles/metron_streaming/defaults/main.yml
@@ -0,0 +1,31 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+source_config_path: "{{ metron_directory }}/config/source"
+threat_intel_bulk_load: True
+threat_intel_bin: "{{ metron_directory }}/bin/threatintel_bulk_load.sh"
+threat_intel_host: "{{ groups.ambari_master[0] }}"
+threat_intel_work_dir: /tmp/ti_bulk
+threat_intel_csv_filename: "threat_ip.csv"
+threat_intel_csv_filepath: "../roles/metron_streaming/templates/{{ threat_intel_csv_filename }}"
+
+topology_name: full_topology.yml
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: ipfix
+snort_topic: snort
+enrichments_topic: enrichments

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/extractor.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/extractor.json b/deployment/roles/metron_streaming/files/extractor.json
new file mode 100644
index 0000000..81429e8
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/extractor.json
@@ -0,0 +1,11 @@
+{
+  "config": {
+    "columns": {
+      "ip": 0
+    },
+    "indicator_column": "ip",
+    "separator": ","
+  },
+  "extractor": "CSV"
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/bro-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/bro-config.json b/deployment/roles/metron_streaming/files/source/bro-config.json
new file mode 100644
index 0000000..34109b8
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/bro-config.json
@@ -0,0 +1,14 @@
+{
+  "index": "bro",
+  "batchSize": 5,
+  "enrichmentFieldMap":
+  {
+    "geo": ["ip_dst_addr", "ip_src_addr"],
+    "host": ["host"]
+  },
+  "threatIntelFieldMap":
+  {
+    "ip": ["ip_dst_addr", "ip_src_addr"]
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/pcap-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/pcap-config.json b/deployment/roles/metron_streaming/files/source/pcap-config.json
new file mode 100644
index 0000000..4b9c639
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/pcap-config.json
@@ -0,0 +1,14 @@
+{
+  "index": "pcap",
+  "batchSize": 5,
+  "enrichmentFieldMap":
+  {
+    "geo": ["ip_src_addr", "ip_dst_addr"],
+    "host": ["ip_src_addr", "ip_dst_addr"]
+  },
+  "threatIntelFieldMap":
+  {
+    "ip": ["ip_src_addr", "ip_dst_addr"]
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/snort-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/snort-config.json b/deployment/roles/metron_streaming/files/source/snort-config.json
new file mode 100644
index 0000000..1208637
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/snort-config.json
@@ -0,0 +1,14 @@
+{
+  "index": "snort",
+  "batchSize": 1,
+  "enrichmentFieldMap":
+  {
+    "geo": ["ip_dst_addr", "ip_src_addr"],
+    "host": ["host"]
+  },
+  "threatIntelFieldMap":
+  {
+    "ip": ["ip_dst_addr", "ip_src_addr"]
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/files/source/yaf-config.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/source/yaf-config.json b/deployment/roles/metron_streaming/files/source/yaf-config.json
new file mode 100644
index 0000000..65de961
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/source/yaf-config.json
@@ -0,0 +1,14 @@
+{
+  "index": "yaf",
+  "batchSize": 5,
+  "enrichmentFieldMap":
+  {
+    "geo": ["ip_dst_addr", "ip_src_addr"],
+    "host": ["host"]
+  },
+  "threatIntelFieldMap":
+  {
+    "ip": ["ip_dst_addr", "ip_src_addr"]
+  }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/handlers/main.yml b/deployment/roles/metron_streaming/handlers/main.yml
index 112c5ca..634d591 100644
--- a/deployment/roles/metron_streaming/handlers/main.yml
+++ b/deployment/roles/metron_streaming/handlers/main.yml
@@ -15,5 +15,5 @@
 #  limitations under the License.
 #
 ---
-- name: restart elasticsearch
-  service: name=elasticsearch state=restarted
+- name: Load Source Config
+  shell: java -cp {{ metron_directory }}/lib/{{ metron_jar_name }}::/usr/hdp/current/hadoop-client/lib/slf4j-api-1.7.10.jar org.apache.metron.utils.SourceConfigUtils -p {{ source_config_path }} -z {{ zookeeper_url }} && touch {{ source_config_path }}/configured

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/full_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/full_topology.yml b/deployment/roles/metron_streaming/tasks/full_topology.yml
new file mode 100644
index 0000000..060caf8
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/full_topology.yml
@@ -0,0 +1,26 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+
+- name: Submit Metron topologies
+  command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
+  with_items:
+    - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
+    - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index 7d6fe9c..c1e1642 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -52,6 +52,10 @@
     - "etc"
     - "topologies"
 
+- name: Get Default mysql passowrd
+  include_vars: "../roles/mysql_server/vars/main.yml"
+  when: mysql_root_password is undefined
+
 - name: Configure Metron topologies
   lineinfile: >
     dest={{ metron_properties_config_path }}
@@ -59,6 +63,7 @@
     line="{{ item.line }}"
   with_items:
     - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
+    - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
     - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
     - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
     - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
@@ -70,7 +75,8 @@
     - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
     - { regexp: "threat.intel.ip.table=", line: "threat.intel.ip.table={{ threatintel_ip_hbase_table }}" }
     - { regexp: "threat.intel.ip.cf=", line: "threat.intel.ip.cf=t" }
-    - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.search[0] }}" }
+    - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
+    - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
 
 - name: Add Elasticsearch templates for topologies
   uri:
@@ -80,11 +86,11 @@
     status_code: 200
     body_format: json
 
-- name: Submit Metron topologies
-  command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
-  ignore_errors: yes
-  with_items:
-    - "{{ metron_directory }}/config/topologies/pcap/remote.yaml"
-    - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
-    - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
-    - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+- include: source_config.yml
+  run_once: true
+- include: threat_intel.yml
+  run_once: true
+  when: threat_intel_bulk_load == True
+
+- include: "{{ topology_name }}"
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/small_topology.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/small_topology.yml b/deployment/roles/metron_streaming/tasks/small_topology.yml
new file mode 100644
index 0000000..6707210
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/small_topology.yml
@@ -0,0 +1,26 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+
+- name: Submit Metron topologies
+  command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
+  with_items:
+    - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
+    - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/source_config.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/source_config.yml b/deployment/roles/metron_streaming/tasks/source_config.yml
new file mode 100644
index 0000000..9233bac
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/source_config.yml
@@ -0,0 +1,31 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create Source Config Directory
+  file:
+    path: "{{ source_config_path }}"
+    state: directory
+
+- name: Copy Source Config Files
+  copy:
+    src: "{{ item }}"
+    dest: "{{ source_config_path }}"
+    mode: 0644
+  with_fileglob:
+    - ../roles/metron_streaming/files/source/*.json
+  notify: Load Source Config
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/tasks/threat_intel.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/threat_intel.yml b/deployment/roles/metron_streaming/tasks/threat_intel.yml
new file mode 100644
index 0000000..0439e46
--- /dev/null
+++ b/deployment/roles/metron_streaming/tasks/threat_intel.yml
@@ -0,0 +1,48 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create root user HDFS directory
+  command: su - hdfs -c "hdfs dfs -mkdir -p /user/root && hdfs dfs -chown root:root /user/root"
+
+- name: Create Bulk load working Directory
+  file:
+    path: "{{ threat_intel_work_dir }}"
+    state: directory
+
+- name: Copy extractor.json to {{ inventory_hostname }}
+  copy:
+    src: ../roles/metron_streaming/files/extractor.json
+    dest: "{{  threat_intel_work_dir }}"
+    mode: 0644
+
+- name: Copy Bulk Load CSV File
+  template:
+    src: "{{ threat_intel_csv_filepath }}"
+    dest: "{{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }}"
+    mode: 0644
+
+- name: Copy Bulk Load CSV File to HDFS
+  command: "hdfs dfs -put {{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }} ."
+
+- name: Run Threat Intel Bulk Load
+  shell: "{{ threat_intel_bin }} -f t --table malicious_ip -e {{ threat_intel_work_dir }}/extractor.json  -i /user/root && touch {{ threat_intel_work_dir }}/loaded"
+  args:
+    creates: "{{ threat_intel_work_dir }}/loaded"
+
+- name: Clean up HDFS File
+  command: "hdfs dfs -rm {{ threat_intel_csv_filename }}"
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/metron_streaming/templates/threat_ip.csv
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/templates/threat_ip.csv b/deployment/roles/metron_streaming/templates/threat_ip.csv
new file mode 100644
index 0000000..3ac38f3
--- /dev/null
+++ b/deployment/roles/metron_streaming/templates/threat_ip.csv
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#Add single column of ip address to alert
+#Public lists are available on the internet
+# example: 
+23.113.113.105
+24.107.205.249
+24.108.62.255
+24.224.153.71
+27.4.1.212
+27.131.149.102
+31.24.30.31
+31.131.251.33
+31.186.99.250
+31.192.209.119
+31.192.209.150
+31.200.244.17
+37.34.52.185
+37.58.112.101
+37.99.146.27
+37.128.132.96
+37.140.195.177
+37.140.199.100

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/geoip_ddl.sql b/deployment/roles/mysql/files/geoip_ddl.sql
deleted file mode 100644
index 02616c6..0000000
--- a/deployment/roles/mysql/files/geoip_ddl.sql
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-CREATE DATABASE IF NOT EXISTS GEO;
-
-USE GEO;
-
-DROP TABLE IF EXISTS `blocks`;
-CREATE TABLE  `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
-int(10) unsigned NOT NULL, PRIMARY KEY  (`startIPNum`,`endIPNum`) )
-ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
-
-DROP TABLE IF EXISTS `location`;
-CREATE TABLE  `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
- default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
-NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
-  (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
-
-load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv'  into table `blocks`  fields terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
-load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv'  into table `location`  fields terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
-
-
-DELIMITER $$
-DROP FUNCTION IF EXISTS `IPTOLOCID` $$
-CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
-  BEGIN
-    DECLARE ipn INTEGER UNSIGNED;
-    DECLARE locID_var INTEGER;
-    IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
-    END IF;
-    SET ipn = INET_ATON(ip);
-    SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
-    RETURN locID_var;
-  END
-$$
-DELIMITER ;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/mylogin.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/mylogin.cnf b/deployment/roles/mysql/files/mylogin.cnf
deleted file mode 100644
index b8d5781..0000000
--- a/deployment/roles/mysql/files/mylogin.cnf
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-[client]
-user=root
-password=P@ssw0rd
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm b/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm
deleted file mode 100644
index 8603602..0000000
Binary files a/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm and /dev/null differ

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/handlers/main.yml b/deployment/roles/mysql/handlers/main.yml
deleted file mode 100644
index 112c5ca..0000000
--- a/deployment/roles/mysql/handlers/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-- name: restart elasticsearch
-  service: name=elasticsearch state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/tasks/main.yml b/deployment/roles/mysql/tasks/main.yml
deleted file mode 100644
index 91db896..0000000
--- a/deployment/roles/mysql/tasks/main.yml
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-- name: Create temporary directories
-  file:
-    path: "/tmp/{{ item }}"
-    state: directory
-    mode: 0755
-  with_items:
-    - "geoip"
-
-- name: Distribute Mysql
-  copy:
-    src: "{{ mysql_rpm_version }}.rpm"
-    dest: /tmp
-
-- name: Install Msyql Yum Repository
-  yum:
-    name: "/tmp/{{ mysql_rpm_version }}.rpm"
-
-- name: Install MySQL
-  yum:
-    name: "{{ item }}"
-    state: latest
-  with_items:
-    - "mysql-community-server"
-    - "MySQL-python"
-
-- name: Start MySQL
-  service:
-    name: mysqld
-    state: started
-    enabled: yes
-
-- name: Retrieve temporary root password
-  shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
-  args:
-    creates: ~/.my.cnf
-  register: temp_root_password
-
-- name: Update mysql root password
-  command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
-  ignore_errors: yes
-  args:
-    creates: ~/.my.cnf
-
-- name: Copy mylogin.cnf
-  copy:
-    src:  mylogin.cnf
-    dest: ~/.my.cnf
-
-- name: Download GeoIP databases
-  unarchive:
-    src:  http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
-    dest: /tmp/geoip
-    copy: no
-    creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
-
-- name: Copy to MySQL import directory
-  shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
-
-- name: Copy DDL
-  copy:
-    src: geoip_ddl.sql
-    dest: /tmp/geoip_ddl.sql
-
-- name: Import GeoIP DDL
-  mysql_db:
-    name: all
-    state: import
-    target: /tmp/geoip_ddl.sql

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/templates/.my.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/templates/.my.cnf b/deployment/roles/mysql/templates/.my.cnf
deleted file mode 100644
index d5c0825..0000000
--- a/deployment/roles/mysql/templates/.my.cnf
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-[client]
-user=root
-password={{ mysql_root_password }}
-host=localhost
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/vars/main.yml b/deployment/roles/mysql/vars/main.yml
deleted file mode 100644
index ccf2426..0000000
--- a/deployment/roles/mysql/vars/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-mysql_rpm_version: mysql57-community-release-el6-7.noarch
-mysql_root_password: P@ssw0rd
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_client/tasks/main.yml b/deployment/roles/mysql_client/tasks/main.yml
new file mode 100644
index 0000000..5c98eb9
--- /dev/null
+++ b/deployment/roles/mysql_client/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+
+- name: Get Default mysql passowrd
+  include_vars: "../roles/mysql_server/vars/main.yml"
+  when: mysql_root_password is undefined
+
+- name: Allow remote login to mysql
+  template:
+    src: "../roles/mysql_client/templates/db_config.sql"
+    dest: "/tmp/{{ansible_fqdn}}.sql"
+  delegate_to: "{{ groups.mysql[0] }}"
+
+- name: Import DB_Config
+  mysql_db:
+    name: "all"
+    state: "import"
+    target: "/tmp/{{ansible_fqdn}}.sql"
+  delegate_to: "{{ groups.mysql[0] }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_client/templates/db_config.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_client/templates/db_config.sql b/deployment/roles/mysql_client/templates/db_config.sql
new file mode 100644
index 0000000..c407a13
--- /dev/null
+++ b/deployment/roles/mysql_client/templates/db_config.sql
@@ -0,0 +1,21 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+
+CREATE USER 'root'@'{{ ansible_fqdn }}' IDENTIFIED BY '{{ mysql_root_password }}';
+SET PASSWORD FOR 'root'@'{{ ansible_fqdn }}' = PASSWORD('{{ mysql_root_password }}');
+GRANT ALL PRIVILEGES ON *.* to 'root'@'{{ ansible_fqdn }}' WITH GRANT OPTION;
+FLUSH PRIVILEGES;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/files/geoip_ddl.sql b/deployment/roles/mysql_server/files/geoip_ddl.sql
new file mode 100644
index 0000000..02616c6
--- /dev/null
+++ b/deployment/roles/mysql_server/files/geoip_ddl.sql
@@ -0,0 +1,49 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+CREATE DATABASE IF NOT EXISTS GEO;
+
+USE GEO;
+
+DROP TABLE IF EXISTS `blocks`;
+CREATE TABLE  `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
+int(10) unsigned NOT NULL, PRIMARY KEY  (`startIPNum`,`endIPNum`) )
+ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
+
+DROP TABLE IF EXISTS `location`;
+CREATE TABLE  `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
+ default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
+NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
+  (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
+
+load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv'  into table `blocks`  fields terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
+load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv'  into table `location`  fields terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
+
+
+DELIMITER $$
+DROP FUNCTION IF EXISTS `IPTOLOCID` $$
+CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
+  BEGIN
+    DECLARE ipn INTEGER UNSIGNED;
+    DECLARE locID_var INTEGER;
+    IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
+    END IF;
+    SET ipn = INET_ATON(ip);
+    SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
+    RETURN locID_var;
+  END
+$$
+DELIMITER ;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm b/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm
new file mode 100644
index 0000000..8603602
Binary files /dev/null and b/deployment/roles/mysql_server/files/mysql57-community-release-el6-7.noarch.rpm differ

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/handlers/main.yml b/deployment/roles/mysql_server/handlers/main.yml
new file mode 100644
index 0000000..112c5ca
--- /dev/null
+++ b/deployment/roles/mysql_server/handlers/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: restart elasticsearch
+  service: name=elasticsearch state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/tasks/main.yml b/deployment/roles/mysql_server/tasks/main.yml
new file mode 100644
index 0000000..987c160
--- /dev/null
+++ b/deployment/roles/mysql_server/tasks/main.yml
@@ -0,0 +1,86 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Create temporary directories
+  file:
+    path: "/tmp/{{ item }}"
+    state: directory
+    mode: 0755
+  with_items:
+    - "geoip"
+
+- name: Distribute Mysql
+  copy:
+    src: "{{ mysql_rpm_version }}.rpm"
+    dest: /tmp
+
+- name: Install Msyql Yum Repository
+  yum:
+    name: "/tmp/{{ mysql_rpm_version }}.rpm"
+
+- name: Install MySQL
+  yum:
+    name: "{{ item }}"
+    state: latest
+  with_items:
+    - "mysql-community-server"
+    - "MySQL-python"
+
+- name: Start MySQL
+  service:
+    name: mysqld
+    state: started
+    enabled: yes
+
+- name: Retrieve temporary root password
+  shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
+  args:
+    creates: ~/.my.cnf
+  register: temp_root_password
+
+- name: Update mysql root password
+  command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
+  ignore_errors: yes
+  args:
+    creates: ~/.my.cnf
+
+- name: Create .my.cnf
+  template:
+    src: "../roles/mysql_server/templates/.my.cnf"
+    dest: ~/.my.cnf
+
+
+- name: Download GeoIP databases
+  unarchive:
+    src:  http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
+    dest: /tmp/geoip
+    copy: no
+    creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
+
+- name: Copy to MySQL import directory
+  shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
+
+- name: Copy DDL
+  copy:
+    src: geoip_ddl.sql
+    dest: /tmp/geoip_ddl.sql
+
+- name: Import GeoIP DDL
+  mysql_db:
+    name: all
+    state: import
+    target: /tmp/geoip_ddl.sql

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/templates/.my.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/templates/.my.cnf b/deployment/roles/mysql_server/templates/.my.cnf
new file mode 100644
index 0000000..d5c0825
--- /dev/null
+++ b/deployment/roles/mysql_server/templates/.my.cnf
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+[client]
+user=root
+password={{ mysql_root_password }}
+host=localhost
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/mysql_server/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql_server/vars/main.yml b/deployment/roles/mysql_server/vars/main.yml
new file mode 100644
index 0000000..ccf2426
--- /dev/null
+++ b/deployment/roles/mysql_server/vars/main.yml
@@ -0,0 +1,20 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+mysql_rpm_version: mysql57-community-release-el6-7.noarch
+mysql_root_password: P@ssw0rd
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/defaults/main.yml b/deployment/roles/pcap_replay/defaults/main.yml
new file mode 100644
index 0000000..b1fae1e
--- /dev/null
+++ b/deployment/roles/pcap_replay/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+pcap_replay_interface: eth0
+pcap_path: /opt/pcap-replay
+tcpreplay_version: 4.1.1
+tcpreplay_prefix: /opt

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/templates/pcap-replay
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/templates/pcap-replay b/deployment/roles/pcap_replay/templates/pcap-replay
index 56dc40c..b9ae0c3 100644
--- a/deployment/roles/pcap_replay/templates/pcap-replay
+++ b/deployment/roles/pcap_replay/templates/pcap-replay
@@ -24,7 +24,7 @@
 
 DAEMON_PATH="{{ pcap_path }}"
 PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null`
-IFACE="{{ pcap_replay_interface | default("eth0") }}"
+IFACE="{{ pcap_replay_interface }}"
 EXTRA_ARGS="${@:2}"
 DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay"
 DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/pcap_replay/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/vars/main.yml b/deployment/roles/pcap_replay/vars/main.yml
deleted file mode 100644
index b1fae1e..0000000
--- a/deployment/roles/pcap_replay/vars/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-pcap_replay_interface: eth0
-pcap_path: /opt/pcap-replay
-tcpreplay_version: 4.1.1
-tcpreplay_prefix: /opt

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/tap_interface/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/tap_interface/defaults/main.yml b/deployment/roles/tap_interface/defaults/main.yml
new file mode 100644
index 0000000..ca752b4
--- /dev/null
+++ b/deployment/roles/tap_interface/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+tap_if: tap0
+tap_ip: 10.0.0.1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/tap_interface/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/tap_interface/tasks/main.yml b/deployment/roles/tap_interface/tasks/main.yml
new file mode 100644
index 0000000..d4590f7
--- /dev/null
+++ b/deployment/roles/tap_interface/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+- name: Install tunctl
+  yum: name=tunctl
+
+#TODO - only run when tap_if does not exist
+- name: Create {{ tap_if }}
+  command: tunctl -p
+
+- name: Bring up {{ tap_if }} on {{ tap_ip }}
+  command: ifconfig {{ tap_if }} {{ tap_ip }} up
+
+- name:  Put {{ tap_if }} in PROMISC
+  command: ip link set {{ tap_if }} promisc on
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/defaults/main.yml b/deployment/roles/yaf/defaults/main.yml
new file mode 100644
index 0000000..f804cb5
--- /dev/null
+++ b/deployment/roles/yaf/defaults/main.yml
@@ -0,0 +1,29 @@
+#
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+fixbuf_version: 1.7.1
+yaf_version: 2.8.0
+yaf_home: /opt/yaf
+yaf_topic: ipfix
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
+yaf: /usr/local/bin/yaf
+yaf_args: ""
+yafscii: /usr/local/bin/yafscii
+yaf_log: /var/log/yaf.log
+yaf_lock: /var/lock/subsys/yaf
+kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml
index 1e1194d..468a4f9 100644
--- a/deployment/roles/yaf/tasks/main.yml
+++ b/deployment/roles/yaf/tasks/main.yml
@@ -43,13 +43,6 @@
 - name: Install kafka
   yum: name=kafka
 
-- set_fact:
-    yaf: /usr/local/bin/yaf
-    yafscii: /usr/local/bin/yafscii
-    yaf_log: /var/log/yaf.log
-    yaf_lock: /var/lock/subsys/yaf
-    kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh
-
 - name: Check for Java at "{{ java_home }}"
   stat: path="{{ java_home }}"
   register: jdk_dir
@@ -69,6 +62,6 @@
   when: not jdk_dir.stat.exists
 
 - name: Start yaf
-  shell: "daemonize -c {{ yaf_home }} -e {{ yaf_log }} -o {{ yaf_log }} -l {{ yaf_lock }} {{ yaf }} --in {{ sniff_interface }} --live pcap | {{ yafscii }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}"
+  shell: "daemonize -c {{ yaf_home }} -e {{ yaf_log }} -o {{ yaf_log }} -l {{ yaf_lock }} {{ yaf }} --in {{ sniff_interface }} --live pcap {{ yaf_args }} | {{ yafscii }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}"
   args:
     creates: "{{ yaf_lock }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/deployment/roles/yaf/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/vars/main.yml b/deployment/roles/yaf/vars/main.yml
deleted file mode 100644
index 1d53958..0000000
--- a/deployment/roles/yaf/vars/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-fixbuf_version: 1.7.1
-yaf_version: 2.8.0
-yaf_home: /opt/yaf
-yaf_topic: ipfix
-hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/pom.xml b/metron-streaming/Metron-Common/pom.xml
index c4fc5aa..605c7ed 100644
--- a/metron-streaming/Metron-Common/pom.xml
+++ b/metron-streaming/Metron-Common/pom.xml
@@ -222,6 +222,11 @@
                 <version>1.4</version>
                 <configuration>
                     <createDependencyReducedPom>true</createDependencyReducedPom>
+                    <artifactSet>
+                        <excludes>
+                            <exclude>*slf4j*</exclude>
+                        </excludes>
+                    </artifactSet>
                 </configuration>
                 <executions>
                     <execution>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
index 6d094ee..a8fda69 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/BulkMessageWriterBolt.java
@@ -66,6 +66,7 @@ public class BulkMessageWriterBolt extends ConfiguredBolt {
   @Override
   public void execute(Tuple tuple) {
     JSONObject message = (JSONObject) tuple.getValueByField("message");
+    message.put("index." + bulkMessageWriter.getClass().getSimpleName().toLowerCase() + ".ts", "" + System.currentTimeMillis());
     String sourceType = TopologyUtils.getSourceType(message);
     SourceConfig configuration = configurations.get(sourceType);
     int batchSize = configuration != null ? configuration.getBatchSize() : 1;
@@ -80,7 +81,9 @@ public class BulkMessageWriterBolt extends ConfiguredBolt {
       sourceMessageMap.put(sourceType, messageList);
     } else {
       try {
-        bulkMessageWriter.write(sourceType, configuration, tupleList, messageList);
+
+        String esType = sourceType + "_doc";
+        bulkMessageWriter.write(esType, configuration, tupleList, messageList);
         for(Tuple t: tupleList) {
           collector.ack(t);
         }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
index dac1c0a..653eade 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/bolt/JoinBolt.java
@@ -27,6 +27,7 @@ import com.google.common.cache.CacheBuilder;
 import com.google.common.cache.CacheLoader;
 import com.google.common.cache.LoadingCache;
 import com.google.common.collect.Sets;
+import org.json.simple.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import java.util.HashMap;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/2e9f2c6c/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
index 7079d5c..6f43739 100644
--- a/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
+++ b/metron-streaming/Metron-Common/src/main/java/org/apache/metron/domain/Enrichment.java
@@ -20,10 +20,12 @@ package org.apache.metron.domain;
 import org.apache.metron.enrichment.interfaces.EnrichmentAdapter;
 
 import java.io.Serializable;
+import java.util.List;
 
 public class Enrichment<T extends EnrichmentAdapter> implements Serializable {
 
   private String type;
+  private List<String> fields;
   private T adapter;
 
   public Enrichment() {}
@@ -33,6 +35,15 @@ public class Enrichment<T extends EnrichmentAdapter> implements Serializable {
     this.adapter = adapter;
   }
 
+
+  public List<String> getFields() {
+    return fields;
+  }
+
+  public void setFields(List<String> fields) {
+    this.fields = fields;
+  }
+
   public String getType() {
     return type;
   }



Mime
View raw message