metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From l...@apache.org
Subject [3/3] incubator-metron git commit: METRON-283 Migrate Geo Enrichment outside of MySQL (justinleet) closes apache/incubator-metron#421
Date Thu, 26 Jan 2017 16:21:56 GMT
METRON-283 Migrate Geo Enrichment outside of MySQL (justinleet) closes apache/incubator-metron#421


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/fa6f3df5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/fa6f3df5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/fa6f3df5

Branch: refs/heads/master
Commit: fa6f3df51b881e84aa7a9525f24c627fa28ca62e
Parents: c74c7de
Author: justinleet <justinjleet@gmail.com>
Authored: Thu Jan 26 11:19:44 2017 -0500
Committer: leet <leet@apache.org>
Committed: Thu Jan 26 11:19:44 2017 -0500

----------------------------------------------------------------------
 LICENSE                                         |  17 ++
 dependencies_with_url.csv                       |   4 +-
 metron-deployment/README.md                     |   7 +-
 metron-deployment/amazon-ec2/conf/defaults.yml  |   1 -
 metron-deployment/amazon-ec2/playbook.yml       |   2 +-
 .../inventory/devimage-vagrant/hosts            |   4 -
 .../inventory/full-dev-platform/group_vars/all  |   1 -
 .../inventory/full-dev-platform/hosts           |   4 -
 .../inventory/metron_example/group_vars/all     |   1 -
 .../inventory/metron_example/hosts              |   5 -
 .../inventory/quick-dev-platform/group_vars/all |   1 -
 .../inventory/quick-dev-platform/hosts          |   4 -
 .../METRON/CURRENT/role_command_order.json      |   3 +-
 .../METRON/CURRENT/configuration/metron-env.xml |  64 +----
 .../common-services/METRON/CURRENT/metainfo.xml |  50 ----
 .../CURRENT/package/files/addMysqlUser.sh       |  51 ----
 .../CURRENT/package/files/createMysqlGeoIp.sh   |  53 -----
 .../CURRENT/package/files/removeMysqlUser.sh    |  43 ----
 .../package/scripts/enrichment_commands.py      |  35 ++-
 .../package/scripts/enrichment_master.py        |   4 +
 .../CURRENT/package/scripts/mysql_server.py     |  63 -----
 .../CURRENT/package/scripts/mysql_service.py    |  46 ----
 .../CURRENT/package/scripts/mysql_users.py      |  58 -----
 .../CURRENT/package/scripts/mysql_utils.py      |  56 -----
 .../package/scripts/params/params_linux.py      |  30 +--
 .../package/scripts/params/status_params.py     |   8 +-
 .../package/templates/enrichment.properties.j2  |   7 -
 .../METRON/CURRENT/service_advisor.py           |  10 -
 .../METRON/CURRENT/themes/metron_theme.json     |  69 +-----
 .../docker/rpm-docker/SPECS/metron.spec         |   5 +-
 .../packer-build/metron-centos-6.7.json         |   2 +-
 metron-deployment/playbooks/metron_install.yml  |  17 --
 .../roles/metron_streaming/defaults/main.yml    |   1 +
 .../roles/metron_streaming/tasks/geoip.yml      |  22 ++
 .../metron_streaming/tasks/hdfs_filesystem.yml  |  15 ++
 .../roles/metron_streaming/tasks/main.yml       |   3 +
 .../roles/metron_streaming/tasks/topologies.yml |   6 -
 metron-deployment/roles/monit/defaults/main.yml |   1 -
 .../roles/monit/tasks/monit-definitions.yml     |   4 -
 .../roles/monit/templates/monit/mysql.monit     |  21 --
 .../roles/mysql_client/tasks/main.yml           |  35 ---
 .../roles/mysql_client/templates/db_config.sql  |  21 --
 .../roles/mysql_server/defaults/main.yml        |  21 --
 .../roles/mysql_server/files/geoip_ddl.sql      |  49 ----
 .../roles/mysql_server/handlers/main.yml        |  19 --
 .../roles/mysql_server/meta/main.yml            |  21 --
 .../roles/mysql_server/tasks/geoip.yml          |  37 ---
 .../roles/mysql_server/tasks/main.yml           |  20 --
 .../roles/mysql_server/tasks/mysql-password.yml |  33 ---
 .../roles/mysql_server/tasks/mysql.yml          |  60 -----
 .../roles/mysql_server/templates/.my.cnf        |  20 --
 metron-deployment/roles/snort/files/snort.conf  |   2 -
 metron-docker/.gitignore                        |   1 -
 metron-docker/README.md                         |   2 -
 metron-docker/compose/docker-compose.yml        |  10 -
 metron-docker/compose/mysql/Dockerfile          |  37 ---
 metron-docker/compose/mysql/bin/init-mysql.sh   |  19 --
 metron-docker/compose/mysql/bin/start.sh        |  21 --
 metron-docker/compose/storm/Dockerfile          |   5 +-
 metron-docker/install-metron.sh                 |   3 -
 metron-docker/pom.xml                           |  36 ---
 metron-platform/metron-common/README.md         |   8 +
 .../common/bolt/ConfiguredParserBolt.java       |   2 +-
 .../common/configuration/Configurations.java    |   3 +-
 .../org/apache/metron/common/dsl/Context.java   |   7 +-
 .../metron-data-management/README.md            |  15 ++
 metron-platform/metron-data-management/pom.xml  |   2 +-
 .../nonbulk/geo/GeoEnrichmentLoader.java        | 233 +++++++++++++++++++
 .../src/main/scripts/geo_enrichment_load.sh     |  41 ++++
 .../nonbulk/geo/GeoEnrichmentLoaderTest.java    |  91 ++++++++
 metron-platform/metron-enrichment/pom.xml       |  18 +-
 .../src/main/config/enrichment.properties       |   7 -
 .../src/main/flux/enrichment/remote.yaml        |  17 --
 .../src/main/flux/enrichment/test.yaml          |   2 +-
 .../adapters/cif/AbstractCIFAdapter.java        |   3 +-
 .../adapters/cif/CIFHbaseAdapter.java           |   6 +-
 .../enrichment/adapters/geo/GeoAdapter.java     |  81 +++----
 .../adapters/geo/GeoLiteDatabase.java           | 184 +++++++++++++++
 .../adapters/host/AbstractHostAdapter.java      |   3 +-
 .../adapters/host/HostFromJSONListAdapter.java  |   6 +-
 .../host/HostFromPropertiesFileAdapter.java     |   6 +-
 .../enrichment/adapters/jdbc/JdbcAdapter.java   |   5 +-
 .../simplehbase/SimpleHBaseAdapter.java         |  10 +-
 .../adapters/stellar/StellarAdapter.java        |   6 +-
 .../threatintel/ThreatIntelAdapter.java         |  11 +-
 .../enrichment/bolt/GenericEnrichmentBolt.java  |  14 +-
 .../enrichment/bolt/ThreatIntelJoinBolt.java    |  12 +
 .../interfaces/EnrichmentAdapter.java           |   6 +-
 .../stellar/GeoEnrichmentFunctions.java         | 110 +++++++++
 .../src/main/resources/ddl/geoip_ddl.sql        |  49 ----
 .../enrichment/adapters/geo/GeoAdapterTest.java |  96 +++-----
 .../adapters/geo/GeoLiteDatabaseTest.java       | 181 ++++++++++++++
 .../host/HostFromJSONListAdapterTest.java       |   2 +-
 .../host/HostFromPropertiesFileAdapterTest.java |   4 +-
 .../simplehbase/SimpleHBaseAdapterTest.java     |   2 +-
 .../threatintel/ThreatIntelAdapterTest.java     |   2 +-
 .../bolt/GenericEnrichmentBoltTest.java         |  17 +-
 .../bolt/ThreatIntelJoinBoltTest.java           |   9 +
 .../integration/EnrichmentIntegrationTest.java  |  71 +++---
 .../integration/mock/MockGeoAdapter.java        |  72 ------
 .../stellar/GeoEnrichmentFunctionsTest.java     | 150 ++++++++++++
 .../GeoLite/GeoIP2-City-Test-2.mmdb.gz          | Bin 0 -> 10011 bytes
 .../resources/GeoLite/GeoIP2-City-Test.mmdb.gz  | Bin 0 -> 10011 bytes
 .../resources/GeoMysqlAdapterTest.properties    |  27 ---
 .../resources/TestSchemas/GeoMySqlSchema.json   |  42 ----
 metron-platform/metron-hbase/pom.xml            |   1 -
 .../src/main/config/zookeeper/global.json       |   4 +-
 .../apache/metron/parsers/bolt/ParserBolt.java  |   3 +
 .../metron/parsers/bolt/ParserBoltTest.java     |  39 +++-
 .../integration/WriterBoltIntegrationTest.java  |   1 +
 .../resources/GeoLite/GeoIP2-City-Test.mmdb.gz  | Bin 0 -> 10011 bytes
 111 files changed, 1393 insertions(+), 1558 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 9d2a8df..27aaf94 100644
--- a/LICENSE
+++ b/LICENSE
@@ -210,6 +210,23 @@ This product bundles some test examples from the Stix project (metron-platform/m
 
 This product bundles wait-for-it.sh, which is available under a "MIT Software License" license.  For details, see https://github.com/vishnubob/wait-for-it
 
+------------------------------------------------------------------------------------
+
+------------------------------------------------------------------------------------
+  Creative Commons Attribution-ShareAlike 4.0 International License
+------------------------------------------------------------------------------------
+
+This product includes GeoLite2 data created by MaxMind, available from
+<a href="http://www.maxmind.com">http://www.maxmind.com</a>.
+
+------------------------------------------------------------------------------------
+  Creative Commons Attribution-ShareAlike 3.0 License
+------------------------------------------------------------------------------------
+
+This product includes a MaxMind DB created by MaxMind, available from
+<a href="https://github.com/maxmind/MaxMind-DB">https://github.com/maxmind/MaxMind-DB</a>.
+
+------------------------------------------------------------------------------------
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/dependencies_with_url.csv
----------------------------------------------------------------------
diff --git a/dependencies_with_url.csv b/dependencies_with_url.csv
index 67b5373..ca8fe12 100644
--- a/dependencies_with_url.csv
+++ b/dependencies_with_url.csv
@@ -18,6 +18,8 @@ com.esotericsoftware:minlog:jar:1.3.0:compile,New BSD License,http://code.google
 com.esotericsoftware:reflectasm:jar:1.10.1:compile,New BSD License,http://code.google.com/p/minlog/
 com.google.protobuf:protobuf-java:jar:2.5.0:compile,New BSD license,http://code.google.com/p/protobuf
 com.jcraft:jsch:jar:0.1.42:compile,BSD,http://www.jcraft.com/jsch/
+com.maxmind.db:maxmind-db:jar:1.2.1:compile,CC-BY-SA 3.0,https://github.com/maxmind/MaxMind-DB
+com.maxmind.geoip2:geoip2:jar:2.8.0:compile,Apache v2,https://github.com/maxmind/GeoIP2-java
 com.sun.xml.bind:jaxb-impl:jar:2.2.3-1:compile,CDDL,http://jaxb.java.net/
 com.sun.xml.bind:jaxb-impl:jar:2.2.5-2:compile,CDDL,http://jaxb.java.net/
 com.twitter:jsr166e:jar:1.1.0:compile,CC0 1.0 Universal,http://github.com/twitter/jsr166e
@@ -32,7 +34,6 @@ javax.xml.stream:stax-api:jar:1.0-2:compile,COMMON DEVELOPMENT AND DISTRIBUTION
 jline:jline:jar:0.9.94:compile,BSD,http://jline.sourceforge.net
 junit:junit:jar:4.12:compile,Eclipse Public License 1.0,http://junit.org
 junit:junit:jar:4.4:compile,Common Public License Version 1.0,http://junit.org
-mysql:mysql-connector-java:jar:5.1.31:compile,ACCEPTABLE,http://dev.mysql.com/usingmysql/java/
 net.sf.jopt-simple:jopt-simple:jar:3.2:compile,The MIT License,http://jopt-simple.sourceforge.net
 net.sf.jopt-simple:jopt-simple:jar:4.9:compile,The MIT License,http://jopt-simple.sourceforge.net
 net.sf.saxon:Saxon-HE:jar:9.5.1-5:compile,Mozilla Public License Version 2.0,http://www.saxonica.com/
@@ -218,3 +219,4 @@ org.yaml:snakeyaml:jar:1.15:compile,Apache License Version 2.0,http://www.snakey
 ring-cors:ring-cors:jar:0.1.5:compile,Eclipse Public License 1.0,https://github.com/r0man/ring-cors
 xerces:xercesImpl:jar:2.9.1:compile,ASLv2,http://xerces.apache.org/xerces2-j
 xml-apis:xml-apis:jar:1.3.04:compile,ASLv2,http://xml.apache.org/commons/components/external/
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/README.md b/metron-deployment/README.md
index 2abf363..120f835 100644
--- a/metron-deployment/README.md
+++ b/metron-deployment/README.md
@@ -100,7 +100,7 @@ This will set up
 - Metron Parsers
 - Enrichment
 - Indexing
-- GeoIP database on MySQL
+- GeoIP data
 - Optional Elasticsearch
 - Optional Kibana
 
@@ -134,15 +134,13 @@ Notably, the URL for the GeoIP database that is preloaded (and is prefilled by d
 After installation, a custom action is available in Ambari (where stop / start services are) to install Elasticsearch templates.  Similar to this, a custom Kibana action to Load Template is available.
 
 #### Offline installation
-Currently there is only one point that would reach out to the internet during an install.  This is the URL for the GeoIP database information that is preloaded into MySQL.
+Currently there is only one point that would reach out to the internet during an install.  This is the URL for the GeoIP database information.
 
 The RPMs DO NOT reach out to the internet (because there is currently no hosting for them).  They look on the local filesystem in `/localrepo`.
 
 ### Current Limitations
 There are a set of limitations that should be addressed based to improve the current state of the mpacks.
 
-- MySQL install should be optional (and allow for using an existing instance).
-- MySQL should not be installed on a node already running a MySQL instance (e.g. an Ambari Server using MySQL as its database).
 - There is currently no hosting for RPMs remotely.  They will have to be built locally.
 - Colocation of appropriate services should be enforced by Ambari.  See [#Installing Management Pack] for more details.
 - Storm's topology.classpath is not updated with the Metron service install and needs to be updated separately.
@@ -177,6 +175,5 @@ rpm -i <package>
 ```
 
 ## TODO
-- migrate existing MySQL/GeoLite playbook
 - Support Ubuntu deployments
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index f4baa5a..d3e84ec 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -17,7 +17,6 @@
 ---
 # which services should be started?
 services_to_start:
-  - mysql
   - elasticsearch
   - pcap-service
   - kibana

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml
index 1204b52..a0b15dc 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -31,7 +31,7 @@
     - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,metron_kafka_topics,metron_hbase_tables,metron,ec2,pcap_server
     - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2
     - include: tasks/create-hosts.yml host_count=2 host_type=search,metron,ec2
-    - include: tasks/create-hosts.yml host_count=1 host_type=web,mysql,metron,ec2
+    - include: tasks/create-hosts.yml host_count=1 host_type=web,metron,ec2
   tags:
     - ec2
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/inventory/devimage-vagrant/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/devimage-vagrant/hosts b/metron-deployment/inventory/devimage-vagrant/hosts
index 36ec7fa..98e8681 100644
--- a/metron-deployment/inventory/devimage-vagrant/hosts
+++ b/metron-deployment/inventory/devimage-vagrant/hosts
@@ -39,9 +39,6 @@ node1
 [sensors]
 node1
 
-[mysql]
-node1
-
 [pcap_server]
 node1
 
@@ -50,7 +47,6 @@ enrichment
 search
 web
 sensors
-mysql
 metron_kafka_topics
 metron_hbase_tables
 pcap_server

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 2d07879..2f644af 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -17,7 +17,6 @@
 
 # which services should be started?
 services_to_start:
-  - mysql
   - elasticsearch
   - pcap-service
   - kibana

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/inventory/full-dev-platform/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/hosts b/metron-deployment/inventory/full-dev-platform/hosts
index 7dacc20..98e8681 100644
--- a/metron-deployment/inventory/full-dev-platform/hosts
+++ b/metron-deployment/inventory/full-dev-platform/hosts
@@ -42,15 +42,11 @@ node1
 [pcap_server]
 node1
 
-[mysql]
-node1
-
 [metron:children]
 enrichment
 search
 web
 sensors
-mysql
 metron_kafka_topics
 metron_hbase_tables
 pcap_server

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/group_vars/all b/metron-deployment/inventory/metron_example/group_vars/all
index 38f1440..d6a31f3 100644
--- a/metron-deployment/inventory/metron_example/group_vars/all
+++ b/metron-deployment/inventory/metron_example/group_vars/all
@@ -17,7 +17,6 @@
 ---
 # which services should be started?
 services_to_start:
-  - mysql
   - elasticsearch
   - pcap-service
   - kibana

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/inventory/metron_example/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/metron_example/hosts b/metron-deployment/inventory/metron_example/hosts
index d2d540f..c53466d 100644
--- a/metron-deployment/inventory/metron_example/hosts
+++ b/metron-deployment/inventory/metron_example/hosts
@@ -51,18 +51,13 @@ node12
 [sensors]
 node1
 
-#same as mysql in 12 node topology
 [web]
 node12
 
-[mysql]
-node12
-
 [metron:children]
 enrichment
 search
 web
 sensors
-mysql
 metron_kafka_topics
 metron_hbase_tables

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/inventory/quick-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/quick-dev-platform/group_vars/all b/metron-deployment/inventory/quick-dev-platform/group_vars/all
index e27e528..0153a0d 100644
--- a/metron-deployment/inventory/quick-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/quick-dev-platform/group_vars/all
@@ -17,7 +17,6 @@
 
 # which services should be started?
 services_to_start:
-  - mysql
   - elasticsearch
   - pcap-service
   - kibana

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/inventory/quick-dev-platform/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/quick-dev-platform/hosts b/metron-deployment/inventory/quick-dev-platform/hosts
index 7dacc20..98e8681 100644
--- a/metron-deployment/inventory/quick-dev-platform/hosts
+++ b/metron-deployment/inventory/quick-dev-platform/hosts
@@ -42,15 +42,11 @@ node1
 [pcap_server]
 node1
 
-[mysql]
-node1
-
 [metron:children]
 enrichment
 search
 web
 sensors
-mysql
 metron_kafka_topics
 metron_hbase_tables
 pcap_server

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
index 0f4cd39..e08f401 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/role_command_order.json
@@ -5,8 +5,7 @@
         "_comment" : "dependencies for all cases",
         "METRON_INDEXING-INSTALL" : ["METRON_PARSERS-INSTALL"],
         "METRON_PARSERS-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START"],
-        "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "ENRICHMENT_MYSQL_SERVER-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
-        "METRON_ENRICHMENT_MYSQL_SERVER-STOP" : ["METRON_ENRICHMENT_MASTER-STOP"],
+        "METRON_ENRICHMENT_MASTER-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START", "HBASE_MASTER-START", "HBASE_REGIONSERVER-START"],
         "METRON_ENRICHMENT_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_ENRICHMENT_MASTER-START"],
         "METRON_INDEXING-START" : ["NAMENODE-START", "ZOOKEEPER_SERVER-START", "KAFKA_BROKER-START", "STORM_REST_API-START","METRON_PARSERS-START"],
         "METRON_SERVICE_CHECK-SERVICE_CHECK" : ["METRON_PARSERS-START","METRON_INDEXING-START"]

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 7484d31..6015965 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -69,42 +69,6 @@
         <display-name>Metron Parsers</display-name>
     </property>
     <property>
-        <name>metron_enrichment_db_user</name>
-        <value>metron</value>
-        <description>Database username to use to connect to the database.</description>
-        <display-name>Enrichment Database User</display-name>
-    </property>
-    <property>
-        <name>metron_enrichment_db_port</name>
-        <display-name>Metron Enrichment Database Port</display-name>
-        <value>3306</value>
-        <description>Database port to use to connect to the database.</description>
-    </property>
-    <property>
-        <name>metron_enrichment_db_password</name>
-        <value></value>
-        <property-type>PASSWORD</property-type>
-        <display-name>Metron Enrichment Database Password</display-name>
-        <description>Password to use against database</description>
-        <value-attributes>
-            <type>password</type>
-            <overridable>false</overridable>
-        </value-attributes>
-        <on-ambari-upgrade add="true"/>
-    </property>
-    <property>
-        <name>mysql_admin_password</name>
-        <value></value>
-        <property-type>PASSWORD</property-type>
-        <display-name>MySQL root user password</display-name>
-        <description>Password to use to add Metron user to MySQL</description>
-        <value-attributes>
-            <type>password</type>
-            <overridable>false</overridable>
-        </value-attributes>
-        <on-ambari-upgrade add="true"/>
-    </property>
-    <property>
         <name>metron_indexing_topology</name>
         <value>indexing</value>
         <description>The Storm topology name for Indexing</description>
@@ -118,7 +82,7 @@
     </property>
     <property>
         <name>geoip_url</name>
-        <value>http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz</value>
+        <value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value>
         <description>Location of the GeoIP data to load.</description>
         <display-name>GEOIP Load Datafile URL</display-name>
     </property>
@@ -173,32 +137,6 @@
         <value></value>
     </property>
     <property>
-        <name>install_mysql</name>
-        <description>Install New MySQL Instance for Enrichments</description>
-        <display-name>Install MySQL</display-name>
-        <value>Yes</value>
-        <value-attributes>
-            <overridable>false</overridable>
-            <type>value-list</type>
-            <entries>
-                <entry>
-                    <value>Yes</value>
-                    <label>Yes</label>
-                </entry>
-                <entry>
-                    <value>No</value>
-                    <label>No</label>
-                </entry>
-            </entries>
-            <selection-cardinality>1</selection-cardinality>
-        </value-attributes>
-    </property>
-    <property>
-        <name>mysql_host</name>
-        <display-name>MySQL Host Address</display-name>
-        <value></value>
-    </property>
-    <property>
         <name>global-json</name>
         <display-name>global.json template</display-name>
         <description>This is the jinja template for global.json file</description>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 2677f60..3c71328 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -113,29 +113,6 @@
         </component>
 
         <component>
-          <name>METRON_ENRICHMENT_MYSQL_SERVER</name>
-          <displayName>MySQL Server</displayName>
-          <category>MASTER</category>
-          <cardinality>1</cardinality>
-          <versionAdvertised>false</versionAdvertised>
-          <clientsToUpdateConfigs></clientsToUpdateConfigs>
-          <commandScript>
-            <script>scripts/mysql_server.py</script>
-            <scriptType>PYTHON</scriptType>
-          </commandScript>
-          <customCommands>
-            <customCommand>
-              <name>CLEAN</name>
-              <commandScript>
-                <script>scripts/mysql_server.py</script>
-                <scriptType>PYTHON</scriptType>
-                <timeout>600</timeout>
-              </commandScript>
-            </customCommand>
-          </customCommands>
-        </component>
-
-        <component>
           <name>METRON_INDEXING</name>
           <displayName>Metron Indexing</displayName>
           <category>MASTER</category>
@@ -234,33 +211,6 @@
             </package>
           </packages>
         </osSpecific>
-        <osSpecific>
-          <osFamily>redhat7</osFamily>
-          <packages>
-            <package>
-              <name>mysql-community-release</name>
-              <skipUpgrade>true</skipUpgrade>
-            </package>
-            <package>
-              <name>mysql-community-server</name>
-              <skipUpgrade>true</skipUpgrade>
-            </package>
-            <package>MySQL-python</package>
-          </packages>
-        </osSpecific>
-        <osSpecific>
-          <osFamily>redhat6</osFamily>
-          <packages>
-            <package>
-              <name>mysql-server</name>
-              <skipUpgrade>true</skipUpgrade>
-            </package>
-            <package>
-              <name>mysql</name>
-              <skipUpgrade>true</skipUpgrade>
-            </package>
-          </packages>
-        </osSpecific>
       </osSpecifics>
       <commandScript>
         <script>scripts/service_check.py</script>

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh
deleted file mode 100755
index 8a6a788..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/addMysqlUser.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-mysqldbuser=$1
-mysqldbpasswd=$2
-mysqldbhost=$3
-mysqlqdminpassword=$4
-myhostname=$(hostname -f)
-
-
-echo "Adding user ${mysqldbuser}@${mysqldbhost} and ${mysqldbuser}@localhost"
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-expect "mysql>"
-send "CREATE USER '${mysqldbuser}'@'${mysqldbhost}' IDENTIFIED BY '${mysqldbpasswd}';\r"
-expect "mysql>"
-send "CREATE USER '${mysqldbuser}'@'localhost' IDENTIFIED BY '${mysqldbpasswd}';\r"
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'%' IDENTIFIED BY '${mysqldbpasswd}';\r"
-log_user 1
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'${mysqldbhost}';\r"
-expect "mysql>"
-send "GRANT ALL PRIVILEGES ON GEO.* TO '${mysqldbuser}'@'localhost';\r"
-expect "mysql>"
-send "flush privileges;\r"
-send "\q"
-EOF

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh
deleted file mode 100755
index 40f11e1..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/createMysqlGeoIp.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/sh
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-from ../scripts/params import params
-
-geoipscript=$1
-geoipurl=$2
-mysqlqdminpassword=$3
-
-# Download and extract the actual GeoIP files
-mkdir -p /tmp/geoip
-mkdir -p /var/lib/mysql-files/
-
-# Stage the GeoIP data
-pushd /tmp/geoip
-curl -O ${geoipurl}
-tar xf GeoLiteCity-latest.tar.xz
-cp /tmp/geoip/*/*.csv /var/lib/mysql-files/
-popd
-
-# Load MySQL with the GeoIP data
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-log_user 1
-expect "mysql>"
-send "source ${geoipscript}\r"
-expect "mysql>"
-send "show databases;\r"
-send "\q"
-EOF

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh
deleted file mode 100755
index 3fc7afb..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/removeMysqlUser.sh
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-
-mysqldbuser=$1
-userhost=$2
-mysqlqdminpassword=$3
-myhostname=$(hostname -f)
-sudo_prefix="/var/lib/ambari-agent/ambari-sudo.sh -H -E"
-
-echo "Removing user $mysqldbuser@$userhost"
-expect <<EOF
-log_user 0
-# start mysql process using password prompt
-spawn mysql -u root -p
-expect "password:"
-send "${mysqlqdminpassword}\r"
-# echo all output until the end
-log_user 1
-expect "mysql>"
-send "DROP USER '${mysqldbuser}'@'${userhost}';\r"
-expect "mysql>"
-send "flush privileges;;\r"
-send "\q"
-EOF
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index cb9cf22..bc73c87 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -17,7 +17,6 @@ limitations under the License.
 
 import os
 import time
-
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute, File
 
@@ -74,15 +73,39 @@ class EnrichmentCommands:
             yum_repo_types[repo_type]()
             Logger.info("Writing out repo file")
             repo_template = ("echo \"[METRON-${metron.version}]\n"
-                            "name=Metron ${metron.version} packages\n"
-                            "baseurl={0}\n"
-                            "gpgcheck=0\n"
-                            "enabled=1\n\""
-                         "   > /etc/yum.repos.d/metron.repo")
+                             "name=Metron ${metron.version} packages\n"
+                             "baseurl={0}\n"
+                             "gpgcheck=0\n"
+                             "enabled=1\n\""
+                             "   > /etc/yum.repos.d/metron.repo")
             Execute(repo_template.format(self.__params.repo_url))
         else:
             raise ValueError("Unsupported repo type '{0}'".format(repo_type))
 
+    def init_geo(self):
+        Logger.info("Creating HDFS location for GeoIP database")
+        self.__params.HdfsResource(self.__params.geoip_hdfs_dir,
+                                   type="directory",
+                                   action="create_on_execute",
+                                   owner=self.__params.metron_user,
+                                   group=self.__params.user_group,
+                                   mode=0775,
+                                   )
+
+        Logger.info("Creating and loading GeoIp database")
+        command_template = """{0}/bin/geo_enrichment_load.sh \
+                                -g {1} \
+                                -r {2} \
+                                -z {3}"""
+        command = command_template.format(self.__params.metron_home,
+                                          self.__params.geoip_url,
+                                          self.__params.geoip_hdfs_dir,
+                                          self.__params.zookeeper_quorum
+                                          )
+        Logger.info("Executing command " + command)
+        Execute(command, user=self.__params.metron_user, tries=1, logoutput=True)
+        Logger.info("Done intializing GeoIP data")
+
     def init_kafka_topics(self):
         Logger.info('Creating Kafka topics')
         command_template = """{0}/kafka-topics.sh \

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index d9da4f6..b02ba5c 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -22,6 +22,7 @@ from resource_management.libraries.functions.format import format
 from resource_management.libraries.script import Script
 
 from enrichment_commands import EnrichmentCommands
+import metron_service
 
 
 class Enrichment(Script):
@@ -48,9 +49,12 @@ class Enrichment(Script):
         env.set_params(params)
         commands = EnrichmentCommands(params)
 
+        metron_service.load_global_config(params)
+
         if not commands.is_configured():
             commands.init_kafka_topics()
             commands.create_hbase_tables()
+            commands.init_geo()
             commands.set_configured()
 
         commands.start_enrichment_topology()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py
deleted file mode 100755
index fcc83af..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_server.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.script.script import Script
-
-import mysql_users
-from mysql_service import mysql_service
-from mysql_utils import mysql_configure
-from resource_management.core.resources.packaging import Package
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-class MysqlServer(Script):
-    def install(self, env):
-        self.install_packages(env)
-        self.configure(env)
-
-    def clean(self, env):
-        from params import params
-        env.set_params(params)
-        if params.install_mysql == 'Yes':
-            mysql_users.mysql_deluser()
-
-    def configure(self, env, upgrade_type=None, config_dir=None):
-        from params import params
-        env.set_params(params)
-        mysql_configure()
-
-    def start(self, env, rolling_restart=False):
-        from params import params
-        env.set_params(params)
-        mysql_service(daemon_name=params.daemon_name, action='start')
-
-    def stop(self, env, rolling_restart=False):
-        from params import params
-        env.set_params(params)
-        mysql_service(daemon_name=params.daemon_name, action='stop')
-
-    def status(self, env):
-        from params import status_params
-        env.set_params(status_params)
-
-        mysql_service(daemon_name=status_params.daemon_name, action='status')
-
-
-if __name__ == "__main__":
-    MysqlServer().execute()

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py
deleted file mode 100755
index 2e0ce8b..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_service.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.exceptions import ComponentIsNotRunning, Fail
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions.format import format
-
-
-def mysql_service(daemon_name=None, action='start'):
-    status_cmd = format("pgrep -l '^{mysql_process_name}$'")
-    cmd = ('service', daemon_name, action)
-
-    if action == 'status':
-        try:
-            Execute(status_cmd)
-        except Fail:
-            raise ComponentIsNotRunning()
-    elif action == 'stop':
-        Execute(cmd,
-                logoutput=True,
-                only_if=status_cmd,
-                sudo=True,
-                )
-    elif action == 'start':
-        Execute(cmd,
-                logoutput=True,
-                not_if=status_cmd,
-                sudo=True,
-                )

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py
deleted file mode 100755
index b778e85..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_users.py
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-
-
-# Used to add metron access to the needed components
-def mysql_adduser():
-    from params import params
-
-    File(params.mysql_adduser_path,
-         mode=0755,
-         content=StaticFile('addMysqlUser.sh')
-         )
-
-    add_user_cmd = format("bash -x {mysql_adduser_path} {metron_user} {enrichment_metron_user_passwd!p} {mysql_host} {mysql_admin_password!p}")
-    Execute(add_user_cmd,
-            tries=3,
-            try_sleep=5,
-            logoutput=False,
-            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'
-            )
-
-
-# Removes hive metron from components
-def mysql_deluser():
-    from params import params
-
-    File(params.mysql_deluser_path,
-         mode=0755,
-         content=StaticFile('removeMysqlUser.sh')
-         )
-
-    del_user_cmd = format("bash -x {mysql_deluser_path} {metron_user} {mysql_host} {mysql_admin_password!p}")
-    Execute(del_user_cmd,
-            tries=3,
-            try_sleep=5,
-            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-            )

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py
deleted file mode 100755
index 63cdb38..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/mysql_utils.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Execute, File
-from resource_management.core.source import StaticFile
-from resource_management.libraries.functions.format import format
-from mysql_service import mysql_service
-
-import mysql_users
-
-
-def mysql_configure():
-    from params import params
-
-    if params.install_mysql == 'Yes':
-        mysql_service(daemon_name=params.daemon_name, action='start')
-
-    replace_bind_address = ('sed', '-i', 's|^bind-address[ \t]*=.*|bind-address = 0.0.0.0|', params.mysql_configname)
-    Execute(replace_bind_address,
-            sudo=True,
-            )
-
-    mysql_users.mysql_adduser()
-
-    File(params.mysql_create_geoip_path,
-         mode=0755,
-         content=StaticFile('createMysqlGeoIp.sh')
-         )
-
-    geoip_setup_cmd = format("bash -x {mysql_create_geoip_path} {geoip_ddl} {geoip_url} {mysql_admin_password!p}")
-
-    Execute(geoip_setup_cmd,
-            tries=3,
-            try_sleep=5,
-            path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
-            )
-
-    if params.install_mysql == 'Yes':
-        mysql_service(daemon_name=params.daemon_name, action='stop')

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 6b3052d..4e00971 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -42,9 +42,8 @@ hostname = config['hostname']
 user_group = config['configurations']['cluster-env']['user_group']
 metron_home = status_params.metron_home
 parsers = status_params.parsers
-metron_ddl_dir = metron_home + '/ddl'
-geoip_ddl = metron_ddl_dir + '/geoip_ddl.sql'
 geoip_url = config['configurations']['metron-env']['geoip_url']
+geoip_hdfs_dir = "/apps/metron/geo/default/"
 metron_indexing_topology = status_params.metron_indexing_topology
 metron_user = config['configurations']['metron-env']['metron_user']
 metron_group = config['configurations']['metron-env']['metron_group']
@@ -110,6 +109,7 @@ if has_kafka_host:
     kafka_brokers += ':' + kafka_broker_port
 
 metron_apps_hdfs_dir = config['configurations']['metron-env']['metron_apps_hdfs_dir']
+
 # the double "format" is not an error - we are pulling in a jinja-templated param. This is a bit of a hack, but works
 # well enough until we find a better way via Ambari
 metron_apps_indexed_hdfs_dir = format(format(config['configurations']['metron-env']['metron_apps_indexed_hdfs_dir']))
@@ -129,32 +129,6 @@ hdfs_site = config['configurations']['hdfs-site']
 default_fs = config['configurations']['core-site']['fs.defaultFS']
 dfs_type = default("/commandParams/dfs_type", "")
 
-# MYSQL
-if OSCheck.is_ubuntu_family():
-    mysql_configname = '/etc/mysql/my.cnf'
-else:
-    mysql_configname = '/etc/my.cnf'
-
-daemon_name = status_params.daemon_name
-
-install_mysql = config['configurations']['metron-env']['install_mysql']
-mysql_admin_password = config['configurations']['metron-env']['mysql_admin_password']
-
-# There will always be exactly one mysql_host
-mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][0]
-
-mysql_port = config['configurations']['metron-env']['metron_enrichment_db_port']
-
-mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
-mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
-mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
-
-enrichment_metron_user = config['configurations']['metron-env']['metron_enrichment_db_user']
-enrichment_metron_user_passwd = config['configurations']['metron-env']['metron_enrichment_db_password']
-enrichment_metron_user_passwd = unicode(enrichment_metron_user_passwd) if not is_empty(
-    enrichment_metron_user_passwd) else enrichment_metron_user_passwd
-mysql_process_name = status_params.mysql_process_name
-
 # create partial functions with common arguments for every HdfsResource call
 # to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
 HdfsResource = functools.partial(

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index 92c9d06..4880247 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -42,13 +42,7 @@ enrichment_cf = 't'
 threatintel_table = 'threatintel'
 threatintel_cf = 't'
 
-mysql_process_name = 'mysqld'
-if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family():
-    daemon_name = 'mysql'
-else:
-    daemon_name = 'mysqld'
-
-# ing
+# Indexing
 metron_indexing_topology = config['configurations']['metron-env']['metron_indexing_topology']
 indexing_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_configured'
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2
index 14403de..dc108f7 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment.properties.j2
@@ -23,13 +23,6 @@ enrichment.output.topic=indexing
 enrichment.error.topic=enrichments_error
 threat.intel.error.topic=threatintel_error
 
-##### MySQL #####
-
-mysql.ip={{mysql_host}}
-mysql.port={{mysql_port}}
-mysql.username={{enrichment_metron_user}}
-mysql.password={{enrichment_metron_user_passwd}}
-
 ##### Metrics #####
 
 #reporters

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
index 72e349b..4c492ab 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/service_advisor.py
@@ -44,7 +44,6 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
         metronParsersHost = self.getHosts(componentsList, "METRON_PARSERS")[0]
         metronEnrichmentMaster = self.getHosts(componentsList, "METRON_ENRICHMENT_MASTER")[0]
         metronIndexingHost = self.getHosts(componentsList, "METRON_INDEXING")[0]
-        metronEnrichmentMysqlServer = self.getHosts(componentsList, "METRON_ENRICHMENT_MYSQL_SERVER")[0]
 
         hbaseClientHosts = self.getHosts(componentsList, "HBASE_CLIENT")
         hdfsClientHosts = self.getHosts(componentsList, "HDFS_CLIENT")
@@ -72,10 +71,6 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
             message = "Metron Indexing must be co-located with Metron Parsers on {0}".format(metronParsersHost)
             items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_INDEXING', "host": metronIndexingHost })
 
-        if metronParsersHost != metronEnrichmentMysqlServer:
-            message = "Metron MySQL Server must be co-located with Metron Parsers on {0}".format(metronParsersHost)
-            items.append({ "type": 'host-component', "level": 'ERROR', "message": message, "component-name": 'METRON_ENRICHMENT_MYSQL_SERVER', "host": metronEnrichmentMysqlServer })
-
         # Enrichment Master also needs ZK Client, but this is already guaranteed by being colocated with Parsers Master
         if metronParsersHost not in zookeeperClientHosts:
             message = "Metron must be co-located with an instance of Zookeeper Client"
@@ -102,11 +97,6 @@ class METRON${metron.short.version}ServiceAdvisor(service_advisor.ServiceAdvisor
         return items
 
     def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
-        #Suggest mysql server hostname
-        mySQLServerHost = self.getComponentHostNames(services, "METRON", "METRON_ENRICHMENT_MYSQL_SERVER")[0]
-        putMetronEnvProperty = self.putProperty(configurations, "metron-env", services)
-        putMetronEnvProperty("mysql_host",mySQLServerHost)
-
         #Suggest Storm Rest URL
         if "storm-site" in services["configurations"]:
             stormUIServerHost = self.getComponentHostNames(services, "STORM", "STORM_UI_SERVER")[0]

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index b939182..1f56070 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -6,7 +6,7 @@
         "tabs": [
           {
             "name": "metron_general",
-            "display-name": "Settings",
+            "display-name": "Index Settings",
             "layout": {
               "tab-columns": "1",
               "tab-rows": "1",
@@ -17,22 +17,13 @@
                   "column-index": "0",
                   "row-span": "1",
                   "column-span": "1",
-                  "section-columns": "2",
+                  "section-columns": "1",
                   "section-rows": "1",
                   "subsections": [
                     {
-                      "name": "subsection-general-database",
-                      "display-name" : "Database Settings",
-                      "row-index": "0",
-                      "column-index": "0",
-                      "row-span": "1",
-                      "column-span": "1"
-                    },
-                    {
                       "name": "subsection-general-indexing",
-                      "display-name": "Index Settings",
                       "row-index": "0",
-                      "column-index": "1",
+                      "column-index": "0",
                       "row-span": "1",
                       "column-span": "1"
                     }
@@ -76,36 +67,6 @@
       "configuration-layout": "default",
       "configs": [
         {
-          "config": "metron-env/install_mysql",
-          "subsection-name": "subsection-general-database"
-        },
-        {
-          "config": "metron-env/mysql_admin_password",
-          "subsection-name": "subsection-general-database",
-          "depends-on": [
-            {
-              "configs":[
-                "metron-env/install_mysql"
-              ],
-              "if": "${metron-env/install_mysql} === No",
-              "then": {
-                "property_value_attributes": {
-                  "visible": true
-                }
-              },
-              "else": {
-                "property_value_attributes": {
-                  "visible": false
-                }
-              }
-            }
-          ]
-        },
-        {
-          "config": "metron-env/metron_enrichment_db_password",
-          "subsection-name": "subsection-general-database"
-        },
-        {
           "config": "metron-env/metron_enrichment_db_port",
           "subsection-name": "subsection-general-database"
         },
@@ -155,30 +116,6 @@
     },
     "widgets": [
       {
-        "config": "metron-env/install_mysql",
-        "widget": {
-          "type": "toggle"
-        }
-      },
-      {
-        "config": "metron-env/mysql_admin_password",
-        "widget": {
-          "type": "password"
-        }
-      },
-      {
-        "config": "metron-env/metron_enrichment_db_password",
-        "widget": {
-          "type": "password"
-        }
-      },
-      {
-        "config": "metron-env/metron_enrichment_db_port",
-        "widget": {
-          "type": "text-field"
-        }
-      },
-      {
         "config": "metron-env/es_hosts",
         "widget": {
           "type": "text-field"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 0d56d7a..8a6607b 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -177,6 +177,7 @@ This package installs the Metron Parser files
 %dir %{metron_home}/bin
 %dir %{metron_home}/lib
 %{metron_home}/bin/Whois_CSV_to_JSON.py
+%{metron_home}/bin/geo_enrichment_load.sh
 %{metron_home}/bin/flatfile_loader.sh
 %{metron_home}/bin/prune_elasticsearch_indices.sh
 %{metron_home}/bin/prune_hdfs_files.sh
@@ -224,7 +225,6 @@ This package installs the Metron Enrichment files
 %dir %{metron_home}/config
 %dir %{metron_home}/config/zookeeper
 %dir %{metron_home}/config/zookeeper/enrichments
-%dir %{metron_home}/ddl
 %dir %{metron_home}/flux
 %dir %{metron_home}/flux/enrichment
 %{metron_home}/bin/latency_summarizer.sh
@@ -235,7 +235,6 @@ This package installs the Metron Enrichment files
 %{metron_home}/config/zookeeper/enrichments/websphere.json
 %{metron_home}/config/zookeeper/enrichments/yaf.json
 %{metron_home}/config/zookeeper/enrichments/asa.json
-%{metron_home}/ddl/geoip_ddl.sql
 %{metron_home}/flux/enrichment/remote.yaml
 %exclude %{metron_home}/flux/enrichment/test.yaml
 %attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar
@@ -317,6 +316,8 @@ This package installs the Metron Profiler %{metron_home}
 # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 %changelog
+* Thu Jan 19 2017 Justin Leet <justinjleet@gmail.com> - 0.3.0
+- Replace GeoIP files with new implementation
 * Thu Nov 03 2016 David Lyle <dlyle65535@gmail.com> - 0.2.1
 - Add ASA parser/enrichment configuration files 
 * Thu Jul 21 2016 Michael Miklavcic <michael.miklavcic@gmail.com> - 0.2.1

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/packer-build/metron-centos-6.7.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packer-build/metron-centos-6.7.json b/metron-deployment/packer-build/metron-centos-6.7.json
index e6de665..a4cf625 100644
--- a/metron-deployment/packer-build/metron-centos-6.7.json
+++ b/metron-deployment/packer-build/metron-centos-6.7.json
@@ -89,7 +89,7 @@
       "sftp_command": "/usr/libexec/openssh/sftp-server",
       "user" : "vagrant",
       "host_alias" : "node1",
-      "groups" : ["packer","ambari_master","ambari_slave","metron_kafka_topics","metron_hbase_tables","enrichment","search","web","sensors","mysql","pcap_server","metron"]
+      "groups" : ["packer","ambari_master","ambari_slave","metron_kafka_topics","metron_hbase_tables","enrichment","search","web","sensors","pcap_server","metron"]
     },
     {
       "type": "shell",

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
index 9f2d231..359808a 100644
--- a/metron-deployment/playbooks/metron_install.yml
+++ b/metron-deployment/playbooks/metron_install.yml
@@ -60,23 +60,6 @@
     - solr
 
 #
-# mysql
-#
-- hosts: mysql
-  become: true
-  roles:
-    - role: mysql_server
-  tags:
-    - mysql-server
-
-- hosts: ambari_slave
-  become: true
-  roles:
-    - role: mysql_client
-  tags:
-    - mysql-client
-
-#
 # parsers, enrichment, and indexing topologies
 #
 - hosts: metron_hbase_tables

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/defaults/main.yml b/metron-deployment/roles/metron_streaming/defaults/main.yml
index ff9ab39..5cab9e4 100644
--- a/metron-deployment/roles/metron_streaming/defaults/main.yml
+++ b/metron-deployment/roles/metron_streaming/defaults/main.yml
@@ -64,6 +64,7 @@ metron_profiler_properties_config_path: "{{ metron_directory }}/config/profiler.
 hbase_config_path: "/etc/hbase/conf"
 hdfs_config_path: "/etc/hadoop/conf"
 pcap_hdfs_path: "/apps/metron/pcap"
+geo_hdfs_path: "/apps/metron/geo/default"
 
 threat_intel_bulk_load: True
 threat_intel_bin: "{{ metron_directory }}/bin/threatintel_bulk_load.sh"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/metron_streaming/tasks/geoip.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/geoip.yml b/metron-deployment/roles/metron_streaming/tasks/geoip.yml
new file mode 100644
index 0000000..b26f889
--- /dev/null
+++ b/metron-deployment/roles/metron_streaming/tasks/geoip.yml
@@ -0,0 +1,22 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+---
+
+- name: Load Geo DB
+  shell: "{{ metron_directory }}/bin/geo_enrichment_load.sh -z {{ zookeeper_url }}"
+  become: yes
+  become_user: hdfs

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml b/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
index 252e671..125d41d 100644
--- a/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
@@ -30,12 +30,27 @@
   become: yes
   become_user: hdfs
 
+- name: Create Metron HDFS geo directory
+  command: hdfs dfs -mkdir -p {{ geo_hdfs_path }}
+  become: yes
+  become_user: hdfs
+
 - name: Assign hdfs as owner of HDFS output directory
   command: hdfs dfs -chown hdfs:hadoop {{ metron_hdfs_output_dir }}
   become: yes
   become_user: hdfs
 
+- name: Assign hdfs as owner of HDFS Geo directory
+  command: hdfs dfs -chown hdfs:hadoop {{ geo_hdfs_path }}
+  become: yes
+  become_user: hdfs
+
 - name: Assign permissions of HDFS output directory
   command: hdfs dfs -chmod 775 {{ metron_hdfs_output_dir }}
   become: yes
+  become_user: hdfs
+
+- name: Assign permissions of HDFS geo directory
+  command: hdfs dfs -chmod 775 {{ geo_hdfs_path }}
+  become: yes
   become_user: hdfs
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml
index 0945e3d..498b8dd 100644
--- a/metron-deployment/roles/metron_streaming/tasks/main.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/main.yml
@@ -41,6 +41,9 @@
 - include: source_config.yml
   run_once: true
 
+- include: geoip.yml
+  run_once: true
+
 - include: threat_intel.yml
   run_once: true
   when: threat_intel_bulk_load == True

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/metron_streaming/tasks/topologies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/topologies.yml b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
index a09904d..827e861 100644
--- a/metron-deployment/roles/metron_streaming/tasks/topologies.yml
+++ b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
@@ -15,10 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Get Default mysql password
-  include_vars: "../roles/mysql_server/defaults/main.yml"
-  when: mysql_root_password is undefined
-  
 - name: Configure Metron Pcap Topology
   lineinfile: >
     dest={{ metron_pcap_properties_config_path }}
@@ -44,8 +40,6 @@
     - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
     - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
     - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
-    - { regexp: "mysql.ip=", line: "mysql.ip={{ groups.mysql[0] }}" }
-    - { regexp: "mysql.password=", line: "mysql.password={{ mysql_root_password }}" }
 
 
 - name: Configure Metron Solr topology

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/monit/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/defaults/main.yml b/metron-deployment/roles/monit/defaults/main.yml
index 608be38..c1c7583 100644
--- a/metron-deployment/roles/monit/defaults/main.yml
+++ b/metron-deployment/roles/monit/defaults/main.yml
@@ -24,5 +24,4 @@ topology_stop_timeout: 120
 
 bro_pid_file: /usr/local/bro/spool/bro/.pid
 elasticsearch_pid_file: /var/run/elasticsearch/elasticsearch.pid
-mysql_pid_file: /var/run/mysqld/mysqld.pid
 snort_alert_csv_path: /var/log/snort/alert.csv

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
index 1d29795..b6ea821 100644
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -32,10 +32,6 @@
   template: src=monit/kibana.monit dest={{ monit_config_home }}/kibana.monit
   when: ("web" in group_names) and (install_elasticsearch | default(True))
 
-- name: Create monit definition for mysql
-  template: src=monit/mysql.monit dest={{ monit_config_home }}/mysql.monit
-  when: ("mysql" in group_names)
-
 - name: Create monit definition for parsers
   template: src=monit/parsers.monit dest={{ monit_config_home }}/parsers.monit
   when: ("enrichment" in group_names)

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/monit/templates/monit/mysql.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/mysql.monit b/metron-deployment/roles/monit/templates/monit/mysql.monit
deleted file mode 100644
index 555b444..0000000
--- a/metron-deployment/roles/monit/templates/monit/mysql.monit
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-check process mysql with pidfile {{ mysql_pid_file }}
-  start program = "/etc/init.d/mysqld start"
-  stop program = "/etc/init.d/mysqld stop"
-  if does not exist then restart
-  group metron

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/mysql_client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_client/tasks/main.yml b/metron-deployment/roles/mysql_client/tasks/main.yml
deleted file mode 100644
index 82d57b8..0000000
--- a/metron-deployment/roles/mysql_client/tasks/main.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-
-- name: Get Default mysql password
-  include_vars: "../roles/mysql_server/defaults/main.yml"
-  when: mysql_root_password is undefined
-
-- name: Allow remote login to mysql
-  template:
-    src: "../roles/mysql_client/templates/db_config.sql"
-    dest: "/tmp/{{ansible_fqdn}}.sql"
-  delegate_to: "{{ groups.mysql[0] }}"
-
-- name: Import DB_Config
-  mysql_db:
-    name: "all"
-    state: "import"
-    target: "/tmp/{{ansible_fqdn}}.sql"
-  ignore_errors: True
-  delegate_to: "{{ groups.mysql[0] }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/mysql_client/templates/db_config.sql
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_client/templates/db_config.sql b/metron-deployment/roles/mysql_client/templates/db_config.sql
deleted file mode 100644
index c407a13..0000000
--- a/metron-deployment/roles/mysql_client/templates/db_config.sql
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-
-CREATE USER 'root'@'{{ ansible_fqdn }}' IDENTIFIED BY '{{ mysql_root_password }}';
-SET PASSWORD FOR 'root'@'{{ ansible_fqdn }}' = PASSWORD('{{ mysql_root_password }}');
-GRANT ALL PRIVILEGES ON *.* to 'root'@'{{ ansible_fqdn }}' WITH GRANT OPTION;
-FLUSH PRIVILEGES;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/mysql_server/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/defaults/main.yml b/metron-deployment/roles/mysql_server/defaults/main.yml
deleted file mode 100644
index 08a69a5..0000000
--- a/metron-deployment/roles/mysql_server/defaults/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-mysql_rpm_version: mysql57-community-release-el6-7.noarch
-mysql_yum_repo_url: https://dev.mysql.com/get/{{ mysql_rpm_version }}.rpm
-mysql_root_password: P@ssw0rd
-geodata_file: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/mysql_server/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/files/geoip_ddl.sql b/metron-deployment/roles/mysql_server/files/geoip_ddl.sql
deleted file mode 100644
index 02616c6..0000000
--- a/metron-deployment/roles/mysql_server/files/geoip_ddl.sql
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
-CREATE DATABASE IF NOT EXISTS GEO;
-
-USE GEO;
-
-DROP TABLE IF EXISTS `blocks`;
-CREATE TABLE  `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
-int(10) unsigned NOT NULL, PRIMARY KEY  (`startIPNum`,`endIPNum`) )
-ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
-
-DROP TABLE IF EXISTS `location`;
-CREATE TABLE  `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
- default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
-NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
-  (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
-
-load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv'  into table `blocks`  fields terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
-load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv'  into table `location`  fields terminated by ',' optionally enclosed by '"'  lines terminated by '\n' ignore 2 lines;
-
-
-DELIMITER $$
-DROP FUNCTION IF EXISTS `IPTOLOCID` $$
-CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
-  BEGIN
-    DECLARE ipn INTEGER UNSIGNED;
-    DECLARE locID_var INTEGER;
-    IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
-    END IF;
-    SET ipn = INET_ATON(ip);
-    SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
-    RETURN locID_var;
-  END
-$$
-DELIMITER ;

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/mysql_server/handlers/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/handlers/main.yml b/metron-deployment/roles/mysql_server/handlers/main.yml
deleted file mode 100644
index 112c5ca..0000000
--- a/metron-deployment/roles/mysql_server/handlers/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-- name: restart elasticsearch
-  service: name=elasticsearch state=restarted

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/mysql_server/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/meta/main.yml b/metron-deployment/roles/mysql_server/meta/main.yml
deleted file mode 100644
index 6f97a14..0000000
--- a/metron-deployment/roles/mysql_server/meta/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-dependencies:
-  - python-pip
-  - libselinux-python
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/fa6f3df5/metron-deployment/roles/mysql_server/tasks/geoip.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/mysql_server/tasks/geoip.yml b/metron-deployment/roles/mysql_server/tasks/geoip.yml
deleted file mode 100644
index 407042e..0000000
--- a/metron-deployment/roles/mysql_server/tasks/geoip.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
----
-- name: Download GeoIP databases from specified URL
-  unarchive:
-    src: "{{ geodata_file }}"
-    dest: /tmp/geoip
-    copy: no
-    creates: /tmp/geoip/*/GeoLiteCity-Blocks.csv
-
-- name: Copy to MySQL import directory
-  shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
-
-- name: Copy DDL
-  copy:
-    src: geoip_ddl.sql
-    dest: /tmp/geoip_ddl.sql
-
-- name: Import GeoIP DDL
-  mysql_db:
-    name: all
-    state: import
-    target: /tmp/geoip_ddl.sql



Mime
View raw message