metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [48/52] [abbrv] metron git commit: METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984
Date Wed, 18 Apr 2018 15:00:18 GMT
METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/82212ba8
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/82212ba8
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/82212ba8

Branch: refs/heads/feature/METRON-1211-extensions-parsers-gradual
Commit: 82212ba818a0ef3b92505e8d144487c69a8d4a44
Parents: 3fcbf8b
Author: nickwallen <nick@nickallen.org>
Authored: Tue Apr 17 09:43:16 2018 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Tue Apr 17 09:43:16 2018 -0400

----------------------------------------------------------------------
 .../packaging/ambari/metron-mpack/pom.xml       |   7 +-
 .../configuration/metron-enrichment-env.xml     | 130 +++-
 .../package/scripts/enrichment_commands.py      |  20 +-
 .../package/scripts/enrichment_master.py        |  12 +-
 .../package/scripts/params/params_linux.py      |  19 +-
 .../enrichment-splitjoin.properties.j2          |  63 ++
 .../templates/enrichment-unified.properties.j2  |  60 ++
 .../METRON/CURRENT/themes/metron_theme.json     | 151 ++++-
 .../docker/rpm-docker/SPECS/metron.spec         |   5 +-
 .../main/config/enrichment-splitjoin.properties |  63 ++
 .../config/enrichment-splitjoin.properties.j2   |  63 ++
 .../main/config/enrichment-unified.properties   |  69 +++
 .../config/enrichment-unified.properties.j2     |  60 ++
 .../src/main/config/enrichment.properties       |  64 --
 .../src/main/config/enrichment.properties.j2    |  63 --
 .../main/flux/enrichment/remote-splitjoin.yaml  | 590 +++++++++++++++++++
 .../main/flux/enrichment/remote-unified.yaml    |  71 ++-
 .../src/main/flux/enrichment/remote.yaml        | 590 -------------------
 .../main/scripts/start_enrichment_topology.sh   |  16 +-
 .../integration/EnrichmentIntegrationTest.java  |  61 +-
 .../UnifiedEnrichmentIntegrationTest.java       |  71 +++
 21 files changed, 1433 insertions(+), 815 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/pom.xml b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
index 491e8dd..1843eb7 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/pom.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/pom.xml
@@ -110,7 +110,8 @@
                                 <resource>
                                     <directory>${basedir}/../../../../metron-platform/metron-enrichment/src/main/config</directory>
                                     <includes>
-                                        <include>enrichment.properties.j2</include>
+                                        <include>enrichment-splitjoin.properties.j2</include>
+                                        <include>enrichment-unified.properties.j2</include>
                                     </includes>
                                     <filtering>false</filtering>
                                 </resource>
@@ -171,8 +172,10 @@
                         <fileset>
                             <directory>${basedir}/src/main/resources/common-services/METRON/CURRENT/package/templates</directory>
                             <includes>
-                                <include>enrichment.properties.j2</include>
+                                <include>enrichment-unified.properties.j2</include>
+                                <include>enrichment-splitjoin.properties.j2</include>
                                 <include>elasticsearch.properties.j2</include>
+                                <include>hdfs.properties.j2</include>
                             </includes>
                         </fileset>
                     </filesets>

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
index 9737660..81b135c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-enrichment-env.xml
@@ -18,6 +18,10 @@
   limitations under the License.
 -->
 <configuration supports_final="true">
+
+  <!--
+    enrichment adapter parameters
+  -->
   <property>
     <name>geoip_url</name>
     <value>http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.mmdb.gz</value>
@@ -33,6 +37,10 @@
       <type>content</type>
     </value-attributes>
   </property>
+
+  <!--
+    kafka parameters
+  -->
   <property>
     <name>enrichment_kafka_start</name>
     <description>Enrichment Topology Spout Offset</description>
@@ -81,6 +89,10 @@
     <value>indexing</value>
     <display-name>Threat Intel Error Topic</display-name>
   </property>
+
+  <!--
+    hbase parameters
+  -->
   <property>
     <name>enrichment_hbase_table</name>
     <value>enrichment</value>
@@ -105,6 +117,10 @@
     <description>The HBase column family which will hold threatintel data in HBase.</description>
     <display-name>HBase Table Column Family</display-name>
   </property>
+
+  <!--
+    storm common parameters
+  -->
   <property>
     <name>enrichment_workers</name>
     <description>Number of Workers for the Enrichment Topology</description>
@@ -129,70 +145,156 @@
   <property>
     <name>enrichment_topology_max_spout_pending</name>
     <description>Spout Max Pending Tuples for the Enrichment Topology</description>
-    <value/>
-    <display-name>Enrichment Max Pending</display-name>
+    <value>500</value>
+    <display-name>Enrichment Max Spout Pending</display-name>
     <value-attributes>
       <empty-value-valid>true</empty-value-valid>
     </value-attributes>
   </property>
   <property>
+    <name>enrichment_topology</name>
+    <description>Which Enrichment topology to execute</description>
+    <value>Split-Join</value>
+    <display-name>Enrichment Topology</display-name>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>Split-Join</value>
+        </entry>
+        <entry>
+          <value>Unified</value>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+    </value-attributes>
+  </property>
+
+  <!--
+    split-join topology parameters
+  -->
+  <property>
     <name>enrichment_join_cache_size</name>
-    <description>Enrichment Join Bolt Cache Max Size</description>
+    <description>Enrichment join bolt max cache size for the Split Join Enrichment Topology</description>
     <value>100000</value>
-    <display-name>Enrichment Join Size</display-name>
+    <display-name>Enrichment Join Max Cache Size</display-name>
   </property>
   <property>
     <name>threatintel_join_cache_size</name>
-    <description>Threat Intel Join Bolt Cache Max Size</description>
+    <description>Threat Intel join bolt max cache size for the Split Join Enrichment Topology</description>
     <value>100000</value>
-    <display-name>Threat Intel Join Size</display-name>
+    <display-name>Threat Intel Join Max Cache Size</display-name>
   </property>
   <property>
     <name>enrichment_kafka_spout_parallelism</name>
-    <description>Kafka Spout Parallelism for the Enrichment Topology</description>
+    <description>Kafka spout parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Enrichment Spout Parallelism</display-name>
   </property>
   <property>
     <name>enrichment_split_parallelism</name>
-    <description>Enrichment Split Bolt Parallelism for the Enrichment Topology</description>
+    <description>Enrichment split bolt parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Enrichment Split Parallelism</display-name>
   </property>
   <property>
     <name>enrichment_stellar_parallelism</name>
-    <description>Stellar Enrichment Bolt Parallelism for the Enrichment Topology</description>
+    <description>Stellar enrichment bolt parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Stellar Enrichment Parallelism</display-name>
   </property>
   <property>
     <name>enrichment_join_parallelism</name>
-    <description>Enrichment Join Bolt Parallelism for the Enrichment Topology</description>
+    <description>Enrichment join bolt parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Enrichment Join Parallelism</display-name>
   </property>
   <property>
     <name>threat_intel_split_parallelism</name>
-    <description>Threat Intel Split Bolt Parallelism for the Enrichment Topology</description>
+    <description>Threat Intel split bolt parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Threat Intel Split Parallelism</display-name>
   </property>
   <property>
     <name>threat_intel_stellar_parallelism</name>
-    <description>Threat Intel Stellar Bolt Parallelism for the Enrichment Topology</description>
+    <description>Threat Intel stellar bolt parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Threat Intel Stellar Parallelism</display-name>
   </property>
   <property>
     <name>threat_intel_join_parallelism</name>
-    <description>Threat Intel Join Bolt Parallelism for the Enrichment Topology</description>
+    <description>Threat Intel join bolt parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Threat Intel Join Parallelism</display-name>
   </property>
   <property>
     <name>kafka_writer_parallelism</name>
-    <description>Kafka Writer Parallelism for the Enrichment Topology</description>
+    <description>Kafka writer parallelism for the Split Join Enrichment Topology</description>
     <value>1</value>
     <display-name>Enrichment Kafka Writer Parallelism</display-name>
   </property>
+
+  <!--
+    unified topology parameters
+  -->
+  <property>
+    <name>unified_kafka_spout_parallelism</name>
+    <description>Kafka spout parallelism for the Unified Enrichment Topology</description>
+    <value>1</value>
+    <display-name>Unified Enrichment Spout Parallelism</display-name>
+  </property>
+  <property>
+    <name>unified_enrichment_parallelism</name>
+    <description>Enrichment parallelism for the Unified Enrichment Topology</description>
+    <value>1</value>
+    <display-name>Unified Enrichment Parallelism</display-name>
+  </property>
+  <property>
+    <name>unified_threat_intel_parallelism</name>
+    <description>Threat Intel parallelism for the Unified Enrichment Topology</description>
+    <value>1</value>
+    <display-name>Unified Threat Intel Parallelism</display-name>
+  </property>
+  <property>
+    <name>unified_kafka_writer_parallelism</name>
+    <description>Kafka writer parallelism for the Unified Enrichment Topology</description>
+    <value>1</value>
+    <display-name>Unified Kafka Writer Parallelism</display-name>
+  </property>
+  <property>
+    <name>unified_enrichment_cache_size</name>
+    <description>Enrichment max cache size for the Unified Enrichment Topology</description>
+    <value>100000</value>
+    <display-name>Unified Enrichment Cache Size</display-name>
+  </property>
+  <property>
+    <name>unified_threat_intel_cache_size</name>
+    <description>Threat Intel Max Cache Size for the Unified Enrichment Topology</description>
+    <value>100000</value>
+    <display-name>Unified Threat Intel Cache Size</display-name>
+  </property>
+  <property>
+    <name>unified_enrichment_threadpool_size</name>
+    <description>Enrichment thread pool size for the Unified Enrichment Topology</description>
+    <value>1</value>
+    <display-name>Unified Enrichment Thread Pool Size</display-name>
+  </property>
+  <property>
+    <name>unified_enrichment_threadpool_type</name>
+    <description>Enrichment thread pool type for the Unified Enrichment Topology</description>
+    <display-name>Unified Enrichment Thread Pool Type</display-name>
+    <value>FIXED</value>
+    <value-attributes>
+      <type>value-list</type>
+      <entries>
+        <entry>
+          <value>FIXED</value>
+        </entry>
+        <entry>
+          <value>WORK_STEALING</value>
+        </entry>
+      </entries>
+      <selection-cardinality>1</selection-cardinality>
+    </value-attributes>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index f9ec547..a1bdbed 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -129,13 +129,21 @@ class EnrichmentCommands:
         Logger.info("Starting Metron enrichment topology: {0}".format(self.__enrichment_topology))
 
         if not self.is_topology_active(env):
-            start_cmd_template = """{0}/bin/start_enrichment_topology.sh \
-                                        -s {1} \
-                                        -z {2}"""
+
+            # which enrichment topology needs started?
+            if self.__params.enrichment_topology == "Unified":
+                topology_flux = "{0}/flux/enrichment/remote-unified.yaml".format(self.__params.metron_home)
+                topology_props = "{0}/config/enrichment-unified.properties".format(self.__params.metron_home)
+            elif self.__params.enrichment_topology == "Split-Join":
+                topology_flux = "{0}/flux/enrichment/remote-splitjoin.yaml".format(self.__params.metron_home)
+                topology_props = "{0}/config/enrichment-splitjoin.properties".format(self.__params.metron_home)
+            else:
+                raise Fail("Unexpected enrichment topology; name=" + self.__params.enrichment_topology)
+
+            # start the topology
+            start_cmd_template = """{0}/bin/start_enrichment_topology.sh --remote {1} --filter {2}"""
             Logger.info('Starting ' + self.__enrichment_topology)
-            start_cmd = start_cmd_template.format(self.__params.metron_home,
-                                                  self.__enrichment_topology,
-                                                  self.__params.zookeeper_quorum)
+            start_cmd = start_cmd_template.format(self.__params.metron_home, topology_flux, topology_props)
             Execute(start_cmd, user=self.__params.metron_user, tries=3, try_sleep=5, logoutput=True)
         else:
             Logger.info('Enrichment topology already running')

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
index 24feb81..cada1d2 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_master.py
@@ -38,11 +38,15 @@ class Enrichment(Script):
         env.set_params(params)
 
         Logger.info("Running enrichment configure")
-        File(format("{metron_config_path}/enrichment.properties"),
-             content=Template("enrichment.properties.j2"),
+        File(format("{metron_config_path}/enrichment-splitjoin.properties"),
+             content=Template("enrichment-splitjoin.properties.j2"),
              owner=params.metron_user,
-             group=params.metron_group
-             )
+             group=params.metron_group)
+
+        File(format("{metron_config_path}/enrichment-unified.properties"),
+            content=Template("enrichment-unified.properties.j2"),
+            owner=params.metron_user,
+            group=params.metron_group)
 
         if not metron_service.is_zk_configured(params):
           metron_service.init_zk_config(params)

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index ccce022..f44d05f 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -256,20 +256,27 @@ user_settings_hbase_table = status_params.user_settings_hbase_table
 user_settings_hbase_cf = status_params.user_settings_hbase_cf
 
 # Enrichment
+metron_enrichment_topology = status_params.metron_enrichment_topology
 geoip_url = config['configurations']['metron-enrichment-env']['geoip_url']
 enrichment_host_known_hosts = config['configurations']['metron-enrichment-env']['enrichment_host_known_hosts']
+
+# Enrichment - Kafka
 enrichment_kafka_start = config['configurations']['metron-enrichment-env']['enrichment_kafka_start']
 enrichment_input_topic = status_params.enrichment_input_topic
 enrichment_output_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic']
 enrichment_error_topic = config['configurations']['metron-enrichment-env']['enrichment_error_topic']
 threatintel_error_topic = config['configurations']['metron-enrichment-env']['threatintel_error_topic']
-metron_enrichment_topology = status_params.metron_enrichment_topology
+
+# Enrichment - Storm common parameters
 enrichment_workers = config['configurations']['metron-enrichment-env']['enrichment_workers']
 enrichment_acker_executors = config['configurations']['metron-enrichment-env']['enrichment_acker_executors']
 if not len(enrichment_topology_worker_childopts) == 0:
     enrichment_topology_worker_childopts += ' '
 enrichment_topology_worker_childopts += config['configurations']['metron-enrichment-env']['enrichment_topology_worker_childopts']
 enrichment_topology_max_spout_pending = config['configurations']['metron-enrichment-env']['enrichment_topology_max_spout_pending']
+enrichment_topology = config['configurations']['metron-enrichment-env']['enrichment_topology']
+
+# Enrichment - Split Join topology
 enrichment_join_cache_size = config['configurations']['metron-enrichment-env']['enrichment_join_cache_size']
 threatintel_join_cache_size = config['configurations']['metron-enrichment-env']['threatintel_join_cache_size']
 enrichment_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['enrichment_kafka_spout_parallelism']
@@ -281,6 +288,16 @@ threat_intel_stellar_parallelism = config['configurations']['metron-enrichment-e
 threat_intel_join_parallelism = config['configurations']['metron-enrichment-env']['threat_intel_join_parallelism']
 kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['kafka_writer_parallelism']
 
+# Enrichment - Unified topology
+unified_kafka_spout_parallelism = config['configurations']['metron-enrichment-env']['unified_kafka_spout_parallelism']
+unified_enrichment_parallelism = config['configurations']['metron-enrichment-env']['unified_enrichment_parallelism']
+unified_threat_intel_parallelism = config['configurations']['metron-enrichment-env']['unified_threat_intel_parallelism']
+unified_kafka_writer_parallelism = config['configurations']['metron-enrichment-env']['unified_kafka_writer_parallelism']
+unified_enrichment_cache_size = config['configurations']['metron-enrichment-env']['unified_enrichment_cache_size']
+unified_threat_intel_cache_size = config['configurations']['metron-enrichment-env']['unified_threat_intel_cache_size']
+unified_enrichment_threadpool_size = config['configurations']['metron-enrichment-env']['unified_enrichment_threadpool_size']
+unified_enrichment_threadpool_type = config['configurations']['metron-enrichment-env']['unified_enrichment_threadpool_type']
+
 # Profiler
 metron_profiler_topology = 'profiler'
 profiler_input_topic = config['configurations']['metron-enrichment-env']['enrichment_output_topic']

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
new file mode 100644
index 0000000..a0b21c9
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-splitjoin.properties.j2
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start={{enrichment_kafka_start}}
+
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### JoinBolt #####
+enrichment.join.cache.size={{enrichment_join_cache_size}}
+threat.intel.join.cache.size={{threatintel_join_cache_size}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
+enrichment.split.parallelism={{enrichment_split_parallelism}}
+enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}
+enrichment.join.parallelism={{enrichment_join_parallelism}}
+threat.intel.split.parallelism={{threat_intel_split_parallelism}}
+threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}}
+threat.intel.join.parallelism={{threat_intel_join_parallelism}}
+kafka.writer.parallelism={{kafka_writer_parallelism}}

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2
new file mode 100644
index 0000000..8c28c49
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/enrichment-unified.properties.j2
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+kafka.start={{enrichment_kafka_start}}
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{unified_kafka_spout_parallelism}}
+enrichment.parallelism={{unified_enrichment_parallelism}}
+threat.intel.parallelism={{unified_threat_intel_parallelism}}
+kafka.writer.parallelism={{unified_kafka_writer_parallelism}}
+
+##### Caches #####
+enrichment.cache.size={{unified_enrichment_cache_size}}
+threat.intel.cache.size={{unified_threat_intel_cache_size}}
+
+##### Threads #####
+enrichment.threadpool.size={{unified_enrichment_threadpool_size}}
+enrichment.threadpool.type={{unified_enrichment_threadpool_type}}

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
index 364b3ef..06bc155 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/themes/metron_theme.json
@@ -65,7 +65,7 @@
             "display-name": "Enrichment",
             "layout": {
               "tab-columns": "1",
-              "tab-rows": "3",
+              "tab-rows": "5",
               "sections": [
                 {
                   "name": "section-enrichment-adapters",
@@ -123,6 +123,44 @@
                       "column-span": "1"
                     }
                   ]
+                },
+                {
+                  "name": "section-enrichment-splitjoin",
+                  "row-index": "3",
+                  "column-index": "0",
+                  "row-span": "1",
+                  "column-span": "1",
+                  "section-columns": "1",
+                  "section-rows": "1",
+                  "subsections": [
+                    {
+                      "name": "subsection-enrichment-splitjoin",
+                      "display-name": "Split Join Topology",
+                      "row-index": "0",
+                      "column-index": "0",
+                      "row-span": "1",
+                      "column-span": "1"
+                    }
+                  ]
+                },
+                {
+                  "name": "section-enrichment-unified",
+                  "row-index": "4",
+                  "column-index": "0",
+                  "row-span": "1",
+                  "column-span": "1",
+                  "section-columns": "1",
+                  "section-rows": "1",
+                  "subsections": [
+                    {
+                      "name": "subsection-enrichment-unified",
+                      "display-name": "Unified Topology",
+                      "row-index": "0",
+                      "column-index": "0",
+                      "row-span": "1",
+                      "column-span": "1"
+                    }
+                  ]
                 }
               ]
             }
@@ -462,44 +500,80 @@
           "subsection-name": "subsection-enrichment-storm"
         },
         {
-          "config": "metron-enrichment-env/enrichment_join_cache_size",
+          "config": "metron-enrichment-env/enrichment_topology",
           "subsection-name": "subsection-enrichment-storm"
         },
         {
+          "config": "metron-enrichment-env/enrichment_join_cache_size",
+          "subsection-name": "subsection-enrichment-splitjoin"
+        },
+        {
           "config": "metron-enrichment-env/threatintel_join_cache_size",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/enrichment_kafka_spout_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/enrichment_split_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/enrichment_stellar_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/enrichment_join_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/threat_intel_split_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/threat_intel_stellar_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/threat_intel_join_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
         },
         {
           "config": "metron-enrichment-env/kafka_writer_parallelism",
-          "subsection-name": "subsection-enrichment-storm"
+          "subsection-name": "subsection-enrichment-splitjoin"
+        },
+        {
+          "config": "metron-enrichment-env/unified_kafka_spout_parallelism",
+          "subsection-name": "subsection-enrichment-unified"
+        },
+        {
+          "config": "metron-enrichment-env/unified_enrichment_parallelism",
+          "subsection-name": "subsection-enrichment-unified"
+        },
+        {
+          "config": "metron-enrichment-env/unified_threat_intel_parallelism",
+          "subsection-name": "subsection-enrichment-unified"
+        },
+        {
+          "config": "metron-enrichment-env/unified_kafka_writer_parallelism",
+          "subsection-name": "subsection-enrichment-unified"
+        },
+        {
+          "config": "metron-enrichment-env/unified_enrichment_cache_size",
+          "subsection-name": "subsection-enrichment-unified"
+        },
+        {
+          "config": "metron-enrichment-env/unified_threat_intel_cache_size",
+          "subsection-name": "subsection-enrichment-unified"
+        },
+        {
+          "config": "metron-enrichment-env/unified_enrichment_threadpool_size",
+          "subsection-name": "subsection-enrichment-unified"
+        },
+        {
+          "config": "metron-enrichment-env/unified_enrichment_threadpool_type",
+          "subsection-name": "subsection-enrichment-unified"
         },
         {
           "config": "metron-indexing-env/ra_indexing_kafka_start",
@@ -838,6 +912,12 @@
         }
       },
       {
+        "config": "metron-enrichment-env/enrichment_topology",
+        "widget": {
+          "type": "combo"
+        }
+      },
+      {
         "config": "metron-enrichment-env/enrichment_join_cache_size",
         "widget": {
           "type": "text-field"
@@ -897,7 +977,54 @@
           "type": "text-field"
         }
       },
-
+      {
+        "config": "metron-enrichment-env/unified_kafka_spout_parallelism",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/unified_enrichment_parallelism",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/unified_threat_intel_parallelism",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/unified_kafka_writer_parallelism",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/unified_enrichment_cache_size",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/unified_threat_intel_cache_size",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/unified_enrichment_threadpool_size",
+        "widget": {
+          "type": "text-field"
+        }
+      },
+      {
+        "config": "metron-enrichment-env/unified_enrichment_threadpool_type",
+        "widget": {
+          "type": "combo"
+        }
+      },
       {
         "config": "metron-indexing-env/batch_indexing_kafka_start",
         "widget": {

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
index 6b35dae..1f40105 100644
--- a/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
+++ b/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
@@ -273,13 +273,14 @@ This package installs the Metron Enrichment files
 %dir %{metron_home}/flux/enrichment
 %{metron_home}/bin/latency_summarizer.sh
 %{metron_home}/bin/start_enrichment_topology.sh
-%{metron_home}/config/enrichment.properties
+%{metron_home}/config/enrichment-splitjoin.properties
+%{metron_home}/config/enrichment-unified.properties
 %{metron_home}/config/zookeeper/enrichments/bro.json
 %{metron_home}/config/zookeeper/enrichments/snort.json
 %{metron_home}/config/zookeeper/enrichments/websphere.json
 %{metron_home}/config/zookeeper/enrichments/yaf.json
 %{metron_home}/config/zookeeper/enrichments/asa.json
-%{metron_home}/flux/enrichment/remote.yaml
+%{metron_home}/flux/enrichment/remote-splitjoin.yaml
 %{metron_home}/flux/enrichment/remote-unified.yaml
 %attr(0644,root,root) %{metron_home}/lib/metron-enrichment-%{full_version}-uber.jar
 

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties
new file mode 100644
index 0000000..109c2ee
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties
@@ -0,0 +1,63 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+##### Storm #####
+enrichment.workers=1
+enrichment.acker.executors=0
+topology.worker.childopts=
+topology.auto-credentials=
+topology.max.spout.pending=500
+
+##### Kafka #####
+kafka.zk=node1:2181
+kafka.broker=node1:6667
+kafka.security.protocol=PLAINTEXT
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start=UNCOMMITTED_EARLIEST
+
+enrichment.input.topic=enrichments
+enrichment.output.topic=indexing
+enrichment.error.topic=indexing
+threat.intel.error.topic=indexing
+
+##### JoinBolt #####
+enrichment.join.cache.size=100000
+threat.intel.join.cache.size=100000
+
+##### Enrichment #####
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table=enrichment
+enrichment.simple.hbase.cf=t
+enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+##### Threat Intel #####
+threat.intel.tracker.table=access_tracker
+threat.intel.tracker.cf=t
+threat.intel.simple.hbase.table=threatintel
+threat.intel.simple.hbase.cf=t
+
+##### Parallelism #####
+kafka.spout.parallelism=1
+enrichment.split.parallelism=1
+enrichment.stellar.parallelism=1
+enrichment.join.parallelism=1
+threat.intel.split.parallelism=1
+threat.intel.stellar.parallelism=1
+threat.intel.join.parallelism=1
+kafka.writer.parallelism=1

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2
new file mode 100755
index 0000000..a0b21c9
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-splitjoin.properties.j2
@@ -0,0 +1,63 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start={{enrichment_kafka_start}}
+
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### JoinBolt #####
+enrichment.join.cache.size={{enrichment_join_cache_size}}
+threat.intel.join.cache.size={{threatintel_join_cache_size}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
+enrichment.split.parallelism={{enrichment_split_parallelism}}
+enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}
+enrichment.join.parallelism={{enrichment_join_parallelism}}
+threat.intel.split.parallelism={{threat_intel_split_parallelism}}
+threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}}
+threat.intel.join.parallelism={{threat_intel_join_parallelism}}
+kafka.writer.parallelism={{kafka_writer_parallelism}}

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties
new file mode 100644
index 0000000..5338ead
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers=1
+enrichment.acker.executors=0
+topology.worker.childopts=
+topology.auto-credentials=
+topology.max.spout.pending=500
+
+##### Kafka #####
+kafka.zk=node1:2181
+kafka.broker=node1:6667
+kafka.security.protocol=PLAINTEXT
+
+# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
+kafka.start=UNCOMMITTED_EARLIEST
+
+enrichment.input.topic=enrichments
+enrichment.output.topic=indexing
+enrichment.error.topic=indexing
+threat.intel.error.topic=indexing
+
+##### JoinBolt #####
+enrichment.join.cache.size=100000
+threat.intel.join.cache.size=100000
+
+##### Enrichment #####
+hbase.provider.impl=org.apache.metron.hbase.HTableProvider
+enrichment.simple.hbase.table=enrichment
+enrichment.simple.hbase.cf=t
+enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
+{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
+{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
+
+##### Threat Intel #####
+threat.intel.tracker.table=access_tracker
+threat.intel.tracker.cf=t
+threat.intel.simple.hbase.table=threatintel
+threat.intel.simple.hbase.cf=t
+
+##### Parallelism #####
+kafka.spout.parallelism=1
+enrichment.parallelism=1
+threat.intel.parallelism=1
+kafka.writer.parallelism=1
+
+##### Caches #####
+enrichment.cache.size=100000
+threat.intel.cache.size=100000
+
+##### Threads #####
+enrichment.threadpool.size=1
+enrichment.threadpool.type=FIXED

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2
new file mode 100644
index 0000000..8c28c49
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/config/enrichment-unified.properties.j2
@@ -0,0 +1,60 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+
+##### Storm #####
+enrichment.workers={{enrichment_workers}}
+enrichment.acker.executors={{enrichment_acker_executors}}
+topology.worker.childopts={{enrichment_topology_worker_childopts}}
+topology.auto-credentials={{topology_auto_credentials}}
+topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
+
+##### Kafka #####
+kafka.zk={{zookeeper_quorum}}
+kafka.broker={{kafka_brokers}}
+kafka.security.protocol={{kafka_security_protocol}}
+kafka.start={{enrichment_kafka_start}}
+enrichment.input.topic={{enrichment_input_topic}}
+enrichment.output.topic={{enrichment_output_topic}}
+enrichment.error.topic={{enrichment_error_topic}}
+threat.intel.error.topic={{threatintel_error_topic}}
+
+##### Enrichment #####
+hbase.provider.impl={{enrichment_hbase_provider_impl}}
+enrichment.simple.hbase.table={{enrichment_hbase_table}}
+enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
+enrichment.host.known_hosts={{enrichment_host_known_hosts}}
+
+##### Threat Intel #####
+threat.intel.tracker.table={{threatintel_hbase_table}}
+threat.intel.tracker.cf={{threatintel_hbase_cf}}
+threat.intel.simple.hbase.table={{threatintel_hbase_table}}
+threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
+
+##### Parallelism #####
+kafka.spout.parallelism={{unified_kafka_spout_parallelism}}
+enrichment.parallelism={{unified_enrichment_parallelism}}
+threat.intel.parallelism={{unified_threat_intel_parallelism}}
+kafka.writer.parallelism={{unified_kafka_writer_parallelism}}
+
+##### Caches #####
+enrichment.cache.size={{unified_enrichment_cache_size}}
+threat.intel.cache.size={{unified_threat_intel_cache_size}}
+
+##### Threads #####
+enrichment.threadpool.size={{unified_enrichment_threadpool_size}}
+enrichment.threadpool.type={{unified_enrichment_threadpool_type}}

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment.properties
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties b/metron-platform/metron-enrichment/src/main/config/enrichment.properties
deleted file mode 100644
index 9592968..0000000
--- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties
+++ /dev/null
@@ -1,64 +0,0 @@
-#  Licensed to the Apache Software Foundation (ASF) under one
-#  or more contributor license agreements.  See the NOTICE file
-#  distributed with this work for additional information
-#  regarding copyright ownership.  The ASF licenses this file
-#  to you under the Apache License, Version 2.0 (the
-#  "License"); you may not use this file except in compliance
-#  with the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-##### Storm #####
-enrichment.workers=1
-enrichment.acker.executors=0
-topology.worker.childopts=
-topology.auto-credentials=
-topology.max.spout.pending=
-
-##### Kafka #####
-kafka.zk=node1:2181
-kafka.broker=node1:6667
-kafka.security.protocol=PLAINTEXT
-
-# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
-kafka.start=UNCOMMITTED_EARLIEST
-
-enrichment.input.topic=enrichments
-enrichment.output.topic=indexing
-enrichment.error.topic=indexing
-threat.intel.error.topic=indexing
-
-##### JoinBolt #####
-enrichment.join.cache.size=100000
-threat.intel.join.cache.size=100000
-
-##### Enrichment #####
-hbase.provider.impl=org.apache.metron.hbase.HTableProvider
-enrichment.simple.hbase.table=enrichment
-enrichment.simple.hbase.cf=t
-enrichment.host.known_hosts=[{"ip":"10.1.128.236", "local":"YES", "type":"webserver", "asset_value" : "important"},\
-{"ip":"10.1.128.237", "local":"UNKNOWN", "type":"unknown", "asset_value" : "important"},\
-{"ip":"10.60.10.254", "local":"YES", "type":"printer", "asset_value" : "important"}]
-
-
-##### Threat Intel #####
-threat.intel.tracker.table=access_tracker
-threat.intel.tracker.cf=t
-threat.intel.simple.hbase.table=threatintel
-threat.intel.simple.hbase.cf=t
-
-##### Parallelism #####
-kafka.spout.parallelism=1
-enrichment.split.parallelism=1
-enrichment.stellar.parallelism=1
-enrichment.join.parallelism=1
-threat.intel.split.parallelism=1
-threat.intel.stellar.parallelism=1
-threat.intel.join.parallelism=1
-kafka.writer.parallelism=1

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2 b/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
deleted file mode 100755
index 133f9c5..0000000
--- a/metron-platform/metron-enrichment/src/main/config/enrichment.properties.j2
+++ /dev/null
@@ -1,63 +0,0 @@
-{#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#}
-
-##### Storm #####
-enrichment.workers={{enrichment_workers}}
-enrichment.acker.executors={{enrichment_acker_executors}}
-topology.worker.childopts={{enrichment_topology_worker_childopts}}
-topology.auto-credentials={{topology_auto_credentials}}
-topology.max.spout.pending={{enrichment_topology_max_spout_pending}}
-
-##### Kafka #####
-kafka.zk={{zookeeper_quorum}}
-kafka.broker={{kafka_brokers}}
-kafka.security.protocol={{kafka_security_protocol}}
-
-# One of EARLIEST, LATEST, UNCOMMITTED_EARLIEST, UNCOMMITTED_LATEST
-kafka.start={{enrichment_kafka_start}}
-
-enrichment.input.topic={{enrichment_input_topic}}
-enrichment.output.topic={{enrichment_output_topic}}
-enrichment.error.topic={{enrichment_error_topic}}
-threat.intel.error.topic={{threatintel_error_topic}}
-
-##### JoinBolt #####
-enrichment.join.cache.size={{enrichment_join_cache_size}}
-threat.intel.join.cache.size={{threatintel_join_cache_size}}
-
-##### Enrichment #####
-hbase.provider.impl={{enrichment_hbase_provider_impl}}
-enrichment.simple.hbase.table={{enrichment_hbase_table}}
-enrichment.simple.hbase.cf={{enrichment_hbase_cf}}
-enrichment.host.known_hosts={{enrichment_host_known_hosts}}
-
-##### Threat Intel #####
-threat.intel.tracker.table={{threatintel_hbase_table}}
-threat.intel.tracker.cf={{threatintel_hbase_cf}}
-threat.intel.simple.hbase.table={{threatintel_hbase_table}}
-threat.intel.simple.hbase.cf={{threatintel_hbase_cf}}
-
-##### Parallelism #####
-kafka.spout.parallelism={{enrichment_kafka_spout_parallelism}}
-enrichment.split.parallelism={{enrichment_split_parallelism}}
-enrichment.stellar.parallelism={{enrichment_stellar_parallelism}}
-enrichment.join.parallelism={{enrichment_join_parallelism}}
-threat.intel.split.parallelism={{threat_intel_split_parallelism}}
-threat.intel.stellar.parallelism={{threat_intel_stellar_parallelism}}
-threat.intel.join.parallelism={{threat_intel_join_parallelism}}
-kafka.writer.parallelism={{kafka_writer_parallelism}}

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml
new file mode 100644
index 0000000..fd7ceff
--- /dev/null
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-splitjoin.yaml
@@ -0,0 +1,590 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "enrichment"
+config:
+    topology.workers: ${enrichment.workers}
+    topology.acker.executors: ${enrichment.acker.executors}
+    topology.worker.childopts: ${topology.worker.childopts}
+    topology.auto-credentials: ${topology.auto-credentials}
+    topology.max.spout.pending: ${topology.max.spout.pending}
+
+components:
+
+# Enrichment
+    -   id: "stellarEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
+        configMethods:
+            -   name: "ofType"
+                args:
+                    - "ENRICHMENT"
+
+    # Any kafka props for the producer go here.
+    -   id: "kafkaWriterProps"
+        className: "java.util.HashMap"
+        configMethods:
+          -   name: "put"
+              args:
+                  - "security.protocol"
+                  - "${kafka.security.protocol}"
+
+    -   id: "stellarEnrichment"
+        className: "org.apache.metron.enrichment.configuration.Enrichment"
+        constructorArgs:
+            -   "stellar"
+            -   ref: "stellarEnrichmentAdapter"
+
+    -   id: "geoEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
+    -   id: "geoEnrichment"
+        className: "org.apache.metron.enrichment.configuration.Enrichment"
+        constructorArgs:
+            -   "geo"
+            -   ref: "geoEnrichmentAdapter"
+    -   id: "hostEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+        constructorArgs:
+            - '${enrichment.host.known_hosts}'
+    -   id: "hostEnrichment"
+        className: "org.apache.metron.enrichment.configuration.Enrichment"
+        constructorArgs:
+            -   "host"
+            -   ref: "hostEnrichmentAdapter"
+
+    -   id: "simpleHBaseEnrichmentConfig"
+        className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseConfig"
+        configMethods:
+            -   name: "withProviderImpl"
+                args:
+                    - "${hbase.provider.impl}"
+            -   name: "withHBaseTable"
+                args:
+                    - "${enrichment.simple.hbase.table}"
+            -   name: "withHBaseCF"
+                args:
+                    - "${enrichment.simple.hbase.cf}"
+    -   id: "simpleHBaseEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter"
+        configMethods:
+           -    name: "withConfig"
+                args:
+                    - ref: "simpleHBaseEnrichmentConfig"
+    -   id: "simpleHBaseEnrichment"
+        className: "org.apache.metron.enrichment.configuration.Enrichment"
+        constructorArgs:
+          -   "hbaseEnrichment"
+          -   ref: "simpleHBaseEnrichmentAdapter"
+    -   id: "enrichments"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args:
+                    - ref: "geoEnrichment"
+            -   name: "add"
+                args:
+                    - ref: "hostEnrichment"
+            -   name: "add"
+                args:
+                    - ref: "simpleHBaseEnrichment"
+            -   name: "add"
+                args:
+                    - ref: "stellarEnrichment"
+
+    #enrichment error
+    -   id: "enrichmentErrorKafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
+        configMethods:
+            -   name: "withTopic"
+                args:
+                    - "${enrichment.error.topic}"
+            -   name: "withZkQuorum"
+                args:
+                    - "${kafka.zk}"
+            -   name: "withProducerConfigs"
+                args: 
+                    - ref: "kafkaWriterProps"
+
+# Threat Intel
+    -   id: "stellarThreatIntelAdapter"
+        className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
+        configMethods:
+            -   name: "ofType"
+                args:
+                    - "THREAT_INTEL"
+    -   id: "stellarThreatIntelEnrichment"
+        className: "org.apache.metron.enrichment.configuration.Enrichment"
+        constructorArgs:
+            -   "stellar"
+            -   ref: "stellarThreatIntelAdapter"
+    -   id: "simpleHBaseThreatIntelConfig"
+        className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelConfig"
+        configMethods:
+            -   name: "withProviderImpl"
+                args:
+                    - "${hbase.provider.impl}"
+            -   name: "withTrackerHBaseTable"
+                args:
+                    - "${threat.intel.tracker.table}"
+            -   name: "withTrackerHBaseCF"
+                args:
+                    - "${threat.intel.tracker.cf}"
+            -   name: "withHBaseTable"
+                args:
+                    - "${threat.intel.simple.hbase.table}"
+            -   name: "withHBaseCF"
+                args:
+                    - "${threat.intel.simple.hbase.cf}"
+    -   id: "simpleHBaseThreatIntelAdapter"
+        className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelAdapter"
+        configMethods:
+           -    name: "withConfig"
+                args:
+                    - ref: "simpleHBaseThreatIntelConfig"
+    -   id: "simpleHBaseThreatIntelEnrichment"
+        className: "org.apache.metron.enrichment.configuration.Enrichment"
+        constructorArgs:
+          -   "hbaseThreatIntel"
+          -   ref: "simpleHBaseThreatIntelAdapter"
+
+    -   id: "threatIntels"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args:
+                    - ref: "simpleHBaseThreatIntelEnrichment"
+            -   name: "add"
+                args:
+                    - ref: "stellarThreatIntelEnrichment"
+
+    #threatintel error
+    -   id: "threatIntelErrorKafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
+        configMethods:
+            -   name: "withTopic"
+                args:
+                    - "${threat.intel.error.topic}"
+            -   name: "withZkQuorum"
+                args:
+                    - "${kafka.zk}"
+            -   name: "withProducerConfigs"
+                args: 
+                    - ref: "kafkaWriterProps"
+#indexing
+    -   id: "kafkaWriter"
+        className: "org.apache.metron.writer.kafka.KafkaWriter"
+        configMethods:
+            -   name: "withTopic"
+                args:
+                    - "${enrichment.output.topic}"
+            -   name: "withZkQuorum"
+                args:
+                    - "${kafka.zk}"
+            -   name: "withProducerConfigs"
+                args: 
+                    - ref: "kafkaWriterProps"
+
+#kafka/zookeeper
+    # Any kafka props for the consumer go here.
+    -   id: "kafkaProps"
+        className: "java.util.HashMap"
+        configMethods:
+          -   name: "put"
+              args:
+                  - "value.deserializer"
+                  - "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+          -   name: "put"
+              args:
+                  - "key.deserializer"
+                  - "org.apache.kafka.common.serialization.ByteArrayDeserializer"
+          -   name: "put"
+              args:
+                  - "group.id"
+                  - "enrichments"
+          -   name: "put"
+              args:
+                  - "security.protocol"
+                  - "${kafka.security.protocol}"
+
+
+  # The fields to pull out of the kafka messages
+    -   id: "fields"
+        className: "java.util.ArrayList"
+        configMethods:
+          -   name: "add"
+              args:
+                  - "value"
+
+    -   id: "kafkaConfig"
+        className: "org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder"
+        constructorArgs:
+          - ref: "kafkaProps"
+          # topic name
+          - "${enrichment.input.topic}"
+          - "${kafka.zk}"
+          - ref: "fields"
+        configMethods:
+            -   name: "setFirstPollOffsetStrategy"
+                args:
+                    - "${kafka.start}"
+
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+        parallelism: ${kafka.spout.parallelism}
+
+bolts:
+# Enrichment Bolts
+    -   id: "enrichmentSplitBolt"
+        className: "org.apache.metron.enrichment.bolt.EnrichmentSplitterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichments"
+                args:
+                    - ref: "enrichments"
+        parallelism: ${enrichment.split.parallelism}
+
+    -   id: "geoEnrichmentBolt"
+        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichment"
+                args:
+                    - ref: "geoEnrichment"
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+
+    -   id: "stellarEnrichmentBolt"
+        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichment"
+                args:
+                    - ref: "stellarEnrichment"
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+        parallelism: ${enrichment.stellar.parallelism}
+
+    -   id: "hostEnrichmentBolt"
+        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichment"
+                args:
+                    - ref: "hostEnrichment"
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+
+    -   id: "simpleHBaseEnrichmentBolt"
+        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichment"
+                args:
+                    - ref: "simpleHBaseEnrichment"
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+
+    -   id: "enrichmentJoinBolt"
+        className: "org.apache.metron.enrichment.bolt.EnrichmentJoinBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withMaxCacheSize"
+                args: [${enrichment.join.cache.size}]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+        parallelism: ${enrichment.join.parallelism}
+
+    -   id: "enrichmentErrorOutputBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withMessageWriter"
+                args:
+                    - ref: "enrichmentErrorKafkaWriter"
+
+
+# Threat Intel Bolts
+    -   id: "threatIntelSplitBolt"
+        className: "org.apache.metron.enrichment.bolt.ThreatIntelSplitterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichments"
+                args:
+                    - ref: "threatIntels"
+            -   name: "withMessageFieldName"
+                args: ["message"]
+        parallelism: ${threat.intel.split.parallelism}
+
+    -   id: "simpleHBaseThreatIntelBolt"
+        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichment"
+                args:
+                    - ref: "simpleHBaseThreatIntelEnrichment"
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+    -   id: "stellarThreatIntelBolt"
+        className: "org.apache.metron.enrichment.bolt.GenericEnrichmentBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withEnrichment"
+                args:
+                    - ref: "stellarThreatIntelEnrichment"
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+        parallelism: ${threat.intel.stellar.parallelism}
+
+    -   id: "threatIntelJoinBolt"
+        className: "org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withMaxCacheSize"
+                args: [${threat.intel.join.cache.size}]
+            -   name: "withMaxTimeRetain"
+                args: [10]
+        parallelism: ${threat.intel.join.parallelism}
+
+    -   id: "threatIntelErrorOutputBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withMessageWriter"
+                args:
+                    - ref: "threatIntelErrorKafkaWriter"
+
+# Indexing Bolts
+    -   id: "outputBolt"
+        className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
+        constructorArgs:
+            - "${kafka.zk}"
+        configMethods:
+            -   name: "withMessageWriter"
+                args:
+                    - ref: "kafkaWriter"
+        parallelism: ${kafka.writer.parallelism}
+
+
+streams:
+#parser
+    -   name: "spout -> enrichmentSplit"
+        from: "kafkaSpout"
+        to: "enrichmentSplitBolt"
+        grouping:
+            type: LOCAL_OR_SHUFFLE
+
+#enrichment
+    -   name: "enrichmentSplit -> host"
+        from: "enrichmentSplitBolt"
+        to: "hostEnrichmentBolt"
+        grouping:
+            streamId: "host"
+            type: FIELDS
+            args: ["message"]
+
+    -   name: "enrichmentSplit -> geo"
+        from: "enrichmentSplitBolt"
+        to: "geoEnrichmentBolt"
+        grouping:
+            streamId: "geo"
+            type: FIELDS
+            args: ["message"]
+
+    -   name: "enrichmentSplit -> stellar"
+        from: "enrichmentSplitBolt"
+        to: "stellarEnrichmentBolt"
+        grouping:
+            streamId: "stellar"
+            type: FIELDS
+            args: ["message"]
+
+
+    -   name: "enrichmentSplit -> simpleHBaseEnrichmentBolt"
+        from: "enrichmentSplitBolt"
+        to: "simpleHBaseEnrichmentBolt"
+        grouping:
+            streamId: "hbaseEnrichment"
+            type: FIELDS
+            args: ["message"]
+
+    -   name: "splitter -> join"
+        from: "enrichmentSplitBolt"
+        to: "enrichmentJoinBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+
+    -   name: "geo -> join"
+        from: "geoEnrichmentBolt"
+        to: "enrichmentJoinBolt"
+        grouping:
+            streamId: "geo"
+            type: FIELDS
+            args: ["key"]
+
+    -   name: "stellar -> join"
+        from: "stellarEnrichmentBolt"
+        to: "enrichmentJoinBolt"
+        grouping:
+            streamId: "stellar"
+            type: FIELDS
+            args: ["key"]
+
+    -   name: "simpleHBaseEnrichmentBolt -> join"
+        from: "simpleHBaseEnrichmentBolt"
+        to: "enrichmentJoinBolt"
+        grouping:
+            streamId: "hbaseEnrichment"
+            type: FIELDS
+            args: ["key"]
+
+    -   name: "host -> join"
+        from: "hostEnrichmentBolt"
+        to: "enrichmentJoinBolt"
+        grouping:
+            streamId: "host"
+            type: FIELDS
+            args: ["key"]
+
+    # Error output
+    -   name: "geoEnrichmentBolt -> enrichmentErrorOutputBolt"
+        from: "geoEnrichmentBolt"
+        to: "enrichmentErrorOutputBolt"
+        grouping:
+            streamId: "error"
+            type: LOCAL_OR_SHUFFLE
+
+    -   name: "stellarEnrichmentBolt -> enrichmentErrorOutputBolt"
+        from: "stellarEnrichmentBolt"
+        to: "enrichmentErrorOutputBolt"
+        grouping:
+            streamId: "error"
+            type: LOCAL_OR_SHUFFLE
+
+    -   name: "hostEnrichmentBolt -> enrichmentErrorOutputBolt"
+        from: "hostEnrichmentBolt"
+        to: "enrichmentErrorOutputBolt"
+        grouping:
+            streamId: "error"
+            type: LOCAL_OR_SHUFFLE
+
+    -   name: "simpleHBaseEnrichmentBolt -> enrichmentErrorOutputBolt"
+        from: "simpleHBaseEnrichmentBolt"
+        to: "enrichmentErrorOutputBolt"
+        grouping:
+            streamId: "error"
+            type: LOCAL_OR_SHUFFLE
+
+#threat intel
+    -   name: "enrichmentJoin -> threatSplit"
+        from: "enrichmentJoinBolt"
+        to: "threatIntelSplitBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+
+    -   name: "threatSplit -> simpleHBaseThreatIntel"
+        from: "threatIntelSplitBolt"
+        to: "simpleHBaseThreatIntelBolt"
+        grouping:
+            streamId: "hbaseThreatIntel"
+            type: FIELDS
+            args: ["message"]
+
+    -   name: "threatSplit -> stellarThreatIntel"
+        from: "threatIntelSplitBolt"
+        to: "stellarThreatIntelBolt"
+        grouping:
+            streamId: "stellar"
+            type: FIELDS
+            args: ["message"]
+
+
+    -   name: "simpleHBaseThreatIntel -> join"
+        from: "simpleHBaseThreatIntelBolt"
+        to: "threatIntelJoinBolt"
+        grouping:
+            streamId: "hbaseThreatIntel"
+            type: FIELDS
+            args: ["key"]
+
+    -   name: "stellarThreatIntel -> join"
+        from: "stellarThreatIntelBolt"
+        to: "threatIntelJoinBolt"
+        grouping:
+            streamId: "stellar"
+            type: FIELDS
+            args: ["key"]
+
+    -   name: "threatIntelSplit -> threatIntelJoin"
+        from: "threatIntelSplitBolt"
+        to: "threatIntelJoinBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+#output
+    -   name: "threatIntelJoin -> output"
+        from: "threatIntelJoinBolt"
+        to: "outputBolt"
+        grouping:
+            streamId: "message"
+            type: LOCAL_OR_SHUFFLE
+
+    # Error output
+    -   name: "simpleHBaseThreatIntelBolt -> threatIntelErrorOutputBolt"
+        from: "simpleHBaseThreatIntelBolt"
+        to: "threatIntelErrorOutputBolt"
+        grouping:
+            streamId: "error"
+            type: LOCAL_OR_SHUFFLE
+
+    -   name: "stellarThreatIntelBolt -> threatIntelErrorOutputBolt"
+        from: "stellarThreatIntelBolt"
+        to: "threatIntelErrorOutputBolt"
+        grouping:
+            streamId: "error"
+            type: LOCAL_OR_SHUFFLE
+

http://git-wip-us.apache.org/repos/asf/metron/blob/82212ba8/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
index ddc5ffc..d7107d9 100644
--- a/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
+++ b/metron-platform/metron-enrichment/src/main/flux/enrichment/remote-unified.yaml
@@ -27,13 +27,12 @@ config:
     topology.worker.childopts: ${topology.worker.childopts}
     topology.auto-credentials: ${topology.auto-credentials}
     topology.max.spout.pending: ${topology.max.spout.pending}
-    # Change this if you want to adjust the threadpool size
-    metron.threadpool.size: "2C" # Either a number (e.g. 5) or multiple of cores (e.g. 5C = 5 times the number of cores)
-    # Change this if you want to adjust the threadpool type
-    metron.threadpool.type: "FIXED" # FIXED or WORK_STEALING
+    metron.threadpool.size: ${enrichment.threadpool.size} # Either a number (e.g. 5) or multiple of cores (e.g. 5C = 5 times the number of cores)
+    metron.threadpool.type: ${enrichment.threadpool.type} # FIXED or WORK_STEALING
+
 components:
 
-# Enrichment
+    # enrichment
     -   id: "stellarEnrichmentAdapter"
         className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
         configMethods:
@@ -41,7 +40,7 @@ components:
                 args:
                     - "ENRICHMENT"
 
-    # Any kafka props for the producer go here.
+    # any kafka props for the producer go here.
     -   id: "kafkaWriterProps"
         className: "java.util.HashMap"
         configMethods:
@@ -58,15 +57,18 @@ components:
 
     -   id: "geoEnrichmentAdapter"
         className: "org.apache.metron.enrichment.adapters.geo.GeoAdapter"
+
     -   id: "geoEnrichment"
         className: "org.apache.metron.enrichment.configuration.Enrichment"
         constructorArgs:
             -   "geo"
             -   ref: "geoEnrichmentAdapter"
+
     -   id: "hostEnrichmentAdapter"
         className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
         constructorArgs:
             - '${enrichment.host.known_hosts}'
+
     -   id: "hostEnrichment"
         className: "org.apache.metron.enrichment.configuration.Enrichment"
         constructorArgs:
@@ -85,17 +87,20 @@ components:
             -   name: "withHBaseCF"
                 args:
                     - "${enrichment.simple.hbase.cf}"
+
     -   id: "simpleHBaseEnrichmentAdapter"
         className: "org.apache.metron.enrichment.adapters.simplehbase.SimpleHBaseAdapter"
         configMethods:
            -    name: "withConfig"
                 args:
                     - ref: "simpleHBaseEnrichmentConfig"
+
     -   id: "simpleHBaseEnrichment"
         className: "org.apache.metron.enrichment.configuration.Enrichment"
         constructorArgs:
           -   "hbaseEnrichment"
           -   ref: "simpleHBaseEnrichmentAdapter"
+
     -   id: "enrichments"
         className: "java.util.ArrayList"
         configMethods:
@@ -112,7 +117,7 @@ components:
                 args:
                     - ref: "stellarEnrichment"
 
-    #enrichment error
+    # enrichment error
     -   id: "enrichmentErrorKafkaWriter"
         className: "org.apache.metron.writer.kafka.KafkaWriter"
         configMethods:
@@ -123,21 +128,23 @@ components:
                 args:
                     - "${kafka.zk}"
             -   name: "withProducerConfigs"
-                args: 
+                args:
                     - ref: "kafkaWriterProps"
 
-# Threat Intel
+    # threat Intel
     -   id: "stellarThreatIntelAdapter"
         className: "org.apache.metron.enrichment.adapters.stellar.StellarAdapter"
         configMethods:
             -   name: "ofType"
                 args:
                     - "THREAT_INTEL"
+
     -   id: "stellarThreatIntelEnrichment"
         className: "org.apache.metron.enrichment.configuration.Enrichment"
         constructorArgs:
             -   "stellar"
             -   ref: "stellarThreatIntelAdapter"
+
     -   id: "simpleHBaseThreatIntelConfig"
         className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelConfig"
         configMethods:
@@ -156,12 +163,14 @@ components:
             -   name: "withHBaseCF"
                 args:
                     - "${threat.intel.simple.hbase.cf}"
+
     -   id: "simpleHBaseThreatIntelAdapter"
         className: "org.apache.metron.enrichment.adapters.threatintel.ThreatIntelAdapter"
         configMethods:
            -    name: "withConfig"
                 args:
                     - ref: "simpleHBaseThreatIntelConfig"
+
     -   id: "simpleHBaseThreatIntelEnrichment"
         className: "org.apache.metron.enrichment.configuration.Enrichment"
         constructorArgs:
@@ -178,7 +187,7 @@ components:
                 args:
                     - ref: "stellarThreatIntelEnrichment"
 
-    #threatintel error
+    # threatintel error
     -   id: "threatIntelErrorKafkaWriter"
         className: "org.apache.metron.writer.kafka.KafkaWriter"
         configMethods:
@@ -189,9 +198,9 @@ components:
                 args:
                     - "${kafka.zk}"
             -   name: "withProducerConfigs"
-                args: 
+                args:
                     - ref: "kafkaWriterProps"
-#indexing
+    # indexing
     -   id: "kafkaWriter"
         className: "org.apache.metron.writer.kafka.KafkaWriter"
         configMethods:
@@ -202,11 +211,11 @@ components:
                 args:
                     - "${kafka.zk}"
             -   name: "withProducerConfigs"
-                args: 
+                args:
                     - ref: "kafkaWriterProps"
 
-#kafka/zookeeper
-    # Any kafka props for the consumer go here.
+    # kafka/zookeeper
+    # any kafka props for the consumer go here.
     -   id: "kafkaProps"
         className: "java.util.HashMap"
         configMethods:
@@ -228,7 +237,7 @@ components:
                   - "${kafka.security.protocol}"
 
 
-  # The fields to pull out of the kafka messages
+    # the fields to pull out of the kafka messages
     -   id: "fields"
         className: "java.util.ArrayList"
         configMethods:
@@ -251,6 +260,7 @@ components:
 
 
 spouts:
+
     -   id: "kafkaSpout"
         className: "org.apache.metron.storm.kafka.flux.StormKafkaSpout"
         constructorArgs:
@@ -258,7 +268,8 @@ spouts:
         parallelism: ${kafka.spout.parallelism}
 
 bolts:
-# Enrichment Bolts
+
+    # enrichment bolt
     -   id: "enrichmentBolt"
         className: "org.apache.metron.enrichment.bolt.UnifiedEnrichmentBolt"
         constructorArgs:
@@ -268,7 +279,7 @@ bolts:
                 args:
                     - ref: "enrichments"
             -   name: "withMaxCacheSize"
-                args: [${enrichment.join.cache.size}]
+                args: [${enrichment.cache.size}]
             -   name: "withMaxTimeRetain"
                 args: [10]
             -   name: "withCaptureCacheStats"
@@ -278,7 +289,7 @@ bolts:
                     - "ENRICHMENT"
             -   name: "withMessageGetter"
                 args: ["JSON_FROM_POSITION"]
-        parallelism: ${enrichment.join.parallelism}
+        parallelism: ${enrichment.parallelism}
 
     -   id: "enrichmentErrorOutputBolt"
         className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
@@ -290,7 +301,7 @@ bolts:
                     - ref: "enrichmentErrorKafkaWriter"
 
 
-# Threat Intel Bolts
+    # threat intel bolts
     -   id: "threatIntelBolt"
         className: "org.apache.metron.enrichment.bolt.UnifiedEnrichmentBolt"
         constructorArgs:
@@ -300,7 +311,7 @@ bolts:
                 args:
                     - ref: "threatIntels"
             -   name: "withMaxCacheSize"
-                args: [${enrichment.join.cache.size}]
+                args: [${threat.intel.cache.size}]
             -   name: "withMaxTimeRetain"
                 args: [10]
             -   name: "withCaptureCacheStats"
@@ -312,7 +323,7 @@ bolts:
                 args: ["message"]
             -   name: "withMessageGetter"
                 args: ["JSON_FROM_FIELD_BY_REFERENCE"]
-        parallelism: ${threat.intel.join.parallelism}
+        parallelism: ${threat.intel.parallelism}
 
     -   id: "threatIntelErrorOutputBolt"
         className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
@@ -323,7 +334,7 @@ bolts:
                 args:
                     - ref: "threatIntelErrorKafkaWriter"
 
-# Indexing Bolts
+    # output bolt
     -   id: "outputBolt"
         className: "org.apache.metron.writer.bolt.BulkMessageWriterBolt"
         constructorArgs:
@@ -334,16 +345,16 @@ bolts:
                     - ref: "kafkaWriter"
         parallelism: ${kafka.writer.parallelism}
 
-
 streams:
-#parser
+
+    # parser
     -   name: "spout -> enrichmentBolt"
         from: "kafkaSpout"
         to: "enrichmentBolt"
         grouping:
             type: LOCAL_OR_SHUFFLE
 
-    # Error output
+    # error output
     -   name: "enrichmentBolt -> enrichmentErrorOutputBolt"
         from: "enrichmentBolt"
         to: "enrichmentErrorOutputBolt"
@@ -351,7 +362,7 @@ streams:
             streamId: "error"
             type: LOCAL_OR_SHUFFLE
 
-#threat intel
+    # threat intel
     -   name: "enrichmentBolt -> threatIntelBolt"
         from: "enrichmentBolt"
         to: "threatIntelBolt"
@@ -359,7 +370,7 @@ streams:
             streamId: "message"
             type: LOCAL_OR_SHUFFLE
 
-#output
+    # output
     -   name: "threatIntelBolt -> output"
         from: "threatIntelBolt"
         to: "outputBolt"
@@ -367,12 +378,10 @@ streams:
             streamId: "message"
             type: LOCAL_OR_SHUFFLE
 
-    # Error output
+    # error output
     -   name: "threatIntelBolt -> threatIntelErrorOutputBolt"
         from: "threatIntelBolt"
         to: "threatIntelErrorOutputBolt"
         grouping:
             streamId: "error"
             type: LOCAL_OR_SHUFFLE
-
-


Mime
View raw message