metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sirs...@apache.org
Subject [3/5] incubator-metron git commit: Merge branch 'flux' of github.com:sirsean/incubator-metron
Date Fri, 22 Jan 2016 18:11:34 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml
new file mode 100644
index 0000000..e1ea080
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/local.yaml
@@ -0,0 +1,185 @@
+name: "bro-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "broParser"
+        className: "org.apache.metron.parsing.parsers.BasicBroParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/BroExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "broParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "bro"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "bro_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "bro_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "bro_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "testingSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/metrics.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/metrics.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/metrics.conf
deleted file mode 100644
index aa7a6e0..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/metrics.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
new file mode 100644
index 0000000..d3e6326
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
@@ -0,0 +1,199 @@
+name: "bro"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "broParser"
+        className: "org.apache.metron.parsing.parsers.BasicBroParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "broParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "bro"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "bro_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "bro_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "bro_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology.conf
deleted file mode 100644
index b0eeb5a..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology.conf
+++ /dev/null
@@ -1,137 +0,0 @@
-include = ../../etc/env/environment_common.conf
-include = ../../etc/env/es_connection.conf
-include = ../../etc/env/hdfs_connection.conf
-include = ../../etc/env/mysql_connection.conf
-include = metrics.conf
-include = features_enabled.conf
-
-#Global Properties
-
-debug.mode=true
-local.mode=true
-num.workers=1
-
-#Standard 5-tuple fields
-
-source.ip=ip_src_addr
-source.port=ip_src_port
-dest.ip=ip_dst_addr
-dest.port=ip_dst_port
-protocol=protocol
-
-#Test Spout
-spout.test.parallelism.repeat=false
-
-#Kafka Spout
-spout.kafka.topic=bro_raw
-
-#Parsing Bolt
-bolt.parser.adapter=org.apache.metron.parsing.parsers.BasicBroParser
-source.include.protocols=snmp,http,ftp,ssh,ssl,dns,socks,dnp3,smtp,dhcp,modbus,radius,irc
-source.exclude.protocols=x509,files,app_stats
-
-#Host Enrichment
-
-bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.host.enrichment_tag=host
-
-
-#GeoEnrichment
-
-bolt.enrichment.geo.enrichment_tag=geo
-bolt.enrichment.geo.adapter.table=GEO
-bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr
-
-#WhoisEnrichment
-
-bolt.enrichment.whois.hbase.table.name=whois
-bolt.enrichment.whois.enrichment_tag=whois
-bolt.enrichment.whois.fields=host,query
-bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10
-
-#CIF Enrichment
-bolt.enrichment.cif.tablename=cif_table
-bolt.enrichment.cif.fields.host=host,query
-bolt.enrichment.cif.fields.email=email
-bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr
-bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.cif.enrichment_tag=cif
-
-#Threat Enrichment
-bolt.enrichment.threat.tablename=threat_table
-bolt.enrichment.threat.fields=host,query,ip_src_addr,ip_dst_addr
-bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.threat.enrichment_tag=threat
-
-#Indexing Bolt
-bolt.indexing.indexname=bro_index
-bolt.indexing.timestamp=yyyy.MM.ww
-bolt.indexing.documentname=bro_doc
-bolt.indexing.bulk=200
-bolt.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Indexing Bolt
-bolt.alerts.indexing.indexname=alert
-bolt.alerts.indexing.documentname=bro_alert
-bolt.alerts.indexing.timestamp=yyyy.MM.ww
-bolt.alerts.indexing.bulk=1
-bolt.alerts.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Error Indexing Bolt
-bolt.error.indexing.indexname=error
-bolt.error.indexing.timestamp=yyyy.MM
-bolt.error.indexing.documentname=bro_error
-bolt.error.indexing.bulk=1
-bolt.error.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Bolt
-
-
-bolt.alerts.adapter=org.apache.metron.alerts.adapters.ThreatAlertsAdapter
-org.apache.metron.alerts.adapters.ThreatAlertsAdapter.enrichment_tag=Threat_Enrichment
-org.apache.metron.alerts.adapters.ThreatAlertsAdapter.whitelist_table_name = ip_whitelist
-org.apache.metron.alerts.adapters.ThreatAlertsAdapter.blacklist_table_name = ip_blacklist
-org.apache.metron.alerts.adapters.ThreatAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3
-org.apache.metron.alerts.adapters.ThreatAlertsAdapter.port=2181
-org.apache.metron.alerts.adapters.ThreatAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600
-org.apache.metron.alerts.adapters.ThreatAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000
-
-
-#bolt.alerts.adapter=org.apache.metron.alerts.adapters.CIFAlertsAdapter
-#org.apache.metron.alerts.adapters.CIFAlertsAdapter.enrichment_tag=CIF_Enrichment
-#org.apache.metron.alerts.adapters.CIFAlertsAdapter.whitelist_table_name = ip_whitelist
-#org.apache.metron.alerts.adapters.CIFAlertsAdapter.blacklist_table_name = ip_blacklist
-#org.apache.metron.alerts.adapters.CIFAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3
-#org.apache.metron.alerts.adapters.CIFAlertsAdapter.port=2181
-#org.apache.metron.alerts.adapters.CIFAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600
-#org.apache.metron.alerts.adapters.CIFAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000
-
-#bolt.alerts.adapter=org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter
-#org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter.whitelist_table_name = ip_whitelist
-#org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter.blacklist_table_name = ip_blacklist
-#org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter.quorum=zkpr1,zkpr2,zkpr3
-#org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter.port=2181
-#org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600
-#org.apache.metron.alerts.adapters.HbaseWhiteAndBlacklistAdapter._MAX_TIME_RETAIN_MINUTES=1000
-
-
-
-
-
-#HDFS Bolt
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url=hdfs://nn1:8020
-bolt.hdfs.wip.file.path=/bro/wip
-bolt.hdfs.finished.file.path=/bro/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-
-#Kafka Bolt
-bolt.kafka.topic=bro_enriched
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology_identifier.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology_identifier.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology_identifier.conf
deleted file mode 100644
index bb72783..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/topology_identifier.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-#Each topology must have a unique identifier.  This setting is required
-
-topology.id=bro
-instance.id=B001
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/environment_identifier.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/environment_identifier.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/environment_identifier.conf
deleted file mode 100644
index 4e8e005..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/environment_identifier.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-#This file identifies the cluster instance
-
-customer.id=mtd
-datacenter.id=allen
-instance.id=dev
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/features_enabled.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/features_enabled.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/features_enabled.conf
deleted file mode 100644
index 5b45dde..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/features_enabled.conf
+++ /dev/null
@@ -1,113 +0,0 @@
-#Enable and disable features for each topology
-
-#Feature: Test spout 
-##Feature Description: Reads telemetry from file and ingests it into topology.  Used for testing or bulk loading the topology
-
-spout.test.name=TestSpout
-spout.test.enabled=true
-spout.test.num.tasks=1
-spout.test.parallelism.hint=1
-
-#Feature: Kafka spout
-##Feature Description: Acts as a Kafka consumer.  Takes messages from a Kafka topic and ingests them into a topology
-
-spout.kafka.name=KafkaSpout
-spout.kafka.enabled=false
-spout.kafka.num.tasks=1
-spout.kafka.parallelism.hint=1
-
-#Feature: Parser Bolt
-##Feature Description: Parses telemetry from its native format into a native JSON
-
-bolt.parser.name=ParserBolt
-bolt.parser.enabled=true
-bolt.parser.num.tasks=1
-bolt.parser.parallelism.hint=1
-
-#Feature: Host Enrichment
-##Feature Description: Appends information about known hosts to a telemetry message
-
-bolt.enrichment.host.name=HostEnrichment
-bolt.enrichment.host.enabled=true
-bolt.enrichment.host.num.tasks=1
-bolt.enrichment.host.parallelism.hint=1
-
-#Feature: Geo Enrichment
-##Feature Description: Appends geo information about known non-local IPs to a telemetry message
-
-bolt.enrichment.geo.name=GeoEnrichment 
-bolt.enrichment.geo.enabled=true
-bolt.enrichment.geo.num.tasks=1
-bolt.enrichment.geo.parallelism.hint=1
-
-#Feature: Whois Enrichment
-##Feature Description: Appends whois information about known domains to a telemetry message
-
-bolt.enrichment.whois.name=WhoisEnrichment
-bolt.enrichment.whois.enabled=false
-bolt.enrichment.whois.num.tasks=1
-bolt.enrichment.whois.parallelism.hint=1
-
-#Feature: CIF Enrichment
-##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message
-
-bolt.enrichment.cif.name=SIFBolt
-bolt.enrichment.cif.enabled=false
-bolt.enrichment.cif.num.tasks=1
-bolt.enrichment.cif.parallelism.hint=1
-
-#Feature: Threat Enrichment
-##Feature Description: Appends information from Threat intelligence feeds to a telemetry message
-
-bolt.enrichment.threat.name=ThreatBolt
-bolt.enrichment.threat.enabled=false
-bolt.enrichment.threat.num.tasks=1
-bolt.enrichment.threat.parallelism.hint=1
-
-#Feature: Rules-Based Alerts
-##Feature Description: Tags messages with rules-based alerts
-
-bolt.alerts.name=Alerts
-bolt.alerts.enabled=true
-bolt.alerts.num.tasks=1
-bolt.alerts.parallelism.hint=1
-
-#Feature: Indexer
-##Feature Description: Indexes telemetry messages in ElasticSearch or Solr
-
-bolt.indexing.name=IndexBolt
-bolt.indexing.enabled=true
-bolt.indexing.num.tasks=1
-bolt.indexing.parallelism.hint=1
-
-#Feature: Alerts Indexer
-##Feature Description: Indexes alert messages in ElasticSearch or Solr
-
-bolt.alerts.indexing.name=AlertIndexBolt
-bolt.alerts.indexing.enabled=true
-bolt.alerts.indexing.num.tasks=1
-bolt.alerts.indexing.parallelism.hint=1
-
-#Feature: Error Indexer
-##Feature Description: Indexes error messages in ElasticSearch or Solr
-
-bolt.error.indexing.name=ErrorIndexBolt
-bolt.error.indexing.enabled=true
-bolt.error.indexing.num.tasks=1
-bolt.error.indexing.parallelism.hint=1
-
-#Feature: Kafka Bolt
-##Feature Description: Writes telemetry messages back into a Kafka topic
-
-bolt.kafka.name=KafkaBolt
-bolt.kafka.enabled=false
-bolt.kafka.num.tasks=1
-bolt.kafka.parallelism.hint=1
-
-#Feature: HDFS Bolt
-##Feature Description: Writes telemetry messages into HDFS
-
-bolt.hdfs.name=HDFSBolt
-bolt.hdfs.enabled=false
-bolt.hdfs.num.tasks=1
-bolt.hdfs.parallelism.hint=1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml
new file mode 100644
index 0000000..3390dd7
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/local.yaml
@@ -0,0 +1,358 @@
+name: "fireeye-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "fireEyeParser"
+        className: "org.apache.metron.parsing.parsers.BasicFireEyeParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "geoKeys"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args: ["ip_src_addr"]
+            -   name: "add"
+                args: ["ip_dst_addr"]
+    -   id: "geoEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.geo.GeoMysqlAdapter"
+        constructorArgs:
+            - "${mysql.ip}"
+            - ${mysql.port}
+            - "${mysql.username}"
+            - "${mysql.password}"
+            - "GEO"
+    -   id: "hostsKeys"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args: ["ip_src_addr"]
+            -   name: "add"
+                args: ["ip_dst_addr"]
+    -   id: "hostEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+        constructorArgs:
+            - '${org.apache.metron.enrichment.host.known_hosts}'
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "alertsConfig"
+        className: "java.util.HashMap"
+        configMethods:
+            -   name: "put"
+                args: ["whitelist_table_name", "ip_whitelist"]
+            -   name: "put"
+                args: ["blacklist_table_name", "ip_blacklist"]
+            -   name: "put"
+                args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
+            -   name: "put"
+                args: ["port", "2181"]
+            -   name: "put"
+                args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
+            -   name: "put"
+                args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
+    -   id: "alertsAdapter"
+        className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
+        constructorArgs:
+            - ref: "alertsConfig"
+    -   id: "alertsIdentifier"
+        className: "org.json.simple.JSONObject"
+        configMethods:
+            -   name: "put"
+                args: ["environment", "local"]
+            -   name: "put"
+                args: ["topology", "fireeye"]
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/FireeyeExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "fireEyeParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "fireeye"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "geoEnrichmentBolt"
+        className: "org.apache.metron.enrichment.common.GenericEnrichmentBolt"
+        configMethods:
+            -   name: "withEnrichmentTag"
+                args: ["geo"]
+            -   name: "withAdapter"
+                args:
+                    - ref: "geoEnrichmentAdapter"
+            -   name: "withMaxTimeRetain"
+                args: [10]
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withOutputFieldName"
+                args: ["fireeye"]
+            -   name: "withKeys"
+                args:
+                    - ref: "geoKeys"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "hostEnrichmentBolt"
+        className: "org.apache.metron.enrichment.common.GenericEnrichmentBolt"
+        configMethods:
+            -   name: "withEnrichmentTag"
+                args: ["host"]
+            -   name: "withAdapter"
+                args:
+                    - ref: "hostEnrichmentAdapter"
+            -   name: "withMaxTimeRetain"
+                args: [10]
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withOutputFieldName"
+                args: ["fireeye"]
+            -   name: "withKeys"
+                args:
+                    - ref: "hostsKeys"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "fireeye_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "fireeye_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsBolt"
+        className: "org.apache.metron.alerts.TelemetryAlertsBolt"
+        configMethods:
+            -   name: "withIdentifier"
+                args:
+                    - ref: "alertsIdentifier"
+            -   name: "withMaxCacheSize"
+                args: [1000]
+            -   name: "withMaxTimeRetain"
+                args: [3600]
+            -   name: "withAlertsAdapter"
+                args:
+                    - ref: "alertsAdapter"
+            -   name: "withOutputFieldName"
+                args: ["message"]
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "alert"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.ww"
+            -   name: "withDocumentName"
+                args:
+                    - "fireeye_alert"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "fireeye_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "testingSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> geo"
+        from: "parserBolt"
+        to: "geoEnrichmentBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "geo -> host"
+        from: "geoEnrichmentBolt"
+        to: "hostEnrichmentBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "host -> alerts"
+        from: "hostEnrichmentBolt"
+        to: "alertsBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "alerts -> alertsIndexing"
+        from: "alertsBolt"
+        to: "alertsIndexingBolt"
+        grouping:
+            streamId: "message"
+            type: SHUFFLE
+    -   name: "alerts -> indexing"
+        from: "alertsBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "alerts -> errors"
+        from: "alertsBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/metrics.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/metrics.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/metrics.conf
deleted file mode 100644
index aa7a6e0..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/metrics.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
new file mode 100644
index 0000000..487333d
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
@@ -0,0 +1,372 @@
+name: "fireeye"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "fireEyeParser"
+        className: "org.apache.metron.parsing.parsers.BasicFireEyeParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "geoKeys"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args: ["ip_src_addr"]
+            -   name: "add"
+                args: ["ip_dst_addr"]
+    -   id: "geoEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.geo.GeoMysqlAdapter"
+        constructorArgs:
+            - "${mysql.ip}"
+            - ${mysql.port}
+            - "${mysql.username}"
+            - "${mysql.password}"
+            - "GEO"
+    -   id: "hostsKeys"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args: ["ip_src_addr"]
+            -   name: "add"
+                args: ["ip_dst_addr"]
+    -   id: "hostEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.host.HostFromJSONListAdapter"
+        constructorArgs:
+            - '${org.apache.metron.enrichment.host.known_hosts}'
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "alertsConfig"
+        className: "java.util.HashMap"
+        configMethods:
+            -   name: "put"
+                args: ["whitelist_table_name", "ip_whitelist"]
+            -   name: "put"
+                args: ["blacklist_table_name", "ip_blacklist"]
+            -   name: "put"
+                args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
+            -   name: "put"
+                args: ["port", "2181"]
+            -   name: "put"
+                args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
+            -   name: "put"
+                args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
+    -   id: "alertsAdapter"
+        className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
+        constructorArgs:
+            - ref: "alertsConfig"
+    -   id: "alertsIdentifier"
+        className: "org.json.simple.JSONObject"
+        configMethods:
+            -   name: "put"
+                args: ["environment", "local"]
+            -   name: "put"
+                args: ["topology", "fireeye"]
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "fireEyeParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "fireeye"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "geoEnrichmentBolt"
+        className: "org.apache.metron.enrichment.common.GenericEnrichmentBolt"
+        configMethods:
+            -   name: "withEnrichmentTag"
+                args: ["geo"]
+            -   name: "withAdapter"
+                args:
+                    - ref: "geoEnrichmentAdapter"
+            -   name: "withMaxTimeRetain"
+                args: [10]
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withOutputFieldName"
+                args: ["fireeye"]
+            -   name: "withKeys"
+                args:
+                    - ref: "geoKeys"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "hostEnrichmentBolt"
+        className: "org.apache.metron.enrichment.common.GenericEnrichmentBolt"
+        configMethods:
+            -   name: "withEnrichmentTag"
+                args: ["host"]
+            -   name: "withAdapter"
+                args:
+                    - ref: "hostEnrichmentAdapter"
+            -   name: "withMaxTimeRetain"
+                args: [10]
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withOutputFieldName"
+                args: ["fireeye"]
+            -   name: "withKeys"
+                args:
+                    - ref: "hostsKeys"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "fireeye_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "fireeye_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsBolt"
+        className: "org.apache.metron.alerts.TelemetryAlertsBolt"
+        configMethods:
+            -   name: "withIdentifier"
+                args:
+                    - ref: "alertsIdentifier"
+            -   name: "withMaxCacheSize"
+                args: [1000]
+            -   name: "withMaxTimeRetain"
+                args: [3600]
+            -   name: "withAlertsAdapter"
+                args:
+                    - ref: "alertsAdapter"
+            -   name: "withOutputFieldName"
+                args: ["message"]
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "alert"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.ww"
+            -   name: "withDocumentName"
+                args:
+                    - "fireeye_alert"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "fireeye_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> geo"
+        from: "parserBolt"
+        to: "geoEnrichmentBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "geo -> host"
+        from: "geoEnrichmentBolt"
+        to: "hostEnrichmentBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "host -> alerts"
+        from: "hostEnrichmentBolt"
+        to: "alertsBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "alerts -> alertsIndexing"
+        from: "alertsBolt"
+        to: "alertsIndexingBolt"
+        grouping:
+            streamId: "message"
+            type: SHUFFLE
+    -   name: "alerts -> indexing"
+        from: "alertsBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "alerts -> errors"
+        from: "alertsBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology.conf
deleted file mode 100644
index 068b366..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology.conf
+++ /dev/null
@@ -1,110 +0,0 @@
-include = ../../etc/env/environment_common.conf
-include = ../../etc/env/es_connection.conf
-include = ../../etc/env/hdfs_connection.conf
-include = ../../etc/env/mysql_connection.conf
-include = metrics.conf
-include = features_enabled.conf
-
-#Global Properties
-
-debug.mode=true
-local.mode=true
-num.workers=1
-
-#Standard 5-tuple fields
-
-source.ip=ip_src_addr
-source.port=ip_src_port
-dest.ip=ip_dst_addr
-dest.port=ip_dst_port
-protocol=protocol
-
-#Test Spout
-spout.test.parallelism.repeat=false
-
-#Kafka Spout
-spout.kafka.topic=fireeye_raw
-
-#Parser Bolt
-bolt.parser.adapter=org.apache.metron.parsing.parsers.BasicFireEyeParser
-
-#Host Enrichment
-
-bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.host.enrichment_tag=host
-
-
-#GeoEnrichment
-
-bolt.enrichment.geo.enrichment_tag=geo
-bolt.enrichment.geo.adapter.table=GEO
-bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr
-
-#WhoisEnrichment
-
-bolt.enrichment.whois.hbase.table.name=whois
-bolt.enrichment.whois.enrichment_tag=whois
-bolt.enrichment.whois.fields=host
-bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10
-
-#CIF Enrichment
-bolt.enrichment.cif.tablename=cif_table
-bolt.enrichment.cif.fields.host=host
-bolt.enrichment.cif.fields.email=email
-bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr
-bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.cif.enrichment_tag=cif
-
-#Threat Enrichment
-bolt.enrichment.threat.tablename=threat_table
-bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr
-bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.threat.enrichment_tag=threat
-
-#Indexing Bolt
-bolt.indexing.indexname=fireeye_index
-bolt.indexing.timestamp=yyyy.MM.ww
-bolt.indexing.documentname=fireeye_doc
-bolt.indexing.bulk=1
-bolt.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Indexing Bolt
-bolt.alerts.indexing.indexname=alert
-bolt.alerts.indexing.timestamp=yyyy.MM.ww
-bolt.alerts.indexing.documentname=fireeye_alert
-bolt.alerts.indexing.bulk=1
-bolt.alerts.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Error Indexing Bolt
-bolt.error.indexing.indexname=error
-bolt.error.indexing.timestamp=yyyy.MM
-bolt.error.indexing.documentname=fireeye_error
-bolt.error.indexing.bulk=1
-bolt.error.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Bolt
-bolt.alerts.adapter=org.apache.metron.alerts.adapters.CIFAlertsAdapter
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.whitelist_table_name = ip_whitelist
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.blacklist_table_name = ip_blacklist
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.port=2181
-org.apache.metron.alerts.adapters.CIFAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600
-org.apache.metron.alerts.adapters.CIFAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000
-
-#HDFS Bolt
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url=hdfs://nn1:8020
-bolt.hdfs.wip.file.path=/fireeye/wip
-bolt.hdfs.finished.file.path=/fireeye/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-
-#Kafka Bolt
-bolt.kafka.topic=fireeye_enriched
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology_identifier.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology_identifier.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology_identifier.conf
deleted file mode 100644
index 3f1e560..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/topology_identifier.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-#Each topology must have a unique identifier.  This setting is required
-
-topology.id=fireeye
-instance.id=FE001
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/alerts.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/alerts.xml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/alerts.xml
deleted file mode 100644
index f36b881..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/alerts.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1" ?>
-<rule-definitions>
-	<rule>
-		<pattern>.*message.*</pattern>
-		<alert>{"type":"alert","priority":5, "title":"ISE Alert", "body":
-			"Alert triggered by ISE"}
-		</alert>
-	</rule>
-</rule-definitions>
-
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/features_enabled.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/features_enabled.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/features_enabled.conf
deleted file mode 100644
index 730935d..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/features_enabled.conf
+++ /dev/null
@@ -1,113 +0,0 @@
-#Enable and disable features for each topology
-
-#Feature: Test spout 
-##Feature Description: Reads telemetry from file and ingests it into topology.  Used for testing or bulk loading the topology
-
-spout.test.name=TestSpout
-spout.test.enabled=true
-spout.test.num.tasks=1
-spout.test.parallelism.hint=1
-
-#Feature: Kafka spout
-##Feature Description: Acts as a Kafka consumer.  Takes messages from a Kafka topic and ingests them into a topology
-
-spout.kafka.name=KafkaSpout
-spout.kafka.enabled=false
-spout.kafka.num.tasks=1
-spout.kafka.parallelism.hint=1
-
-#Feature: Parser Bolt
-##Feature Description: Parses telemetry from its native format into a native JSON
-
-bolt.parser.name=ParserBolt
-bolt.parser.enabled=true
-bolt.parser.num.tasks=1
-bolt.parser.parallelism.hint=1
-
-#Feature: Host Enrichment
-##Feature Description: Appends information about known hosts to a telemetry message
-
-bolt.enrichment.host.name=HostEnrichment
-bolt.enrichment.host.enabled=false
-bolt.enrichment.host.num.tasks=1
-bolt.enrichment.host.parallelism.hint=1
-
-#Feature: Geo Enrichment
-##Feature Description: Appends geo information about known non-local IPs to a telemetry message
-
-bolt.enrichment.geo.name=GeoEnrichment 
-bolt.enrichment.geo.enabled=false
-bolt.enrichment.geo.num.tasks=1
-bolt.enrichment.geo.parallelism.hint=1
-
-#Feature: Whois Enrichment
-##Feature Description: Appends whois information about known domains to a telemetry message
-
-bolt.enrichment.whois.name=WhoisEnrichment
-bolt.enrichment.whois.enabled=false
-bolt.enrichment.whois.num.tasks=1
-bolt.enrichment.whois.parallelism.hint=1
-
-#Feature: CIF Enrichment
-##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message
-
-bolt.enrichment.cif.name=SIFBolt
-bolt.enrichment.cif.enabled=false
-bolt.enrichment.cif.num.tasks=1
-bolt.enrichment.cif.parallelism.hint=1
-
-#Feature: Threat Enrichment
-##Feature Description: Appends information from Threat intelligence feeds to a telemetry message
-
-bolt.enrichment.threat.name=ThreatBolt
-bolt.enrichment.threat.enabled=false
-bolt.enrichment.threat.num.tasks=1
-bolt.enrichment.threat.parallelism.hint=1
-
-#Feature: Rules-Based Alerts
-##Feature Description: Tags messages with rules-based alerts
-
-bolt.alerts.name=Alerts
-bolt.alerts.enabled=false
-bolt.alerts.num.tasks=1
-bolt.alerts.parallelism.hint=1
-
-#Feature: Indexer
-##Feature Description: Indexes telemetry messages in ElasticSearch or Solr
-
-bolt.indexing.name=IndexBolt
-bolt.indexing.enabled=true
-bolt.indexing.num.tasks=1
-bolt.indexing.parallelism.hint=1
-
-#Feature: Alerts Indexer
-##Feature Description: Indexes alert messages in ElasticSearch or Solr
-
-bolt.alerts.indexing.name=AlertIndexBolt
-bolt.alerts.indexing.enabled=true
-bolt.alerts.indexing.num.tasks=1
-bolt.alerts.indexing.parallelism.hint=1
-
-#Feature: Error Indexer
-##Feature Description: Indexes error messages in ElasticSearch or Solr
-
-bolt.error.indexing.name=ErrorIndexBolt
-bolt.error.indexing.enabled=true
-bolt.error.indexing.num.tasks=1
-bolt.error.indexing.parallelism.hint=1
-
-#Feature: Kafka Bolt
-##Feature Description: Writes telemetry messages back into a Kafka topic
-
-bolt.kafka.name=KafkaBolt
-bolt.kafka.enabled=true
-bolt.kafka.num.tasks=1
-bolt.kafka.parallelism.hint=1
-
-#Feature: HDFS Bolt
-##Feature Description: Writes telemetry messages into HDFS
-
-bolt.hdfs.name=HDFSBolt
-bolt.hdfs.enabled=false
-bolt.hdfs.num.tasks=1
-bolt.hdfs.parallelism.hint=1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml
new file mode 100644
index 0000000..9722413
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/local.yaml
@@ -0,0 +1,185 @@
+name: "ise-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "iseParser"
+        className: "org.apache.metron.parsing.parsers.BasicIseParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/ISESampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "iseParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "ise"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "ise_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "ise_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "ise_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "testingSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/metrics.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/metrics.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/metrics.conf
deleted file mode 100644
index aa7a6e0..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/metrics.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
new file mode 100644
index 0000000..6180f02
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
@@ -0,0 +1,199 @@
+name: "ise"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "iseParser"
+        className: "org.apache.metron.parsing.parsers.BasicIseParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "iseParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "ise"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "ise_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "ise_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "ise_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE


Mime
View raw message