metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sirs...@apache.org
Subject [2/5] incubator-metron git commit: Merge branch 'flux' of github.com:sirsean/incubator-metron
Date Fri, 22 Jan 2016 18:11:33 GMT
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology.conf
deleted file mode 100644
index a563528..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology.conf
+++ /dev/null
@@ -1,101 +0,0 @@
-include = ../../etc/env/environment_common.conf
-include = ../../etc/env/es_connection.conf
-include = ../../etc/env/hdfs_connection.conf
-include = ../../etc/env/mysql_connection.conf
-include = metrics.conf
-include = features_enabled.conf
-
-#Global Properties
-
-debug.mode=true
-local.mode=true
-num.workers=1
-
-#Standard 5-tuple fields
-
-source.ip=ip_src_addr
-source.port=ip_src_port
-dest.ip=ip_dst_addr
-dest.port=ip_dst_port
-protocol=protocol
-
-#Test Spout
-spout.test.parallelism.repeat=false
-
-#Kafka Spout
-spout.kafka.topic=ise_raw
-
-#Parser Bolt
-bolt.parser.adapter=org.apache.metron.parsing.parsers.BasicIseParser
-
-#Host Enrichment
-
-bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.host.enrichment_tag=host
-
-
-#GeoEnrichment
-
-bolt.enrichment.geo.enrichment_tag=geo
-bolt.enrichment.geo.adapter.table=GEO
-bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr
-
-#WhoisEnrichment
-
-bolt.enrichment.whois.hbase.table.name=whois
-bolt.enrichment.whois.enrichment_tag=whois
-bolt.enrichment.whois.fields=host
-bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10
-
-#CIF Enrichment
-bolt.enrichment.cif.tablename=cif_table
-bolt.enrichment.cif.fields.host=host
-bolt.enrichment.cif.fields.email=email
-bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr
-bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.cif.enrichment_tag=cif
-
-#Threat Enrichment
-bolt.enrichment.threat.tablename=threat_table
-bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr
-bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.threat.enrichment_tag=threat
-
-#Indexing Bolt
-bolt.indexing.indexname=ise_index
-bolt.indexing.timestamp=yyyy.MM.ww
-bolt.indexing.documentname=ise_doc
-bolt.indexing.bulk=200
-bolt.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Indexing Bolt
-bolt.alerts.indexing.indexname=alert
-bolt.alerts.indexing.timestamp=yyyy.MM.ww
-bolt.alerts.indexing.documentname=ise_alert
-bolt.alerts.indexing.bulk=1
-bolt.alerts.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Error Indexing Bolt
-bolt.error.indexing.indexname=error
-bolt.error.indexing.timesatmp=yyyy.MM
-bolt.error.indexing.documentname=ise_error
-bolt.error.indexing.bulk=1
-bolt.error.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#HDFS Bolt
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url=hdfs://nn1:8020
-bolt.hdfs.wip.file.path=/ise/wip
-bolt.hdfs.finished.file.path=/ise/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-
-#Kafka Bolt
-bolt.kafka.topic=ise_enriched
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology_identifier.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology_identifier.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology_identifier.conf
deleted file mode 100644
index c500e9f..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/topology_identifier.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-#Each topology must have a unique identifier.  This setting is required
-
-topology.id=ise
-instance.id=I001
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/alerts.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/alerts.xml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/alerts.xml
deleted file mode 100644
index 368f1c0..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/alerts.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="ISO-8859-1" ?>
-<rule-definitions>
-	<rule>
-		<pattern>.*message.*</pattern>
-		<alert>{"type":"alert","priority":5, "title":"Lancope Alert", "body":
-			"Alert triggered by Lancope"}
-		</alert>
-	</rule>
-</rule-definitions>
-
-

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/features_enabled.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/features_enabled.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/features_enabled.conf
deleted file mode 100644
index a4dc14d..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/features_enabled.conf
+++ /dev/null
@@ -1,113 +0,0 @@
-#Enable and disable features for each topology
-
-#Feature: Test spout 
-##Feature Description: Reads telemetry from file and ingests it into topology.  Used for testing or bulk loading the topology
-
-spout.test.name=TestSpout
-spout.test.enabled=true
-spout.test.num.tasks=1
-spout.test.parallelism.hint=1
-
-#Feature: Kafka spout
-##Feature Description: Acts as a Kafka consumer.  Takes messages from a Kafka topic and ingests them into a topology
-
-spout.kafka.name=KafkaSpout
-spout.kafka.enabled=false
-spout.kafka.num.tasks=1
-spout.kafka.parallelism.hint=1
-
-#Feature: Parser Bolt
-##Feature Description: Parses telemetry from its native format into a native JSON
-
-bolt.parser.name=ParserBolt
-bolt.parser.enabled=true
-bolt.parser.num.tasks=1
-bolt.parser.parallelism.hint=1
-
-#Feature: Host Enrichment
-##Feature Description: Appends information about known hosts to a telemetry message
-
-bolt.enrichment.host.name=HostEnrichment
-bolt.enrichment.host.enabled=false
-bolt.enrichment.host.num.tasks=1
-bolt.enrichment.host.parallelism.hint=1
-
-#Feature: Geo Enrichment
-##Feature Description: Appends geo information about known non-local IPs to a telemetry message
-
-bolt.enrichment.geo.name=GeoEnrichment 
-bolt.enrichment.geo.enabled=true
-bolt.enrichment.geo.num.tasks=1
-bolt.enrichment.geo.parallelism.hint=1
-
-#Feature: Whois Enrichment
-##Feature Description: Appends whois information about known domains to a telemetry message
-
-bolt.enrichment.whois.name=WhoisEnrichment
-bolt.enrichment.whois.enabled=false
-bolt.enrichment.whois.num.tasks=1
-bolt.enrichment.whois.parallelism.hint=1
-
-#Feature: CIF Enrichment
-##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message
-
-bolt.enrichment.cif.name=SIFBolt
-bolt.enrichment.cif.enabled=false
-bolt.enrichment.cif.num.tasks=1
-bolt.enrichment.cif.parallelism.hint=1
-
-#Feature: Threat Enrichment
-##Feature Description: Appends information from Threat intelligence feeds to a telemetry message
-
-bolt.enrichment.threat.name=ThreatBolt
-bolt.enrichment.threat.enabled=false
-bolt.enrichment.threat.num.tasks=1
-bolt.enrichment.threat.parallelism.hint=1
-
-#Feature: Rules-Based Alerts
-##Feature Description: Tags messages with rules-based alerts
-
-bolt.alerts.name=Alerts
-bolt.alerts.enabled=true
-bolt.alerts.num.tasks=1
-bolt.alerts.parallelism.hint=1
-
-#Feature: Indexer
-##Feature Description: Indexes telemetry messages in ElasticSearch or Solr
-
-bolt.indexing.name=IndexBolt
-bolt.indexing.enabled=true
-bolt.indexing.num.tasks=1
-bolt.indexing.parallelism.hint=1
-
-#Feature: Alerts Indexer
-##Feature Description: Indexes alert messages in ElasticSearch or Solr
-
-bolt.alerts.indexing.name=AlertIndexBolt
-bolt.alerts.indexing.enabled=true
-bolt.alerts.indexing.num.tasks=1
-bolt.alerts.indexing.parallelism.hint=1
-
-#Feature: Error Indexer
-##Feature Description: Indexes error messages in ElasticSearch or Solr
-
-bolt.error.indexing.name=ErrorIndexBolt
-bolt.error.indexing.enabled=true
-bolt.error.indexing.num.tasks=1
-bolt.error.indexing.parallelism.hint=1
-
-#Feature: Kafka Bolt
-##Feature Description: Writes telemetry messages back into a Kafka topic
-
-bolt.kafka.name=KafkaBolt
-bolt.kafka.enabled=true
-bolt.kafka.num.tasks=1
-bolt.kafka.parallelism.hint=1
-
-#Feature: HDFS Bolt
-##Feature Description: Writes telemetry messages into HDFS
-
-bolt.hdfs.name=HDFSBolt
-bolt.hdfs.enabled=false
-bolt.hdfs.num.tasks=1
-bolt.hdfs.parallelism.hint=1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml
new file mode 100644
index 0000000..f83924a
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/local.yaml
@@ -0,0 +1,320 @@
+name: "lancope-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "lancopeParser"
+        className: "org.apache.metron.parsing.parsers.BasicLancopeParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "geoKeys"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args: ["ip_src_addr"]
+            -   name: "add"
+                args: ["ip_dst_addr"]
+    -   id: "geoEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.geo.GeoMysqlAdapter"
+        constructorArgs:
+            - "${mysql.ip}"
+            - ${mysql.port}
+            - "${mysql.username}"
+            - "${mysql.password}"
+            - "GEO"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "alertsConfig"
+        className: "java.util.HashMap"
+        configMethods:
+            -   name: "put"
+                args: ["whitelist_table_name", "ip_whitelist"]
+            -   name: "put"
+                args: ["blacklist_table_name", "ip_blacklist"]
+            -   name: "put"
+                args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
+            -   name: "put"
+                args: ["port", "2181"]
+            -   name: "put"
+                args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
+            -   name: "put"
+                args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
+    -   id: "alertsAdapter"
+        className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
+        constructorArgs:
+            - ref: "alertsConfig"
+    -   id: "alertsIdentifier"
+        className: "org.json.simple.JSONObject"
+        configMethods:
+            -   name: "put"
+                args: ["environment", "local"]
+            -   name: "put"
+                args: ["topology", "lancope"]
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/LancopeExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "lancopeParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "lancope"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "geoEnrichmentBolt"
+        className: "org.apache.metron.enrichment.common.GenericEnrichmentBolt"
+        configMethods:
+            -   name: "withEnrichmentTag"
+                args: ["geo"]
+            -   name: "withAdapter"
+                args:
+                    - ref: "geoEnrichmentAdapter"
+            -   name: "withMaxTimeRetain"
+                args: [10]
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withOutputFieldName"
+                args: ["lancope"]
+            -   name: "withKeys"
+                args:
+                    - ref: "geoKeys"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "lancope_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "lancope_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsBolt"
+        className: "org.apache.metron.alerts.TelemetryAlertsBolt"
+        configMethods:
+            -   name: "withIdentifier"
+                args:
+                    - ref: "alertsIdentifier"
+            -   name: "withMaxCacheSize"
+                args: [1000]
+            -   name: "withMaxTimeRetain"
+                args: [3600]
+            -   name: "withAlertsAdapter"
+                args:
+                    - ref: "alertsAdapter"
+            -   name: "withOutputFieldName"
+                args: ["message"]
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "alert"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.ww"
+            -   name: "withDocumentName"
+                args:
+                    - "lancope_alert"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "lancope_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "testingSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> geo"
+        from: "parserBolt"
+        to: "geoEnrichmentBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "geo -> alerts"
+        from: "geoEnrichmentBolt"
+        to: "alertsBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "alerts -> alertsIndexing"
+        from: "alertsBolt"
+        to: "alertsIndexingBolt"
+        grouping:
+            streamId: "message"
+            type: SHUFFLE
+    -   name: "alerts -> indexing"
+        from: "alertsBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "alerts -> errors"
+        from: "alertsBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/metrics.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/metrics.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/metrics.conf
deleted file mode 100644
index aa7a6e0..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/metrics.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
new file mode 100644
index 0000000..75a3180
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
@@ -0,0 +1,334 @@
+name: "lancope"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "lancopeParser"
+        className: "org.apache.metron.parsing.parsers.BasicLancopeParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "geoKeys"
+        className: "java.util.ArrayList"
+        configMethods:
+            -   name: "add"
+                args: ["ip_src_addr"]
+            -   name: "add"
+                args: ["ip_dst_addr"]
+    -   id: "geoEnrichmentAdapter"
+        className: "org.apache.metron.enrichment.adapters.geo.GeoMysqlAdapter"
+        constructorArgs:
+            - "${mysql.ip}"
+            - ${mysql.port}
+            - "${mysql.username}"
+            - "${mysql.password}"
+            - "GEO"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "alertsConfig"
+        className: "java.util.HashMap"
+        configMethods:
+            -   name: "put"
+                args: ["whitelist_table_name", "ip_whitelist"]
+            -   name: "put"
+                args: ["blacklist_table_name", "ip_blacklist"]
+            -   name: "put"
+                args: ["quorum", "mon.cluster2.ctolab.hortonworks.com, nn1.cluster2.ctolab.hortonworks.com, nn2.cluster2.ctolab.hortonworks.com"]
+            -   name: "put"
+                args: ["port", "2181"]
+            -   name: "put"
+                args: ["_MAX_CACHE_SIZE_OBJECTS_NUM", "3600"]
+            -   name: "put"
+                args: ["_MAX_TIME_RETAIN_MINUTES", "1000"]
+    -   id: "alertsAdapter"
+        className: "org.apache.metron.alerts.adapters.CIFAlertsAdapter"
+        constructorArgs:
+            - ref: "alertsConfig"
+    -   id: "alertsIdentifier"
+        className: "org.json.simple.JSONObject"
+        configMethods:
+            -   name: "put"
+                args: ["environment", "local"]
+            -   name: "put"
+                args: ["topology", "lancope"]
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "lancopeParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "lancope"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "geoEnrichmentBolt"
+        className: "org.apache.metron.enrichment.common.GenericEnrichmentBolt"
+        configMethods:
+            -   name: "withEnrichmentTag"
+                args: ["geo"]
+            -   name: "withAdapter"
+                args:
+                    - ref: "geoEnrichmentAdapter"
+            -   name: "withMaxTimeRetain"
+                args: [10]
+            -   name: "withMaxCacheSize"
+                args: [10000]
+            -   name: "withOutputFieldName"
+                args: ["lancope"]
+            -   name: "withKeys"
+                args:
+                    - ref: "geoKeys"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "lancope_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "lancope_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsBolt"
+        className: "org.apache.metron.alerts.TelemetryAlertsBolt"
+        configMethods:
+            -   name: "withIdentifier"
+                args:
+                    - ref: "alertsIdentifier"
+            -   name: "withMaxCacheSize"
+                args: [1000]
+            -   name: "withMaxTimeRetain"
+                args: [3600]
+            -   name: "withAlertsAdapter"
+                args:
+                    - ref: "alertsAdapter"
+            -   name: "withOutputFieldName"
+                args: ["message"]
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "alertsIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "alert"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.ww"
+            -   name: "withDocumentName"
+                args:
+                    - "lancope_alert"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "lancope_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> geo"
+        from: "parserBolt"
+        to: "geoEnrichmentBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "geo -> alerts"
+        from: "geoEnrichmentBolt"
+        to: "alertsBolt"
+        grouping:
+            type: FIELDS
+            streamId: "message"
+            args: ["key"]
+    -   name: "alerts -> alertsIndexing"
+        from: "alertsBolt"
+        to: "alertsIndexingBolt"
+        grouping:
+            streamId: "message"
+            type: SHUFFLE
+    -   name: "alerts -> indexing"
+        from: "alertsBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "alerts -> errors"
+        from: "alertsBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology.conf
deleted file mode 100644
index 6551c3f..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology.conf
+++ /dev/null
@@ -1,101 +0,0 @@
-include = ../../etc/env/environment_common.conf
-include = ../../etc/env/es_connection.conf
-include = ../../etc/env/hdfs_connection.conf
-include = ../../etc/env/mysql_connection.conf
-include = metrics.conf
-include = features_enabled.conf
-
-#Global Properties
-
-debug.mode=true
-local.mode=true
-num.workers=1
-
-#Standard 5-tuple fields
-
-source.ip=ip_src_addr
-source.port=ip_src_port
-dest.ip=ip_dst_addr
-dest.port=ip_dst_port
-protocol=protocol
-
-#Test Spout
-spout.test.parallelism.repeat=false
-
-#Kafka Spout
-spout.kafka.topic=lancope_raw
-
-#Parser Bolt
-bolt.parser.adapter=org.apache.metron.parsing.parsers.BasicLancopeParser
-
-#Host Enrichment
-
-bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.host.enrichment_tag=host
-
-
-#GeoEnrichment
-
-bolt.enrichment.geo.enrichment_tag=geo
-bolt.enrichment.geo.adapter.table=GEO
-bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr
-
-#WhoisEnrichment
-
-bolt.enrichment.whois.hbase.table.name=whois
-bolt.enrichment.whois.enrichment_tag=whois
-bolt.enrichment.whois.fields=host
-bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10
-
-#CIF Enrichment
-bolt.enrichment.cif.tablename=cif_table
-bolt.enrichment.cif.fields.host=host
-bolt.enrichment.cif.fields.email=email
-bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr
-bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.cif.enrichment_tag=cif
-
-#Threat Enrichment
-bolt.enrichment.threat.tablename=threat_table
-bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr
-bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.threat.enrichment_tag=threat
-
-#Indexing Bolt
-bolt.indexing.indexname=lancope_index
-bolt.indexing.timestamp=yyyy.MM.ww
-bolt.indexing.documentname=lancope_doc
-bolt.indexing.bulk=200
-bolt.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Indexing Bolt
-bolt.alerts.indexing.indexname=alert
-bolt.alerts.indexing.timestamp=yyyy.MM.ww
-bolt.alerts.indexing.documentname=lancope_alert
-bolt.alerts.indexing.bulk=1
-bolt.alerts.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Error Indexing Bolt
-bolt.error.indexing.indexname=error
-bolt.error.indexing.timestamp=yyyy.MM
-bolt.error.indexing.documentname=lancope_error
-bolt.error.indexing.bulk=1
-bolt.error.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#HDFS Bolt
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url=hdfs://nn1:8020
-bolt.hdfs.wip.file.path=/lancope/wip
-bolt.hdfs.finished.file.path=/lancope/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-
-#Kafka Bolt
-bolt.kafka.topic=lancope_enriched
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology_identifier.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology_identifier.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology_identifier.conf
deleted file mode 100644
index a68084e..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/topology_identifier.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-#Each topology must have a unique identifier.  This setting is required
-
-topology.id=lancope
-instance.id=L001
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/features_enabled.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/features_enabled.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/features_enabled.conf
deleted file mode 100644
index 29ea06d..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/features_enabled.conf
+++ /dev/null
@@ -1,113 +0,0 @@
-#Enable and disable features for each topology
-
-#Feature: Test spout 
-##Feature Description: Reads telemetry from file and ingests it into topology.  Used for testing or bulk loading the topology
-
-spout.test.name=TestSpout
-spout.test.enabled=true
-spout.test.num.tasks=1
-spout.test.parallelism.hint=1
-
-#Feature: Kafka spout
-##Feature Description: Acts as a Kafka consumer.  Takes messages from a Kafka topic and ingests them into a topology
-
-spout.kafka.name=KafkaSpout
-spout.kafka.enabled=false
-spout.kafka.num.tasks=1
-spout.kafka.parallelism.hint=1
-
-#Feature: Parser Bolt
-##Feature Description: Parses telemetry from its native format into a native JSON
-
-bolt.parser.name=ParserBolt
-bolt.parser.enabled=true
-bolt.parser.num.tasks=1
-bolt.parser.parallelism.hint=1
-
-#Feature: Host Enrichment
-##Feature Description: Appends information about known hosts to a telemetry message
-
-bolt.enrichment.host.name=HostEnrichment
-bolt.enrichment.host.enabled=true
-bolt.enrichment.host.num.tasks=1
-bolt.enrichment.host.parallelism.hint=1
-
-#Feature: Geo Enrichment
-##Feature Description: Appends geo information about known non-local IPs to a telemetry message
-
-bolt.enrichment.geo.name=GeoEnrichment 
-bolt.enrichment.geo.enabled=true
-bolt.enrichment.geo.num.tasks=1
-bolt.enrichment.geo.parallelism.hint=1
-
-#Feature: Whois Enrichment
-##Feature Description: Appends whois information about known domains to a telemetry message
-
-bolt.enrichment.whois.name=WhoisEnrichment
-bolt.enrichment.whois.enabled=true
-bolt.enrichment.whois.num.tasks=1
-bolt.enrichment.whois.parallelism.hint=1
-
-#Feature: CIF Enrichment
-##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message
-
-bolt.enrichment.cif.name=CIFBolt
-bolt.enrichment.cif.enabled=true
-bolt.enrichment.cif.num.tasks=1
-bolt.enrichment.cif.parallelism.hint=1
-
-#Feature: Threat Enrichment
-##Feature Description: Appends information from Threat intelligence feeds to a telemetry message
-
-bolt.enrichment.threat.name=ThreatBolt
-bolt.enrichment.threat.enabled=false
-bolt.enrichment.threat.num.tasks=1
-bolt.enrichment.threat.parallelism.hint=1
-
-#Feature: Rules-Based Alerts
-##Feature Description: Tags messages with rules-based alerts
-
-bolt.alerts.name=Alerts
-bolt.alerts.enabled=true
-bolt.alerts.num.tasks=1
-bolt.alerts.parallelism.hint=1
-
-#Feature: Indexer
-##Feature Description: Indexes telemetry messages in ElasticSearch or Solr
-
-bolt.indexing.name=IndexBolt
-bolt.indexing.enabled=true
-bolt.indexing.num.tasks=1
-bolt.indexing.parallelism.hint=1
-
-#Feature: Alerts Indexer
-##Feature Description: Indexes alert messages in ElasticSearch or Solr
-
-bolt.alerts.indexing.name=AlertIndexBolt
-bolt.alerts.indexing.enabled=true
-bolt.alerts.indexing.num.tasks=1
-bolt.alerts.indexing.parallelism.hint=1
-
-#Feature: Error Indexer
-##Feature Description: Indexes error messages in ElasticSearch or Solr
-
-bolt.error.indexing.name=ErrorIndexBolt
-bolt.error.indexing.enabled=true
-bolt.error.indexing.num.tasks=1
-bolt.error.indexing.parallelism.hint=1
-
-#Feature: Kafka Bolt
-##Feature Description: Writes telemetry messages back into a Kafka topic
-
-bolt.kafka.name=KafkaBolt
-bolt.kafka.enabled=false
-bolt.kafka.num.tasks=1
-bolt.kafka.parallelism.hint=1
-
-#Feature: HDFS Bolt
-##Feature Description: Writes telemetry messages into HDFS
-
-bolt.hdfs.name=HDFSBolt
-bolt.hdfs.enabled=false
-bolt.hdfs.num.tasks=1
-bolt.hdfs.parallelism.hint=1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml
new file mode 100644
index 0000000..717b7cf
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/local.yaml
@@ -0,0 +1,165 @@
+name: "paloalto-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "paloAltoParser"
+        className: "org.apache.metron.parsing.parsers.BasicPaloAltoFirewallParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "filenameFormat"
+        className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
+        configMethods:
+            -   name: "withPath"
+                args:
+                    - "${bolt.hdfs.wip.file.path}"
+    -   id: "messageField"
+        className: "backtype.storm.tuple.Fields"
+        constructorArgs:
+            - ["message"]
+    -   id: "recordFormat"
+        className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat"
+        configMethods:
+            -   name: "withFieldDelimiter"
+                args:
+                    - "${bolt.hdfs.field.delimiter}"
+            -   name: "withFields"
+                args:
+                    - ref: "messageField"
+    -   id: "rotationPolicy"
+        className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"
+        constructorArgs:
+            - ${bolt.hdfs.file.rotation.size.in.mb}
+            - MB
+    -   id: "syncPolicy"
+        className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy"
+        constructorArgs:
+            - ${bolt.hdfs.batch.size}
+    -   id: "moveFileAction"
+        className: "org.apache.storm.hdfs.common.rotation.MoveFileAction"
+        configMethods:
+            -   name: "toDestination"
+                args:
+                    - "${bolt.hdfs.finished.file.path}"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/PaloaltoOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "paloAltoParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "paloalto"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "hdfsBolt"
+        className: "org.apache.storm.hdfs.bolt.HdfsBolt"
+        configMethods:
+            -   name: "withFsUrl"
+                args:
+                    - "${bolt.hdfs.file.system.url}"
+            -   name: "withFileNameFormat"
+                args:
+                    - ref: "filenameFormat"
+            -   name: "withRecordFormat"
+                args:
+                    - ref: "recordFormat"
+            -   name: "withRotationPolicy"
+                args:
+                    - ref: "rotationPolicy"
+            -   name: "withSyncPolicy"
+                args:
+                    - ref: "syncPolicy"
+            -   name: "addRotationAction"
+                args:
+                    - ref: "moveFileAction"
+
+streams:
+    -   name: "spout -> parser"
+        from: "testingSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> hdfs"
+        from: "parserBolt"
+        to: "hdfsBolt"
+        grouping:
+            streamId: "message"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/metrics.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/metrics.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/metrics.conf
deleted file mode 100644
index aa7a6e0..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/metrics.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
new file mode 100644
index 0000000..2c0b928
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
@@ -0,0 +1,179 @@
+name: "paloalto"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "paloAltoParser"
+        className: "org.apache.metron.parsing.parsers.BasicPaloAltoFirewallParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "filenameFormat"
+        className: "org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat"
+        configMethods:
+            -   name: "withPath"
+                args:
+                    - "${bolt.hdfs.wip.file.path}"
+    -   id: "messageField"
+        className: "backtype.storm.tuple.Fields"
+        constructorArgs:
+            - ["message"]
+    -   id: "recordFormat"
+        className: "org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat"
+        configMethods:
+            -   name: "withFieldDelimiter"
+                args:
+                    - "${bolt.hdfs.field.delimiter}"
+            -   name: "withFields"
+                args:
+                    - ref: "messageField"
+    -   id: "rotationPolicy"
+        className: "org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy"
+        constructorArgs:
+            - ${bolt.hdfs.file.rotation.size.in.mb}
+            - MB
+    -   id: "syncPolicy"
+        className: "org.apache.storm.hdfs.bolt.sync.CountSyncPolicy"
+        constructorArgs:
+            - ${bolt.hdfs.batch.size}
+    -   id: "moveFileAction"
+        className: "org.apache.storm.hdfs.common.rotation.MoveFileAction"
+        configMethods:
+            -   name: "toDestination"
+                args:
+                    - "${bolt.hdfs.finished.file.path}"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "paloAltoParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "paloalto"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "hdfsBolt"
+        className: "org.apache.storm.hdfs.bolt.HdfsBolt"
+        configMethods:
+            -   name: "withFsUrl"
+                args:
+                    - "${bolt.hdfs.file.system.url}"
+            -   name: "withFileNameFormat"
+                args:
+                    - ref: "filenameFormat"
+            -   name: "withRecordFormat"
+                args:
+                    - ref: "recordFormat"
+            -   name: "withRotationPolicy"
+                args:
+                    - ref: "rotationPolicy"
+            -   name: "withSyncPolicy"
+                args:
+                    - ref: "syncPolicy"
+            -   name: "addRotationAction"
+                args:
+                    - ref: "moveFileAction"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> hdfs"
+        from: "parserBolt"
+        to: "hdfsBolt"
+        grouping:
+            streamId: "message"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology.conf
deleted file mode 100644
index 6ccfed8..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology.conf
+++ /dev/null
@@ -1,113 +0,0 @@
-include = ../../etc/env/environment_common.conf
-include = ../../etc/env/es_connection.conf
-include = ../../etc/env/hdfs_connection.conf
-include = ../../etc/env/mysql_connection.conf
-include = metrics.conf
-include = features_enabled.conf
-
-#Global Properties
-
-debug.mode=true
-local.mode=true
-num.workers=1
-
-#Standard 5-tuple fields
-
-source.ip=ip_src_addr
-source.port=ip_src_port
-dest.ip=ip_dst_addr
-dest.port=ip_dst_port
-protocol=protocol
-
-#Test Spout
-spout.test.parallelism.repeat=false
-
-#Kafka Spout
-spout.kafka.topic=paloalto_raw
-
-#Parser Bolt
-bolt.parser.adapter=org.apache.metron.parsing.parsers.BasicPaloAltoFirewallParser
-
-#Host Enrichment
-
-bolt.enrichment.host.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.host.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.host.enrichment_tag=host
-
-
-#GeoEnrichment
-
-bolt.enrichment.geo.enrichment_tag=geo
-bolt.enrichment.geo.adapter.table=GEO
-bolt.enrichment.geo.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.geo.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.geo.fields=ip_src_addr,ip_dst_addr
-
-#WhoisEnrichment
-
-bolt.enrichment.whois.hbase.table.name=whois
-bolt.enrichment.whois.enrichment_tag=whois
-bolt.enrichment.whois.fields=host
-bolt.enrichment.whois.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.whois.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.whois.fields=host
-
-#CIF Enrichment
-bolt.enrichment.cif.tablename=cif_table
-bolt.enrichment.cif.fields.host=host
-bolt.enrichment.cif.fields.email=email
-bolt.enrichment.cif.fields.ip=ip_src_addr,ip_dst_addr
-bolt.enrichment.cif.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.cif.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.cif.enrichment_tag=cif
-bolt.enrichment.cif.host=host
-
-
-#Threat Enrichment
-bolt.enrichment.threat.tablename=threat_table
-bolt.enrichment.threat.fields=host,ip_src_addr,ip_dst_addr
-bolt.enrichment.threat.MAX_CACHE_SIZE_OBJECTS_NUM=10000
-bolt.enrichment.threat.MAX_TIME_RETAIN_MINUTES=10
-bolt.enrichment.threat.enrichment_tag=threat
-
-#Indexing Bolt
-bolt.indexing.indexname=paloalto_index
-bolt.indexing.timestamp=yyyy.MM.ww
-bolt.indexing.documentname=paloalto_doc
-bolt.indexing.bulk=1
-bolt.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Indexing Bolt
-bolt.alerts.indexing.indexname=alert
-bolt.alerts.indexing.timestamp=yyyy.MM.ww
-bolt.alerts.indexing.documentname=paloalto_alert
-bolt.alerts.indexing.bulk=1
-bolt.alerts.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Error Indexing Bolt
-bolt.error.indexing.indexname=error
-bolt.error.indexing.timestamp.yyyy.MM
-bolt.error.indexing.documentname=paloalto_error
-bolt.error.indexing.bulk=1
-bolt.error.indexing.adapter=org.apache.metron.indexing.adapters.ESTimedRotatingAdapter
-
-#Alerts Bolt
-bolt.alerts.adapter=org.apache.metron.alerts.adapters.CIFAlertsAdapter
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.whitelist_table_name = ip_whitelist
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.blacklist_table_name = ip_blacklist
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.quorum=zkpr1,zkpr2,zkpr3
-org.apache.metron.alerts.adapters.CIFAlertsAdapter.port=2181
-org.apache.metron.alerts.adapters.CIFAlertsAdapter._MAX_CACHE_SIZE_OBJECTS_NUM=3600
-org.apache.metron.alerts.adapters.CIFAlertsAdapter._MAX_TIME_RETAIN_MINUTES=1000
-
-#HDFS Bolt
-bolt.hdfs.batch.size=5000
-bolt.hdfs.field.delimiter=|
-bolt.hdfs.file.rotation.size.in.mb=5
-bolt.hdfs.file.system.url=hdfs://nn1:8020
-bolt.hdfs.wip.file.path=/paloalto/wip
-bolt.hdfs.finished.file.path=/paloalto/rotated
-bolt.hdfs.compression.codec.class=org.apache.hadoop.io.compress.SnappyCodec
-
-#Kafka Bolt
-bolt.kafka.topic=paloalto_enriched
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology_identifier.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology_identifier.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology_identifier.conf
deleted file mode 100644
index 7601122..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/topology_identifier.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-#Each topology must have a unique identifier.  This setting is required
-
-topology.id=paloalto
-instance.id=PA001
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/features_enabled.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/features_enabled.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/features_enabled.conf
deleted file mode 100644
index 9b41fa2..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/features_enabled.conf
+++ /dev/null
@@ -1,118 +0,0 @@
-#Enable and disable features for each topology
-
-#Feature: Test spout 
-##Feature Description: Reads telemetry from file and ingests it into topology.  Used for testing or bulk loading the topology
-
-spout.test.name=TestSpout
-spout.test.enabled=true
-spout.test.num.tasks=1
-spout.test.parallelism.hint=1
-
-#Feature: Kafka spout
-##Feature Description: Acts as a Kafka consumer.  Takes messages from a Kafka topic and ingests them into a topology
-
-spout.kafka.name=KafkaSpout
-spout.kafka.enabled=false
-spout.kafka.num.tasks=1
-spout.kafka.parallelism.hint=1
-
-#Feature: Parser Bolt
-##Feature Description: Parses telemetry from its native format into a native JSON
-
-bolt.parser.name=ParserBolt
-bolt.parser.enabled=true
-bolt.parser.num.tasks=1
-bolt.parser.parallelism.hint=1
-
-#Feature: Host Enrichment
-##Feature Description: Appends information about known hosts to a telemetry message
-
-bolt.enrichment.host.name=HostEnrichment
-bolt.enrichment.host.enabled=false
-bolt.enrichment.host.num.tasks=1
-bolt.enrichment.host.parallelism.hint=1
-
-#Feature: Geo Enrichment
-##Feature Description: Appends geo information about known non-local IPs to a telemetry message
-
-bolt.enrichment.geo.name=GeoEnrichment 
-bolt.enrichment.geo.enabled=false
-bolt.enrichment.geo.num.tasks=1
-bolt.enrichment.geo.parallelism.hint=1
-
-#Feature: Whois Enrichment
-##Feature Description: Appends whois information about known domains to a telemetry message
-
-bolt.enrichment.whois.name=WhoisEnrichment
-bolt.enrichment.whois.enabled=false
-bolt.enrichment.whois.num.tasks=1
-bolt.enrichment.whois.parallelism.hint=1
-
-#Feature: CIF Enrichment
-##Feature Description: Appends information from CIF threat intelligence feeds to a telemetry message
-
-bolt.enrichment.cif.name=SIFBolt
-bolt.enrichment.cif.enabled=false
-bolt.enrichment.cif.num.tasks=1
-bolt.enrichment.cif.parallelism.hint=1
-
-#Feature: Threat Enrichment
-##Feature Description: Appends information from Threat intelligence feeds to a telemetry message
-
-bolt.enrichment.threat.name=ThreatBolt
-bolt.enrichment.threat.enabled=false
-bolt.enrichment.threat.num.tasks=1
-bolt.enrichment.threat.parallelism.hint=1
-
-#Feature: Rules-Based Alerts
-##Feature Description: Tags messages with rules-based alerts
-
-bolt.alerts.name=Alerts
-bolt.alerts.enabled=false
-bolt.alerts.num.tasks=1
-bolt.alerts.parallelism.hint=1
-
-#Feature: Indexer
-##Feature Description: Indexes telemetry messages in ElasticSearch or Solr
-
-bolt.indexing.name=IndexBolt
-bolt.indexing.enabled=true
-bolt.indexing.num.tasks=1
-bolt.indexing.parallelism.hint=1
-
-#Feature: Alerts Indexer
-##Feature Description: Indexes alert messages in ElasticSearch or Solr
-
-bolt.alerts.indexing.name=AlertIndexBolt
-bolt.alerts.indexing.enabled=false
-bolt.alerts.indexing.num.tasks=1
-bolt.alerts.indexing.parallelism.hint=1
-
-#Feature: Error Indexer
-##Feature Description: Indexes error messages in ElasticSearch or Solr
-
-bolt.error.indexing.name=ErrorIndexBolt
-bolt.error.indexing.enabled=true
-bolt.error.indexing.num.tasks=1
-bolt.error.indexing.parallelism.hint=1
-
-#Feature: Kafka Bolt
-##Feature Description: Writes telemetry messages back into a Kafka topic
-
-bolt.kafka.name=KafkaBolt
-bolt.kafka.enabled=false
-bolt.kafka.num.tasks=1
-bolt.kafka.parallelism.hint=1
-
-#Feature: HDFS Bolt
-##Feature Description: Writes telemetry messages into HDFS
-
-bolt.hdfs.name=HDFSBolt
-bolt.hdfs.enabled=true
-bolt.hdfs.num.tasks=1
-bolt.hdfs.parallelism.hint=1
-
-bolt.hbase.name=HBaseBolt
-bolt.hbase.enabled=true
-bolt.hbase.num.tasks=1
-bolt.hbase.parallelism.hint=1
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml
new file mode 100644
index 0000000..49c4bf2
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/local.yaml
@@ -0,0 +1,171 @@
+name: "pcap-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/PCAPExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.PcapParserBolt"
+        configMethods:
+            -   name: "withTsPrecision"
+                args: ["MICRO"]
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "pcap_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "pcap_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "pcap_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "testingSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/metrics.conf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/metrics.conf b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/metrics.conf
deleted file mode 100644
index aa7a6e0..0000000
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/metrics.conf
+++ /dev/null
@@ -1,26 +0,0 @@
-#reporters
-org.apache.metron.metrics.reporter.graphite=true
-org.apache.metron.metrics.reporter.console=false
-org.apache.metron.metrics.reporter.jmx=false
-
-#Graphite Addresses
-
-org.apache.metron.metrics.graphite.address=localhost
-org.apache.metron.metrics.graphite.port=2023
-
-#TelemetryParserBolt
-org.apache.metron.metrics.TelemetryParserBolt.acks=true
-org.apache.metron.metrics.TelemetryParserBolt.emits=true
-org.apache.metron.metrics.TelemetryParserBolt.fails=true
-
-
-#GenericEnrichmentBolt
-org.apache.metron.metrics.GenericEnrichmentBolt.acks=true
-org.apache.metron.metrics.GenericEnrichmentBolt.emits=true
-org.apache.metron.metrics.GenericEnrichmentBolt.fails=true
-
-
-#TelemetryIndexingBolt
-org.apache.metron.metrics.TelemetryIndexingBolt.acks=true
-org.apache.metron.metrics.TelemetryIndexingBolt.emits=true
-org.apache.metron.metrics.TelemetryIndexingBolt.fails=true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/b69e3037/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
new file mode 100644
index 0000000..e3a130c
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
@@ -0,0 +1,185 @@
+name: "pcap"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.PcapParserBolt"
+        configMethods:
+            -   name: "withTsPrecision"
+                args: ["MICRO"]
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "pcap_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "pcap_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "pcap_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE


Mime
View raw message